Support ':' in Apex Event Avro schema fieldnames

Similar to '.' (_DoT_) and '-' (_Dash_), the ':' (_ColoN_) character
can now be used in Apex Event Field names specified using Avro Schema

Change-Id: I320058441a1a1a544b9f1619e45c96e71e5aa9e3
Signed-off-by: JohnKeeney <john.keeney@est.tech>
Issue-ID: POLICY-3301
Signed-off-by: JohnKeeney <john.keeney@est.tech>
(cherry picked from commit fe40844bba4a94e26efc0f1c57d62c97e35bfc79)

4 files changed, 124 insertions, 10 deletions

diff --git a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/main/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaKeyTranslationUtilities.java b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/main/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaKeyTranslationUtilities.java
index 965457206..6229c066e 100644
--- a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/main/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaKeyTranslationUtilities.java
+++ b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/main/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaKeyTranslationUtilities.java
@@ -38,6 +38,8 @@ public final class AvroSchemaKeyTranslationUtilities {
     private static final String DOT_STRING_REPLACEMENT = "_DoT_";
     private static final String DASH_STRING = "-";
     private static final String DASH_STRING_REPLACEMENT = "_DasH_";
+    private static final String COLON_STRING = ":";
+    private static final String COLON_STRING_REPLACEMENT = "_ColoN_";
 
     /**
      * Default constructor to avoid subclassing.
@@ -134,9 +136,13 @@ public final class AvroSchemaKeyTranslationUtilities {
      */
     private static String translateIllegalKey(final String key, final boolean revert) {
         if (revert) {
-            return key.replace(DOT_STRING_REPLACEMENT, DOT_STRING).replace(DASH_STRING_REPLACEMENT, DASH_STRING);
+            return key.replace(DOT_STRING_REPLACEMENT, DOT_STRING)
+                    .replace(DASH_STRING_REPLACEMENT, DASH_STRING)
+                    .replace(COLON_STRING_REPLACEMENT, COLON_STRING);
         } else {
-            return key.replace(DOT_STRING, DOT_STRING_REPLACEMENT).replace(DASH_STRING, DASH_STRING_REPLACEMENT);
+            return key.replace(DOT_STRING, DOT_STRING_REPLACEMENT)
+                    .replace(DASH_STRING, DASH_STRING_REPLACEMENT)
+                    .replace(COLON_STRING, COLON_STRING_REPLACEMENT);
         }
     }
 }
diff --git a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaMapTest.java b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaMapTest.java
index 690234b47..abc9335ae 100644
--- a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaMapTest.java
+++ b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/java/org/onap/policy/apex/plugins/context/schema/avro/AvroSchemaMapTest.java
@@ -22,6 +22,9 @@
 package org.onap.policy.apex.plugins.context.schema.avro;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
 
 import java.io.File;
 import java.io.IOException;
@@ -31,6 +34,7 @@ import org.apache.avro.util.Utf8;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.onap.policy.apex.context.ContextRuntimeException;
 import org.onap.policy.apex.context.SchemaHelper;
 import org.onap.policy.apex.context.impl.schema.SchemaHelperFactory;
 import org.onap.policy.apex.context.parameters.ContextParameterConstants;
@@ -80,7 +84,6 @@ public class AvroSchemaMapTest {
         schemaParameters.setName(ContextParameterConstants.SCHEMA_GROUP_NAME);
         schemaParameters.getSchemaHelperParameterMap().put("AVRO", new AvroSchemaHelperParameters());
         ParameterService.register(schemaParameters);
-
     }
 
     /**
@@ -92,6 +95,100 @@ public class AvroSchemaMapTest {
     }
 
     /**
+     * Test valid schemas with substitutions.
+     *
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    @Test
+    public void testValidSubstitutions() throws IOException {
+        final String subst1 = "{\"type\":\"record\",\"name\":\"Subst1\","
+                + "\"fields\":[{\"name\": \"A_DasH_B\",\"type\":\"string\"}]}";
+        final AxContextSchema avroSubstSchema1 = new AxContextSchema(
+                new AxArtifactKey("AvroSubst1", "0.0.1"), "AVRO", subst1);
+        schemas.getSchemasMap().put(avroSubstSchema1.getKey(), avroSubstSchema1);
+
+        SchemaHelper schemaHelperSubst1 = new SchemaHelperFactory()
+                .createSchemaHelper(testKey, avroSubstSchema1.getKey());
+        final GenericRecord subst1A = (GenericRecord) schemaHelperSubst1.unmarshal("{\"A-B\":\"foo\"}");
+        assertEquals(new Utf8("foo"), subst1A.get("A_DasH_B"));
+        assertNull(subst1A.get("A-B"));
+        final Throwable exception1 = assertThrows(ContextRuntimeException.class,
+                () -> schemaHelperSubst1.unmarshal("{\"A-B\":123}"));
+        assertNotNull(exception1.getCause());
+        assertEquals("Expected string. Got VALUE_NUMBER_INT", exception1.getCause().getMessage());
+
+        final String subst2 = "{\"type\":\"record\",\"name\":\"Subst2\","
+                + "\"fields\":[{\"name\": \"C_DoT_D\",\"type\":\"int\"}]}";
+        final AxContextSchema avroSubstSchema2 = new AxContextSchema(
+                new AxArtifactKey("AvroSubst2", "0.0.1"), "AVRO", subst2);
+        schemas.getSchemasMap().put(avroSubstSchema2.getKey(), avroSubstSchema2);
+
+        final SchemaHelper schemaHelperSubst2 = new SchemaHelperFactory()
+                .createSchemaHelper(testKey, avroSubstSchema2.getKey());
+        final GenericRecord subst2A = (GenericRecord) schemaHelperSubst2.unmarshal("{\"C.D\":123}");
+        assertEquals(123, subst2A.get("C_DoT_D"));
+        assertNull(subst2A.get("C.D"));
+        final Throwable exception2 = assertThrows(ContextRuntimeException.class,
+                () -> schemaHelperSubst2.unmarshal("{\"C_DoT_D\":\"bar\"}"));
+        assertNotNull(exception2.getCause());
+        assertEquals("Expected int. Got VALUE_STRING", exception2.getCause().getMessage());
+
+        final String subst3 = "{\"type\":\"record\",\"name\":\"Subst3\","
+                + "\"fields\":[{\"name\": \"E_ColoN_F\",\"type\":\"boolean\"}]}";
+        final AxContextSchema avroSubstSchema3 = new AxContextSchema(
+                new AxArtifactKey("AvroSubst3", "0.0.1"), "AVRO", subst3);
+        schemas.getSchemasMap().put(avroSubstSchema3.getKey(), avroSubstSchema3);
+
+        final SchemaHelper schemaHelperSubst3 = new SchemaHelperFactory()
+                .createSchemaHelper(testKey, avroSubstSchema3.getKey());
+        final GenericRecord subst3A = (GenericRecord) schemaHelperSubst3.unmarshal("{\"E:F\":true}");
+        assertEquals(true, subst3A.get("E_ColoN_F"));
+        assertNull(subst3A.get("E:F"));
+        final Throwable exception3 = assertThrows(ContextRuntimeException.class,
+                () -> schemaHelperSubst3.unmarshal("{\"E_ColoN_F\":\"gaz\"}"));
+        assertNotNull(exception3.getCause());
+        assertEquals("Expected boolean. Got VALUE_STRING", exception3.getCause().getMessage());
+    }
+
+    /**
+     * Test invalid schemas without substitutions.
+     *
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    @Test
+    public void testInValidSubstitutions() throws IOException {
+        final String fail1 = "{\"type\":\"record\",\"name\":\"Fail1\","
+                + "\"fields\":[{\"name\": \"A-B\",\"type\":\"string\"}]}";
+        final AxContextSchema avroFailSchema1 = new AxContextSchema(
+                new AxArtifactKey("AvroFail1", "0.0.1"), "AVRO", fail1);
+        schemas.getSchemasMap().put(avroFailSchema1.getKey(), avroFailSchema1);
+        final Throwable exception1 = assertThrows(ContextRuntimeException.class,
+                () -> new SchemaHelperFactory().createSchemaHelper(testKey, avroFailSchema1.getKey()));
+        assertNotNull(exception1.getCause());
+        assertEquals("Illegal character in: A-B", exception1.getCause().getMessage());
+
+        final String fail2 = "{\"type\":\"record\",\"name\":\"Fail2\","
+                + "\"fields\":[{\"name\": \"C.D\",\"type\":\"int\"}]}";
+        final AxContextSchema avroFailSchema2 = new AxContextSchema(
+                new AxArtifactKey("AvroFail2", "0.0.1"), "AVRO", fail2);
+        schemas.getSchemasMap().put(avroFailSchema2.getKey(), avroFailSchema2);
+        final Throwable exception2 = assertThrows(ContextRuntimeException.class,
+                () -> new SchemaHelperFactory().createSchemaHelper(testKey, avroFailSchema2.getKey()));
+        assertNotNull(exception2.getCause());
+        assertEquals("Illegal character in: C.D", exception2.getCause().getMessage());
+
+        final String fail3 = "{\"type\":\"record\",\"name\":\"Fail3\","
+                + "\"fields\":[{\"name\": \"E:F\",\"type\":\"boolean\"}]}";
+        final AxContextSchema avroFailSchema3 = new AxContextSchema(
+                new AxArtifactKey("AvroFail3", "0.0.1"), "AVRO", fail3);
+        schemas.getSchemasMap().put(avroFailSchema3.getKey(), avroFailSchema3);
+        final Throwable exception3 = assertThrows(ContextRuntimeException.class,
+                () -> new SchemaHelperFactory().createSchemaHelper(testKey, avroFailSchema3.getKey()));
+        assertNotNull(exception3.getCause());
+        assertEquals("Illegal character in: E:F", exception3.getCause().getMessage());
+    }
+
+    /**
      * Test map init.
      *
      * @throws IOException Signals that an I/O exception has occurred.
@@ -162,7 +259,7 @@ public class AvroSchemaMapTest {
         final SchemaHelper schemaHelper = new SchemaHelperFactory().createSchemaHelper(testKey, avroSchema.getKey());
 
         GenericRecord subRecord = (GenericRecord) schemaHelper.createNewSubInstance("AddressUSRecord");
-        assertEquals(null, subRecord.get("streetAddress"));
+        assertNull(subRecord.get("streetAddress"));
     }
 
     /**
@@ -179,6 +276,12 @@ public class AvroSchemaMapTest {
         final SchemaHelper schemaHelper = new SchemaHelperFactory().createSchemaHelper(testKey, avroSchema.getKey());
 
         testUnmarshalMarshal(schemaHelper, "src/test/resources/data/MapExampleAddressInvalidFields.json");
+
+        String vals = TextFileUtils.getTextFileAsString("src/test/resources/data/MapExampleAddressInvalidFields.json");
+        final HashMap<?, ?> newMapFull = (HashMap<?, ?>) schemaHelper.createNewInstance(vals);
+        final String expect = "{\"street_DasH_address\": \"Wayne Manor\", \"the_DoT_city\": \"Gotham City\", "
+                + "\"the_ColoN_code\": \"BatCave7\"}";
+        assertEquals(expect, newMapFull.get(new Utf8("address_DoT_3")).toString());
     }
 
     /**
diff --git a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/avsc/MapExampleAddressInvalidFields.avsc b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/avsc/MapExampleAddressInvalidFields.avsc
index ddffb4bbc..86c145bfc 100644
--- a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/avsc/MapExampleAddressInvalidFields.avsc
+++ b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/avsc/MapExampleAddressInvalidFields.avsc
@@ -4,7 +4,8 @@
         "name" : "AddressUSRecord",
         "fields" : [
             {"name": "street_DasH_address", "type": "string"},
-            {"name": "the_DoT_city", "type": "string"}
+            {"name": "the_DoT_city", "type": "string"},
+            {"name": "the_ColoN_code", "type": "string"}
             ]
     }
 }
\ No newline at end of file
diff --git a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/data/MapExampleAddressInvalidFields.json b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/data/MapExampleAddressInvalidFields.json
index 6116db4f3..a1a70e95a 100644
--- a/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/data/MapExampleAddressInvalidFields.json
+++ b/plugins/plugins-context/plugins-context-schema/plugins-context-schema-avro/src/test/resources/data/MapExampleAddressInvalidFields.json
@@ -1,18 +1,22 @@
 {
     "address.0" : {
         "street-address" : "1600 Pennsylvania Avenue",
-        "the.city" : "Washington DC"
+        "the.city" : "Washington DC",
+        "the:code" : "12345"
     },
     "address.1" : {
         "street-address" : "Somewhere",
-        "the.city" : "Over the rainbow"
+        "the.city" : "Over the rainbow",
+        "the:code" : "BlueBird"
     },
     "address.2" : {
         "street-address" : "221 B Baker St.",
-        "the.city" : "London"
+        "the.city" : "London",
+        "the:code" : "NW1"
     },
     "address.3" : {
         "street-address" : "Wayne Manor",
-        "the.city" : "Gotham City"
+        "the.city" : "Gotham City",
+        "the:code" : "BatCave7"
     }
-}
\ No newline at end of file
+}