Network Discovery support for network

Issue-ID: SDNC-475
Add support for Openstack network object retrieval.
Moved the attribute translations into Jolt transformations.

Change-Id: I442bfe39a5b84050d7a7fa8b50aaf831e4dd05d0
Signed-off-by: Norm Traxler <normant@amdocs.com>

9 files changed, 191 insertions, 178 deletions

diff --git a/pomba/network-discovery/config/application.properties b/pomba/network-discovery/config/application.properties
index 1032d42..b7d41b7 100644
--- a/pomba/network-discovery/config/application.properties
+++ b/pomba/network-discovery/config/application.properties
@@ -15,13 +15,13 @@ server.ssl.enabled-protocols=TLSv1.1,TLSv1.2
 basicAuth.username=admin
 basicAuth.password=OBF:1u2a1toa1w8v1tok1u30
 
-openstack.identity.url = http://10.69.36.11:5000/v3/auth/tokens
-openstack.identity.user = ralph
-openstack.identity.password = OBF:1w951ugg1vun1uha1w8l
+openstack.identity.url = http://10.69.100.11:5000/v3/auth/tokens
+openstack.identity.user = jbalasub
+openstack.identity.password = OBF:1thj1vn01v1p1toq1to41v2p1vo21th3
 
 openstack.types = vserver, l3-network
-openstack.type.vserver.url    = http://10.69.36.11:8774/v2.1/servers/{0}
-openstack.type.l3-network.url = http://10.69.36.11:9696/v2.0/networks/{0}
+openstack.type.vserver.url    = http://10.69.100.11:8774/v2.1/servers/{0}
+openstack.type.l3-network.url = http://10.69.100.11:9696/v2.0/networks/{0}
 
 openstack.api.microversion = 2.42
 
diff --git a/pomba/network-discovery/config/jolt/l3-network.json b/pomba/network-discovery/config/jolt/l3-network.json
index 0afe4a2..77fa4aa 100644
--- a/pomba/network-discovery/config/jolt/l3-network.json
+++ b/pomba/network-discovery/config/jolt/l3-network.json
@@ -3,23 +3,36 @@
     "operation": "shift",
     "spec": {
       "network": {
-        "id": "l3-network.id",
-        "name": "l3-network.name",
-        "admin_state_up": "l3-network.AdminState",
-        "shared": "l3-network.sharedNetwork",
-        "status": "l3-network.status",
-        "subnets": "l3-network.subnets",
-        "tenant_id": "l3-network.tenantId"
+        "id": "id",
+        "name": "name",
+        "admin_state_up": "tempList[].adminState",
+        "shared": "tempList[].sharedNetwork",
+        "status": "tempList[].status",
+        "subnets" : {
+          "*": "tempList[].subnets"
+         },
+        "tenant_id" : "tempList[].tenantId"
       }
     }
   },
   {
-    "operation": "default",
+    "operation": "shift",
     "spec": {
+      "tempList": {
+        "*": {
+        	"*" : {
+        	   "$": "attributeList[#3].name",
+        	   "@": "attributeList[#3].value"
+        	},
+        	"#ok": "attributeList[#2].dataQuality.status"
+        }
+      },
       "*": {
-        "host-status": "UNKNOWN"
-      }
-    }
-  }
+         "@" : "&"
+      },
+      "#l3-network": "type",
+      "#ok": "dataQuality.status"
+     }
+   }
 
-]
+]
\ No newline at end of file
diff --git a/pomba/network-discovery/config/jolt/vserver.json b/pomba/network-discovery/config/jolt/vserver.json
index cc9e26d..1363be9 100644
--- a/pomba/network-discovery/config/jolt/vserver.json
+++ b/pomba/network-discovery/config/jolt/vserver.json
@@ -3,30 +3,42 @@
     "operation": "shift",
     "spec": {
       "server": {
-        "id": "vserver.id",
-        "name": "vserver.name",
-        "locked":  "vserver.inMaintenance",
-        "OS-EXT-SRV-ATTR:hostname":  "vserver.hostname",
+        "id": "id",
+        "name": "name",
+        "locked":   "tempList[].inMaintenance",
+        "OS-EXT-SRV-ATTR:hostname": "tempList[].hostname",
         "image": {
-            "id": "vserver.imageId"
-        },
-        
-        "status": "vserver.status",
-        "OS-EXT-STS:vm_state": "vserver.vmState",
-        "tenant_id": "vserver.tenantId",
-        "hostId": "vserver.hostId",
-        "OS-EXT-SRV-ATTR:host": "vserver.host",      
-        "host_status": "vserver.hostStatus"
+          "id": "tempList[].imageId"
+         },
+        "status":  "tempList[].status",
+        "OS-EXT-STS:vm_state" : "tempList[].vmState",
+        "user_id" : "tempList[].userId",
+        "tenant_id" : "tempList[].tenantId",
+        "hostId" : "tempList[].hostId",
+        "OS-EXT-SRV-ATTR:host" : "tempList[].host",
+        "host_status" : "tempList[].hostStatus"
       }
     }
-  },
+  }
+  ,
   {
-    "operation": "default",
+    "operation": "shift",
     "spec": {
+      "tempList": {
+        "*": {
+        	"*" : {
+        	   "$": "attributeList[#3].name",
+        	   "@": "attributeList[#3].value"
+        	},
+        	"#ok": "attributeList[#2].dataQuality.status"
+        }
+      },
       "*": {
-        "hostStatus": "UNKNOWN"
-      }
-    }
-  }
+         "@" : "&"
+      },
+      "#vserver": "type",
+      "#ok": "dataQuality.status"
+     }
+   }
 
 ]
diff --git a/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/SpringServiceImpl.java b/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/SpringServiceImpl.java
index 5577398..5e67171 100644
--- a/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/SpringServiceImpl.java
+++ b/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/SpringServiceImpl.java
@@ -17,6 +17,8 @@
  */
 package org.onap.sdnc.apps.pomba.networkdiscovery.service;
 
+import com.google.gson.Gson;
+
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.List;
@@ -32,7 +34,6 @@ import javax.ws.rs.core.Response.Status.Family;
 import org.onap.logging.ref.slf4j.ONAPLogAdapter;
 import org.onap.pomba.common.datatypes.DataQuality;
 import org.onap.sdnc.apps.pomba.networkdiscovery.ApplicationException;
-import org.onap.sdnc.apps.pomba.networkdiscovery.datamodel.Attribute;
 import org.onap.sdnc.apps.pomba.networkdiscovery.datamodel.NetworkDiscoveryNotification;
 import org.onap.sdnc.apps.pomba.networkdiscovery.datamodel.NetworkDiscoveryResponse;
 import org.onap.sdnc.apps.pomba.networkdiscovery.datamodel.Resource;
@@ -87,12 +88,7 @@ public class SpringServiceImpl implements SpringService {
         MessageFormat format = new MessageFormat(openstackURL);
         
         for (String resourceId : resourceIds) {
-            String url = format.format(new Object[] { resourceId });
-            Resource resource = new Resource();
-            resource.setType(resourceType);
-            resource.setId(resourceId);
-            resources.add(resource);
-        
+            String url = format.format(new Object[] { resourceId });        
             Response result;
             try {
                 result = openstackClient.target(url).request().header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON)
@@ -111,14 +107,17 @@ public class SpringServiceImpl implements SpringService {
 
             if (result.getStatusInfo().getFamily() == Family.SUCCESSFUL) {
                 String transformedOutput = TransformationUtil.transform(jsonResult, resourceType);
-        
                 log.debug("Jolt transformed output: {}", transformedOutput);
-        
-                resource.setDataQuality(DataQuality.ok());
-                List<Attribute> attributeList = TransformationUtil.toAttributeList(transformedOutput);
-                resource.setAttributeList(attributeList);
+
+                Gson gson = new Gson();
+                Resource resourceInst = gson.fromJson(transformedOutput, Resource.class);
+                resources.add(resourceInst);
             } else {
+                Resource resource = new Resource();
+                resource.setType(resourceType);
+                resource.setId(resourceId);
                 resource.setDataQuality(DataQuality.error(jsonResult));
+                resources.add(resource);
             }
         }
 
diff --git a/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/util/TransformationUtil.java b/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/util/TransformationUtil.java
index 0984f66..32b213f 100644
--- a/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/util/TransformationUtil.java
+++ b/pomba/network-discovery/src/main/java/org/onap/sdnc/apps/pomba/networkdiscovery/service/util/TransformationUtil.java
@@ -17,23 +17,13 @@ package org.onap.sdnc.apps.pomba.networkdiscovery.service.util;
 

 import com.bazaarvoice.jolt.Chainr;

 import com.bazaarvoice.jolt.JsonUtils;

-import com.google.gson.JsonElement;

-import com.google.gson.JsonParser;

 

 import java.io.File;

-import java.util.ArrayList;

-import java.util.Iterator;

 import java.util.List;

-import java.util.Map.Entry;

-import java.util.Set;

-

-import org.onap.pomba.common.datatypes.DataQuality;

-import org.onap.sdnc.apps.pomba.networkdiscovery.datamodel.Attribute;

 

 public class TransformationUtil {

 

     private static final String CONFIG_JOLT = "config/jolt";

-    private static final String EMPTY_STRING = "";

 

     private TransformationUtil() {

         throw new IllegalStateException("Utility class");

@@ -57,43 +47,4 @@ public class TransformationUtil {
         return JsonUtils.toJsonString(output);

     }

 

-    /**

-     * Converts the second level of JsonElements from the given json to a list of

-     * Network Discovery Attributes.

-     * 

-     * @param json

-     * @return

-     */

-    public static List<Attribute> toAttributeList(String json) {

-

-        JsonParser parser = new JsonParser();

-        JsonElement elem = parser.parse(json);

-

-        Set<Entry<String, JsonElement>> entrySet = elem.getAsJsonObject().entrySet();

-

-        List<Attribute> result = new ArrayList<>();

-

-        Iterator<Entry<String, JsonElement>> iter = entrySet.iterator();

-        while (iter.hasNext()) {

-            Entry<String, JsonElement> next = iter.next();

-

-            JsonElement vserverElem = next.getValue();

-            Set<Entry<String, JsonElement>> vserverEntrySet = vserverElem.getAsJsonObject().entrySet();

-            Iterator<Entry<String, JsonElement>> vserverIter = vserverEntrySet.iterator();

-            while (vserverIter.hasNext()) {

-                Entry<String, JsonElement> vserverNext = vserverIter.next();

-                Attribute attr = new Attribute();

-                attr.setName(vserverNext.getKey());

-                if (vserverNext.getValue().isJsonNull()) {

-                    attr.setValue(EMPTY_STRING);

-                } else {

-                    attr.setValue(vserverNext.getValue().getAsString());

-                }

-                attr.setDataQuality(DataQuality.ok());

-                result.add(attr);

-            }

-        }

-        return result;

-    }

-

 }
\ No newline at end of file
diff --git a/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/NetworkDiscoveryTest.java b/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/NetworkDiscoveryTest.java
index 31b1abe..39239e9 100644
--- a/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/NetworkDiscoveryTest.java
+++ b/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/NetworkDiscoveryTest.java
@@ -213,7 +213,7 @@ public class NetworkDiscoveryTest {
 
     @Test
     public void testDiscoverVserver() throws Exception {
-        String vserverId = UUID.randomUUID().toString();
+        String vserverId = "2c311eae-f542-4173-8a01-582922abd495";
 
         String resourcePath = MessageFormat.format("/v2.1/servers/{0}",
                 new Object[] { vserverId });
diff --git a/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/util/TransformationUtilTest.java b/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/util/TransformationUtilTest.java
index dc4775f..f43fa60 100644
--- a/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/util/TransformationUtilTest.java
+++ b/pomba/network-discovery/src/test/java/org/onap/sdnc/apps/pomba/networkdiscovery/unittest/service/util/TransformationUtilTest.java
@@ -18,21 +18,15 @@
 

 package org.onap.sdnc.apps.pomba.networkdiscovery.unittest.service.util;

 

-import static org.junit.Assert.assertThat;

-

 import com.bazaarvoice.jolt.JsonUtils;

 import com.bazaarvoice.jolt.exception.JsonUnmarshalException;

+import com.google.gson.Gson;

 

-import java.util.HashMap;

-import java.util.List;

-import java.util.Map;

-

-import org.hamcrest.CoreMatchers;

 import org.junit.Assert;

 import org.junit.Rule;

 import org.junit.Test;

 import org.junit.rules.ExpectedException;

-import org.onap.sdnc.apps.pomba.networkdiscovery.datamodel.Attribute;

+import org.onap.sdnc.apps.pomba.networkdiscovery.datamodel.Resource;

 import org.onap.sdnc.apps.pomba.networkdiscovery.service.util.TransformationUtil;

 

 public class TransformationUtilTest {

@@ -49,11 +43,15 @@ public class TransformationUtilTest {
 

         String resultJson = TransformationUtil.transform(JsonUtils.toJsonString(sourceObject), "vserver");

 

+        Gson gson = new Gson();

+        Resource resourceInst = gson.fromJson(resultJson, Resource.class);

+        String resourceInstJson = gson.toJson(resourceInst);

+

         Object expectedObject = JsonUtils.filepathToObject(TEST_RESOURCES + "vserver-expected.json");

 

         Assert.assertEquals("Json transformation result does not match expected content",

                 JsonUtils.toPrettyJsonString(expectedObject),

-                JsonUtils.toPrettyJsonString(JsonUtils.jsonToObject(resultJson)));

+                JsonUtils.toPrettyJsonString(JsonUtils.jsonToObject(resourceInstJson)));

 

     }

 

@@ -63,11 +61,15 @@ public class TransformationUtilTest {
         Object sourceObject = JsonUtils.filepathToObject(TEST_RESOURCES + "l3network-input.json");

         String resultJson = TransformationUtil.transform(JsonUtils.toJsonString(sourceObject), "l3-network");

 

+        Gson gson = new Gson();

+        Resource resourceInst = gson.fromJson(resultJson, Resource.class);

+        String resourceInstJson = gson.toJson(resourceInst);

+

         Object expectedObject = JsonUtils.filepathToObject(TEST_RESOURCES + "l3network-expected.json");

 

         Assert.assertEquals("Json transformation result does not match expected content",

                 JsonUtils.toPrettyJsonString(expectedObject),

-                JsonUtils.toPrettyJsonString(JsonUtils.jsonToObject(resultJson)));

+                JsonUtils.toPrettyJsonString(JsonUtils.jsonToObject(resourceInstJson)));

 

     }

 

@@ -89,47 +91,4 @@ public class TransformationUtilTest {
         TransformationUtil.transform("xxx", "foobar");

     }

 

-    @Test

-    public void testToAttributeList() {

-        Map<String, String> expectedAttributes = new HashMap<String, String>();

-        expectedAttributes.put("name", "norm_bouygues");

-        expectedAttributes.put("hostId", "ea1660efbbedda164379afacdc622305c4b88cebfb84119472d286a8");

-        expectedAttributes.put("hostStatus", "UP");

-        expectedAttributes.put("id", "2c311eae-f542-4173-8a01-582922abd495");

-        expectedAttributes.put("status", "ACTIVE");

-        expectedAttributes.put("vmState", "active");

-        expectedAttributes.put("hostname", "norm-bouygues");

-        expectedAttributes.put("inMaintenance", "true");

-        expectedAttributes.put("imageId", "c0022890-d91f-422c-91c5-3866edeae768");

-        expectedAttributes.put("tenantId", "15ad36d394e744838e947ca90609f805");

-        expectedAttributes.put("host", "Setup-NCSO-OTT-E-C2");

-

-        Object inputJson = JsonUtils.filepathToObject(TEST_RESOURCES + "vserver-expected.json");

-        List<Attribute> resultAttributeList = TransformationUtil.toAttributeList(JsonUtils.toJsonString(inputJson));

-

-        Map<String, String> resultAttributes = new HashMap<>();

-

-        for (Attribute attribute : resultAttributeList) {

-            resultAttributes.put(attribute.getName(), attribute.getValue());

-        }

-        assertThat(expectedAttributes, CoreMatchers.is(resultAttributes));

-    }

-

-    @Test

-    public void testToAttributeListNullJsonValue() {

-        Map<String, String> expectedAttributes = new HashMap<String, String>();

-        expectedAttributes.put("name", "");

-

-        String inputJson = "{\"server\": { \"name\": null }}";

-

-        List<Attribute> resultAttributeList = TransformationUtil.toAttributeList(inputJson);

-

-        Map<String, String> resultAttributes = new HashMap<>();

-

-        for (Attribute attribute : resultAttributeList) {

-            resultAttributes.put(attribute.getName(), attribute.getValue());

-        }

-        assertThat(expectedAttributes, CoreMatchers.is(resultAttributes));

-    }

-

 }

diff --git a/pomba/network-discovery/src/test/resources/jolt/l3network-expected.json b/pomba/network-discovery/src/test/resources/jolt/l3network-expected.json
index be4ad9d..e9e37dd 100644
--- a/pomba/network-discovery/src/test/resources/jolt/l3network-expected.json
+++ b/pomba/network-discovery/src/test/resources/jolt/l3network-expected.json
@@ -1,14 +1,39 @@
 {

-	"l3-network": {

-		"id": "8df84f0a-737a-4628-be9c-c3c78454f9d9",

-		"name": "NET_503",

-		"AdminState": true,

-		"sharedNetwork": true,

-		"status": "ACTIVE",

-		"subnets": [

-			"d0e6ecad-7bc9-4138-b9e8-e742bdf9afd5"

-		],

-		"tenantId": "c44b872f6830498b88c4989d67b2a6b7",

-		"host-status": "UNKNOWN"

-	}

-}

+  "id" : "8df84f0a-737a-4628-be9c-c3c78454f9d9",

+  "name" : "NET_503",

+  "type" : "l3-network",

+  "dataQuality" : {

+    "status" : "ok"

+  },

+  "attributeList" : [ {

+    "name" : "adminState",

+    "value" : "true",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "sharedNetwork",

+    "value" : "true",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "status",

+    "value" : "ACTIVE",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "subnets",

+    "value" : "d0e6ecad-7bc9-4138-b9e8-e742bdf9afd5",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "tenantId",

+    "value" : "c44b872f6830498b88c4989d67b2a6b7",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  } ]

+}
\ No newline at end of file
diff --git a/pomba/network-discovery/src/test/resources/jolt/vserver-expected.json b/pomba/network-discovery/src/test/resources/jolt/vserver-expected.json
index 53d4a79..2ad9f81 100644
--- a/pomba/network-discovery/src/test/resources/jolt/vserver-expected.json
+++ b/pomba/network-discovery/src/test/resources/jolt/vserver-expected.json
@@ -1,15 +1,69 @@
 {

-  "vserver" : {

-    "id" : "2c311eae-f542-4173-8a01-582922abd495",

-    "name" : "norm_bouygues",

-    "inMaintenance" : true,

-    "hostname" : "norm-bouygues",

-    "imageId" : "c0022890-d91f-422c-91c5-3866edeae768",

-    "status" : "ACTIVE",

-    "vmState" : "active",

-    "tenantId" : "15ad36d394e744838e947ca90609f805",

-    "hostId" : "ea1660efbbedda164379afacdc622305c4b88cebfb84119472d286a8",

-    "host" : "Setup-NCSO-OTT-E-C2",

-    "hostStatus" : "UP"

-  }

+  "id" : "2c311eae-f542-4173-8a01-582922abd495",

+  "name" : "norm_bouygues",

+  "type" : "vserver",

+  "dataQuality" : {

+    "status" : "ok"

+  },

+  "attributeList" : [ {

+    "name" : "inMaintenance",

+    "value" : "true",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "hostname",

+    "value" : "norm-bouygues",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "imageId",

+    "value" : "c0022890-d91f-422c-91c5-3866edeae768",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "status",

+    "value" : "ACTIVE",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "vmState",

+    "value" : "active",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "userId",

+    "value" : "bdaf4fcde1574450bfa488b3221033d8",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "tenantId",

+    "value" : "15ad36d394e744838e947ca90609f805",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "hostId",

+    "value" : "ea1660efbbedda164379afacdc622305c4b88cebfb84119472d286a8",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "host",

+    "value" : "Setup-NCSO-OTT-E-C2",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  }, {

+    "name" : "hostStatus",

+    "value" : "UP",

+    "dataQuality" : {

+      "status" : "ok"

+    }

+  } ]

 }
\ No newline at end of file