Adding & Testing method for breaking JSON Data into 'fragments'

Improving Bookstore test model
https://jira.onap.org/browse/CPS-32

Issue-ID: CPS-32

Change-Id: Ie03e03c041233aa908ab55902c1b387f96eb1c2e
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

6 files changed, 324 insertions, 45 deletions

diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/Fragment.java b/cps-service/src/main/java/org/onap/cps/api/impl/Fragment.java
new file mode 100644
index 000000000..252b09e5c
--- /dev/null
+++ b/cps-service/src/main/java/org/onap/cps/api/impl/Fragment.java
@@ -0,0 +1,158 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2020 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.api.impl;
+
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import lombok.Getter;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.model.api.DataSchemaNode;
+import org.opendaylight.yangtools.yang.model.api.Module;
+
+/**
+ * Class to store a Yang Fragment (container or list element).
+ */
+public class Fragment {
+
+    @Getter
+    private String xpath;
+
+    @Getter
+    private final Map<String, Object> attributes = new HashMap<>();
+
+    @Getter
+    private final Module module;
+
+    @Getter
+    private final Fragment parentFragment;
+
+    @Getter
+    private final Set<Fragment> childFragments = new HashSet<>(0);
+
+    private final QName[] qnames;
+
+    private Optional<Set<String>> optionalLeafListNames = Optional.empty();
+
+    /**
+     * Create a root Fragment.
+     *
+     * @param module the Yang module that encompasses this fragment
+     * @param qnames the list of qualified names that points the schema node for this fragment
+     */
+    public static Fragment createRootFragment(final Module module, final QName... qnames) {
+        return new Fragment(null, module, qnames);
+    }
+
+    /**
+     * Create a Child Fragment under a given Parent Fragment.
+     *
+     * @param parentFragment the parent (can be null for 'root' objects)
+     * @param module         the Yang module that encompasses this fragment
+     * @param qnames         the list of qualified names that points the schema node for this fragment
+     */
+    private Fragment(final Fragment parentFragment, final Module module, final QName... qnames) {
+        this.parentFragment = parentFragment;
+        this.module = module;
+        this.qnames = qnames;
+    }
+
+    /**
+     * Create a Child Fragment where the current Fragment is the parent.
+     *
+     * @param qnameChild The Qualified name for the child (relative to the parent)
+     * @return the child fragment
+     */
+    public Fragment createChildFragment(final QName qnameChild) {
+        final QName[] qnamesForChild = Arrays.copyOf(qnames, qnames.length + 1);
+        qnamesForChild[qnamesForChild.length - 1] = qnameChild;
+        final Fragment childFragment = new Fragment(this, module, qnamesForChild);
+        childFragments.add(childFragment);
+        return childFragment;
+    }
+
+    /**
+     * Define a leaf list by providing its name.
+     * The list is not instantiated until the first value is provided
+     *
+     * @param name the name of the leaf list
+     */
+    public void addLeafListName(final String name) {
+        if (optionalLeafListNames.isEmpty()) {
+            optionalLeafListNames = Optional.of(new HashSet<>());
+        }
+        optionalLeafListNames.get().add(name);
+    }
+
+    /**
+     * Add a leaf or leaf list value.
+     * For Leaf lists it is essential to first define the attribute is a leaf list by using addLeafListName method
+     *
+     * @param name  the name of the leaf (or leaf list)
+     * @param value the value of the leaf (or element of leaf list)
+     */
+    public void addLeafValue(final String name, final Object value) {
+        if (optionalLeafListNames.isPresent() && optionalLeafListNames.get().contains(name)) {
+            addLeafListValue(name, value);
+        } else {
+            attributes.put(name, value);
+        }
+    }
+
+    private void addLeafListValue(final String name, final Object value) {
+        if (attributes.containsKey(name)) {
+            final ImmutableList<Object> oldList = (ImmutableList<Object>) attributes.get(name);
+            final List<Object> newList = new ArrayList<>(oldList);
+            newList.add(value);
+            attributes.put(name, ImmutableList.copyOf(newList));
+        } else {
+            attributes.put(name, ImmutableList.of(value));
+        }
+    }
+
+    /**
+     * Get the SchemaNodeIdentifier for this fragment.
+     *
+     * @return the SchemaNodeIdentifier
+     */
+    public String getSchemaNodeIdentifier() {
+        final StringBuilder stringBuilder = new StringBuilder();
+        for (final QName qname : qnames) {
+            stringBuilder.append(qname.getLocalName());
+            stringBuilder.append('/');
+        }
+        return stringBuilder.toString();
+    }
+
+    /**
+     * Get the Optional SchemaNode (model) for this data fragment.
+     *
+     * @return the Optional SchemaNode
+     */
+    public Optional<DataSchemaNode> getSchemaNode() {
+        return module.findDataTreeChild(qnames);
+    }
+}
diff --git a/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java b/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java
index e9757eca0..0f05d7d92 100644
--- a/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java
+++ b/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java
@@ -23,15 +23,26 @@ import com.google.gson.stream.JsonReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.ServiceLoader;
+import java.util.logging.Logger;
+import org.onap.cps.api.impl.Fragment;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.DataContainerNode;
+import org.opendaylight.yangtools.yang.data.api.schema.LeafSetNode;
+import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
+import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.ValueNode;
 import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter;
 import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactory;
 import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactorySupplier;
 import org.opendaylight.yangtools.yang.data.codec.gson.JsonParserStream;
 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNormalizedNodeStreamWriter;
 import org.opendaylight.yangtools.yang.data.impl.schema.NormalizedNodeResult;
+import org.opendaylight.yangtools.yang.model.api.Module;
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 import org.opendaylight.yangtools.yang.model.parser.api.YangParser;
 import org.opendaylight.yangtools.yang.model.parser.api.YangParserException;
@@ -43,6 +54,8 @@ public class YangUtils {
 
     private static final YangParserFactory PARSER_FACTORY;
 
+    private static final Logger LOGGER = Logger.getLogger(YangUtils.class.getName());
+
     private YangUtils() {
         throw new IllegalStateException("Utility class");
     }
@@ -57,14 +70,14 @@ public class YangUtils {
 
     /**
      * Parse a file containing yang modules.
-     * @param yangModelFile  a file containing one or more yang modules
-     *                   (please note the file has to have a .yang extension if not an exception will be thrown)
+     *
+     * @param yangModelFile a file containing one or more yang modules. The file has to have a .yang extension.
      * @return a SchemaContext representing the yang model
-     * @throws IOException when the system as an IO issue
+     * @throws IOException         when the system as an IO issue
      * @throws YangParserException when the file does not contain a valid yang structure
      */
     public static SchemaContext parseYangModelFile(final File yangModelFile) throws IOException, YangParserException {
-        YangTextSchemaSource yangTextSchemaSource = YangTextSchemaSource.forFile(yangModelFile);
+        final YangTextSchemaSource yangTextSchemaSource = YangTextSchemaSource.forFile(yangModelFile);
         final YangParser yangParser = PARSER_FACTORY
                 .createParser(StatementParserMode.DEFAULT_MODE);
         yangParser.addSource(yangTextSchemaSource);
@@ -73,18 +86,19 @@ public class YangUtils {
 
     /**
      * Parse a file containing json data for a certain model (schemaContext).
-     * @param jsonData a string containing json data for the given model
+     *
+     * @param jsonData      a string containing json data for the given model
      * @param schemaContext the SchemaContext for the given data
      * @return the NormalizedNode representing the json data
      */
     public static NormalizedNode<?, ?> parseJsonData(final String jsonData, final SchemaContext schemaContext)
             throws IOException {
-        JSONCodecFactory jsonCodecFactory = JSONCodecFactorySupplier.DRAFT_LHOTKA_NETMOD_YANG_JSON_02
+        final JSONCodecFactory jsonCodecFactory = JSONCodecFactorySupplier.DRAFT_LHOTKA_NETMOD_YANG_JSON_02
                 .getShared(schemaContext);
         final NormalizedNodeResult normalizedNodeResult = new NormalizedNodeResult();
         final NormalizedNodeStreamWriter normalizedNodeStreamWriter = ImmutableNormalizedNodeStreamWriter
                 .from(normalizedNodeResult);
-        try (JsonParserStream jsonParserStream = JsonParserStream
+        try (final JsonParserStream jsonParserStream = JsonParserStream
                 .create(normalizedNodeStreamWriter, jsonCodecFactory)) {
             final JsonReader jsonReader = new JsonReader(new StringReader(jsonData));
             jsonParserStream.parse(jsonReader);
@@ -92,8 +106,69 @@ public class YangUtils {
         return normalizedNodeResult.getResult();
     }
 
-    public static void chopNormalizedNode(NormalizedNode<?, ?> tree) {
-        //TODO Toine Siebelink, add code from proto-type (other user story)
+    /**
+     * Break a Normalized Node tree into fragments that can be stored by the persistence service.
+     *
+     * @param tree   the normalized node tree
+     * @param module the module applicable for the data in the normalized node
+     * @return the 'root' Fragment for the tree contain all relevant children etc.
+     */
+    public static Fragment fragmentNormalizedNode(
+            final NormalizedNode<? extends YangInstanceIdentifier.PathArgument, ?> tree,
+            final Module module) {
+        final QName nodeType = tree.getNodeType();
+        final Fragment rootFragment = Fragment.createRootFragment(module, nodeType);
+        fragmentNormalizedNode(rootFragment, tree);
+        return rootFragment;
+    }
+
+    private static void fragmentNormalizedNode(final Fragment currentFragment,
+                                               final NormalizedNode normalizedNode) {
+        if (normalizedNode instanceof DataContainerNode) {
+            inspectContainer(currentFragment, (DataContainerNode) normalizedNode);
+        } else if (normalizedNode instanceof MapNode) {
+            inspectKeyedList(currentFragment, (MapNode) normalizedNode);
+        } else if (normalizedNode instanceof ValueNode) {
+            inspectLeaf(currentFragment, (ValueNode) normalizedNode);
+        } else if (normalizedNode instanceof LeafSetNode) {
+            inspectLeafList(currentFragment, (LeafSetNode) normalizedNode);
+        } else {
+            LOGGER.warning("Cannot normalize " + normalizedNode.getClass());
+        }
+    }
+
+    private static void inspectLeaf(final Fragment currentFragment,
+                                    final ValueNode valueNode) {
+        final Object value = valueNode.getValue();
+        currentFragment.addLeafValue(valueNode.getNodeType().getLocalName(), value);
+    }
+
+    private static void inspectLeafList(final Fragment currentFragment,
+                                        final LeafSetNode leafSetNode) {
+        currentFragment.addLeafListName(leafSetNode.getNodeType().getLocalName());
+        for (final NormalizedNode value : (Collection<NormalizedNode>) leafSetNode.getValue()) {
+            fragmentNormalizedNode(currentFragment, value);
+        }
+    }
+
+    private static void inspectContainer(final Fragment currentFragment,
+                                         final DataContainerNode dataContainerNode) {
+        final Collection<NormalizedNode> leaves = (Collection) dataContainerNode.getValue();
+        for (final NormalizedNode leaf : leaves) {
+            fragmentNormalizedNode(currentFragment, leaf);
+        }
     }
 
+    private static void inspectKeyedList(final Fragment currentFragment,
+                                         final MapNode mapNode) {
+        createNodeForEachListElement(currentFragment, mapNode);
+    }
+
+    private static void createNodeForEachListElement(final Fragment currentFragment, final MapNode mapNode) {
+        final Collection<MapEntryNode> mapEntryNodes = mapNode.getValue();
+        for (final MapEntryNode mapEntryNode : mapEntryNodes) {
+            final Fragment listElementFragment = currentFragment.createChildFragment(mapNode.getNodeType());
+            fragmentNormalizedNode(listElementFragment, mapEntryNode);
+        }
+    }
 }
diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy
index 709378ebe..2a3d5cbc5 100755
--- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy
+++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpServiceImplSpec.groovy
@@ -67,7 +67,7 @@ class CpServiceImplSpec extends Specification {
     }
 
     def assertModule(SchemaContext schemaContext){
-        def optionalModule = schemaContext.findModule('bookstore', Revision.of('2020-09-15'))
+        def optionalModule = schemaContext.findModule('stores', Revision.of('2020-09-15'))
         return schemaContext.modules.size() == 1 && optionalModule.isPresent()
     }
 
@@ -167,4 +167,4 @@ class CpServiceImplSpec extends Specification {
         expect: 'anchor name is returned by service'
             objectUnderTest.createAnchor(anchorDetails) == 'dummyAnchorName'
     }
-}
\ No newline at end of file
+}
diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy
index 8aabc4844..6a463ad6f 100644
--- a/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy
+++ b/cps-service/src/test/groovy/org/onap/cps/utils/YangUtilsSpec.groovy
@@ -19,14 +19,16 @@
 
 package org.onap.cps.utils
 
-import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode
+import org.onap.cps.TestUtils
 import org.opendaylight.yangtools.yang.common.QName
 import org.opendaylight.yangtools.yang.common.Revision
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode
 import org.opendaylight.yangtools.yang.model.parser.api.YangSyntaxErrorException
 import spock.lang.Specification
 import spock.lang.Unroll
 
 class YangUtilsSpec extends Specification{
+
     def 'Parsing a valid Yang Model'() {
         given: 'a yang model (file)'
             def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
@@ -34,14 +36,14 @@ class YangUtilsSpec extends Specification{
             def result = YangUtils.parseYangModelFile(file)
         then: 'the result contain 1 module of the correct name and revision'
             result.modules.size() == 1
-            def optionalModule = result.findModule('bookstore', Revision.of('2020-09-15'))
+            def optionalModule = result.findModule('stores', Revision.of('2020-09-15'))
             optionalModule.isPresent()
     }
 
     @Unroll
-    def 'parsing invalid yang file (#description)'() {
+    def 'Parsing invalid yang file (#description).'() {
         given: 'a file with #description'
-            File file = new File(ClassLoader.getSystemClassLoader().getResource(filename).getFile());
+            File file = new File(ClassLoader.getSystemClassLoader().getResource(filename).getFile())
         when: 'the file is parsed'
             YangUtils.parseYangModelFile(file)
         then: 'an exception is thrown'
@@ -52,29 +54,48 @@ class YangUtilsSpec extends Specification{
             'someOtherFile.txt' | 'no .yang extension' || IllegalArgumentException
     }
 
-    def 'Parsing a valid Json String'() {
+    def 'Parsing a valid Json String.'() {
         given: 'a yang model (file)'
             def jsonData = org.onap.cps.TestUtils.getResourceFileContent('bookstore.json')
         and: 'a model for that data'
             def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
             def schemaContext = YangUtils.parseYangModelFile(file)
         when: 'the json data is parsed'
-            NormalizedNode<?, ?> result = YangUtils.parseJsonData(jsonData, schemaContext);
+            NormalizedNode<?, ?> result = YangUtils.parseJsonData(jsonData, schemaContext)
         then: 'the result is a normalized node of the correct type'
-            result.nodeType == QName.create('org:onap:ccsdk:sample','2020-09-15','bookstore')
+            result.nodeType == QName.create('org:onap:ccsdk:sample', '2020-09-15', 'bookstore')
     }
 
-    def 'Parsing an invalid Json String'() {
+    @Unroll
+    def 'Parsing invalid data: #description.'() {
         given: 'a yang model (file)'
-            def jsonData = '{incomplete json'
-        and: 'a model'
             def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
             def schemaContext = YangUtils.parseYangModelFile(file)
-        when: 'the invalid json is parsed'
-            YangUtils.parseJsonData(jsonData, schemaContext);
-        then: ' an exception is thrown'
+        when: 'invalid data is parsed'
+            YangUtils.parseJsonData(invalidJson, schemaContext)
+        then: 'an exception is thrown'
             thrown(IllegalStateException)
+        where: 'the following invalid json is provided'
+            invalidJson                                       | description
+            '{incomplete json'                                | 'incomplete json'
+            '{"test:bookstore": {"address": "Parnell st." }}' | 'json with un-modelled data'
     }
 
+    def 'Breaking a Json Data Object into fragments.'() {
+        given: 'a Yang module'
+            def file = new File(ClassLoader.getSystemClassLoader().getResource('bookstore.yang').getFile())
+            def schemaContext = YangUtils.parseYangModelFile(file)
+            def module = schemaContext.findModule('stores', Revision.of('2020-09-15')).get()
+        and: 'a normalized node for that model'
+            def jsonData = TestUtils.getResourceFileContent('bookstore.json')
+            def normalizedNode = YangUtils.parseJsonData(jsonData, schemaContext)
+        when: 'the json data is fragmented'
+            def result = YangUtils.fragmentNormalizedNode(normalizedNode, module)
+        then: 'the system creates a (root) fragment without a parent and 2 children (categories)'
+            result.parentFragment == null
+            result.childFragments.size() == 2
+        and: 'each child (category) has the root fragment (result) as parent and in turn as 1 child (a list of books)'
+            result.childFragments.each { it.parentFragment == result && it.childFragments.size() == 1 }
+    }
 
 }
diff --git a/cps-service/src/test/resources/bookstore.json b/cps-service/src/test/resources/bookstore.json
index 44d5d424c..d1b8d6882 100644
--- a/cps-service/src/test/resources/bookstore.json
+++ b/cps-service/src/test/resources/bookstore.json
@@ -1,34 +1,52 @@
 {
    "test:bookstore":{
-      "categories":[
+      "bookstore-name": "Chapters",
+      "categories": [
          {
-            "name":"web",
-            "books":[
+            "code": "01",
+            "name": "SciFi",
+            "books": [
                {
-                  "authors":[
-                     "Toine Siebelink","David Lang"
+                  "authors": [
+                     "Iain M. Banks"
                   ],
-                  "lang":"en",
-                  "price":"123456",
-                  "pub_year":"2020",
-                  "title":"My first book"
+                  "lang": "en",
+                  "price": "895",
+                  "pub_year": "1994",
+                  "title": "Feersum Endjinn"
+               },
+               {
+                  "authors": [
+                     "Ursula K. Le Guin",
+                     "Joe Haldeman",
+                     "Orson Scott Card",
+                     "david Brin",
+                     "Rober Silverberg",
+                     "Dan Simmons",
+                     "Greg Bear"
+                  ],
+                  "lang": "en",
+                  "price": "1099",
+                  "pub_year": "1999",
+                  "title": "Far Horizons"
                }
             ]
          },
          {
-            "name":"art",
-            "books":[
+            "name": "kids",
+            "code": "02",
+            "books": [
                {
-                  "authors":[
-                     "Test"
+                  "authors": [
+                     "Philip Pullman"
                   ],
-                  "lang":"en",
-                  "price":"1234",
-                  "pub_year":"2020",
-                  "title":"My 2nd book"
+                  "lang": "en",
+                  "price": "699",
+                  "pub_year": "1995",
+                  "title": "The Golden Compass"
                }
             ]
          }
       ]
    }
-}
\ No newline at end of file
+}
diff --git a/cps-service/src/test/resources/bookstore.yang b/cps-service/src/test/resources/bookstore.yang
index 01eac5f33..2179fb93d 100644
--- a/cps-service/src/test/resources/bookstore.yang
+++ b/cps-service/src/test/resources/bookstore.yang
@@ -1,6 +1,5 @@
-module bookstore {
+module stores {
     yang-version 1.1;
-
     namespace "org:onap:ccsdk:sample";
 
     prefix book-store;
@@ -18,9 +17,17 @@ module bookstore {
 
     container bookstore {
 
+        leaf bookstore-name {
+            type string;
+        }
+
     list categories {
 
-        key name;
+        key "code";
+
+        leaf code {
+            type string;
+        }
 
         leaf name {
             type string;