Extend model loader to support model-upgrade (part 2)

- add upgrade related methods to common abstract class
- add new (agreed) inventory model
- add InventoryModelLoader
- add more logging for success cases
- simplified constant names considering the context (class name)

Issue-ID: CPS-1804

Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>
Change-Id: I61a5c6d320d340a5c469ce20140f984439ba71a2

7 files changed, 349 insertions, 22 deletions

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java
index 349b1c5b02..cb2e15a3ce 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java
@@ -32,15 +32,14 @@ import org.onap.cps.api.CpsAdminService;
 import org.onap.cps.api.CpsDataService;
 import org.onap.cps.api.CpsModuleService;
 import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException;
+import org.onap.cps.spi.CascadeDeleteAllowed;
 import org.onap.cps.spi.exceptions.AlreadyDefinedException;
 import org.onap.cps.utils.JsonObjectMapper;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.stereotype.Service;
 
 @Slf4j
-@Service
 @RequiredArgsConstructor
 abstract class AbstractModelLoader implements ModelLoader {
 
@@ -69,6 +68,7 @@ abstract class AbstractModelLoader implements ModelLoader {
     }
 
     void waitUntilDataspaceIsAvailable(final String dataspaceName) {
+        log.info("Model Loader start-up, waiting for database to be ready");
         int attemptCount = 0;
         while (cpsAdminService.getDataspace(dataspaceName) == null) {
             if (attemptCount < maximumAttemptCount) {
@@ -92,36 +92,54 @@ abstract class AbstractModelLoader implements ModelLoader {
         } catch (final AlreadyDefinedException alreadyDefinedException) {
             log.warn("Creating new schema set failed as schema set already exists");
         } catch (final Exception exception) {
-            log.error("Creating schema set for subscription model failed: {} ", exception.getMessage());
+            log.error("Creating schema set failed: {} ", exception.getMessage());
             throw new NcmpStartUpException("Creating schema set failed", exception.getMessage());
         }
     }
 
+    void deleteUnusedSchemaSets(final String dataspaceName, final String... schemaSetNames) {
+        for (final String schemaSetName : schemaSetNames) {
+            try {
+                cpsModuleService.deleteSchemaSet(
+                    dataspaceName, schemaSetName, CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED);
+            } catch (final Exception exception) {
+                log.warn("Deleting schema set failed: {} ", exception.getMessage());
+            }
+        }
+    }
+
     void createAnchor(final String dataspaceName, final String schemaSetName, final String anchorName) {
         try {
             cpsAdminService.createAnchor(dataspaceName, schemaSetName, anchorName);
         } catch (final AlreadyDefinedException alreadyDefinedException) {
             log.warn("Creating new anchor failed as anchor already exists");
         } catch (final Exception exception) {
-            log.error("Creating anchor for subscription model failed: {} ", exception.getMessage());
+            log.error("Creating anchor failed: {} ", exception.getMessage());
             throw new NcmpStartUpException("Creating anchor failed", exception.getMessage());
         }
     }
 
-    void createTopLevelDataNode(final String dataspaceName,
-                                        final String anchorName,
-                                        final String dataNodeName) {
+    void createTopLevelDataNode(final String dataspaceName, final String anchorName, final String dataNodeName) {
         final String nodeData = jsonObjectMapper.asJsonString(Map.of(dataNodeName, Map.of()));
         try {
             cpsDataService.saveData(dataspaceName, anchorName, nodeData, OffsetDateTime.now());
         } catch (final AlreadyDefinedException exception) {
             log.warn("Creating new data node '{}' failed as data node already exists", dataNodeName);
         } catch (final Exception exception) {
-            log.error("Creating data node for subscription model failed: {}", exception.getMessage());
+            log.error("Creating data node failed: {}", exception.getMessage());
             throw new NcmpStartUpException("Creating data node failed", exception.getMessage());
         }
     }
 
+    void updateAnchorSchemaSet(final String dataspaceName, final String anchorName, final String schemaSetName) {
+        try {
+            cpsAdminService.updateAnchorSchemaSet(dataspaceName, anchorName, schemaSetName);
+        } catch (final Exception exception) {
+            log.error("Updating schema set failed: {}", exception.getMessage());
+            throw new NcmpStartUpException("Updating schema set failed", exception.getMessage());
+        }
+    }
+
     Map<String, String> createYangResourceToContentMap(final String resourceName) {
         return Map.of(resourceName, getFileContentAsString("models/" + resourceName));
     }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java
new file mode 100644
index 0000000000..5316d666d5
--- /dev/null
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java
@@ -0,0 +1,62 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2023 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.init;
+
+import lombok.extern.slf4j.Slf4j;
+import org.onap.cps.api.CpsAdminService;
+import org.onap.cps.api.CpsDataService;
+import org.onap.cps.api.CpsModuleService;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+public class InventoryModelLoader extends AbstractModelLoader {
+
+    private static final String NEW_MODEL_FILE_NAME = "dmi-registry@2023-08-23.yang";
+    private static final String NEW_SCHEMA_SET_NAME = "dmi-registry-2023-08-23";
+    private static final String DATASPACE_NAME = "NCMP-Admin";
+    private static final String ANCHOR_NAME = "ncmp-dmi-registry";
+
+    public InventoryModelLoader(final CpsAdminService cpsAdminService,
+                                final CpsModuleService cpsModuleService,
+                                final CpsDataService cpsDataService) {
+        super(cpsAdminService, cpsModuleService, cpsDataService);
+    }
+
+    @Override
+    public void onboardOrUpgradeModel() {
+        waitUntilDataspaceIsAvailable(DATASPACE_NAME);
+        updateInventoryModel();
+        log.info("Inventory Model updated successfully");
+    }
+
+    private void updateInventoryModel() {
+        createSchemaSet(DATASPACE_NAME, NEW_SCHEMA_SET_NAME, NEW_MODEL_FILE_NAME);
+        updateAnchorSchemaSet(DATASPACE_NAME, ANCHOR_NAME, NEW_SCHEMA_SET_NAME);
+        deleteOldButNotThePreviousSchemaSets();
+    }
+
+    private void deleteOldButNotThePreviousSchemaSets() {
+        //No schema sets passed in yet, but wil be required for future updates
+        deleteUnusedSchemaSets(DATASPACE_NAME);
+    }
+
+}
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java
index 614efd4f40..891244c7ca 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java
@@ -31,11 +31,11 @@ import org.springframework.stereotype.Service;
 @Service
 public class SubscriptionModelLoader extends AbstractModelLoader {
 
-    private static final String SUBSCRIPTION_MODEL_FILENAME = "subscription.yang";
-    private static final String SUBSCRIPTION_DATASPACE_NAME = "NCMP-Admin";
-    private static final String SUBSCRIPTION_ANCHOR_NAME = "AVC-Subscriptions";
-    private static final String SUBSCRIPTION_SCHEMASET_NAME = "subscriptions";
-    private static final String SUBSCRIPTION_REGISTRY_DATANODE_NAME = "subscription-registry";
+    private static final String MODEL_FILENAME = "subscription.yang";
+    private static final String DATASPACE_NAME = "NCMP-Admin";
+    private static final String ANCHOR_NAME = "AVC-Subscriptions";
+    private static final String SCHEMASET_NAME = "subscriptions";
+    private static final String REGISTRY_DATANODE_NAME = "subscription-registry";
 
     public SubscriptionModelLoader(final CpsAdminService cpsAdminService,
                                    final CpsModuleService cpsModuleService,
@@ -49,18 +49,18 @@ public class SubscriptionModelLoader extends AbstractModelLoader {
     @Override
     public void onboardOrUpgradeModel() {
         if (subscriptionModelLoaderEnabled) {
-            waitUntilDataspaceIsAvailable(SUBSCRIPTION_DATASPACE_NAME);
+            waitUntilDataspaceIsAvailable(DATASPACE_NAME);
             onboardSubscriptionModel();
+            log.info("Subscription Model onboarded successfully");
         } else {
             log.info("Subscription Model Loader is disabled");
         }
     }
 
     private void onboardSubscriptionModel() {
-        createSchemaSet(SUBSCRIPTION_DATASPACE_NAME, SUBSCRIPTION_SCHEMASET_NAME, SUBSCRIPTION_MODEL_FILENAME);
-        createAnchor(SUBSCRIPTION_DATASPACE_NAME, SUBSCRIPTION_SCHEMASET_NAME, SUBSCRIPTION_ANCHOR_NAME);
-        createTopLevelDataNode(SUBSCRIPTION_DATASPACE_NAME, SUBSCRIPTION_ANCHOR_NAME,
-            SUBSCRIPTION_REGISTRY_DATANODE_NAME);
+        createSchemaSet(DATASPACE_NAME, SCHEMASET_NAME, MODEL_FILENAME);
+        createAnchor(DATASPACE_NAME, SCHEMASET_NAME, ANCHOR_NAME);
+        createTopLevelDataNode(DATASPACE_NAME, ANCHOR_NAME, REGISTRY_DATANODE_NAME);
     }
 
 }
diff --git a/cps-ncmp-service/src/main/resources/models/dmi-registry@2023-08-23.yang b/cps-ncmp-service/src/main/resources/models/dmi-registry@2023-08-23.yang
new file mode 100644
index 0000000000..bb7604d91a
--- /dev/null
+++ b/cps-ncmp-service/src/main/resources/models/dmi-registry@2023-08-23.yang
@@ -0,0 +1,131 @@
+module dmi-registry {
+
+  yang-version 1.1;
+
+  namespace "org:onap:cps:ncmp";
+
+  prefix dmi-reg;
+
+  contact "toine.siebelink@est.tech";
+
+  revision "2023-08-23" {
+    description
+    "Added ModuleSetTag";
+  }
+
+  revision "2022-05-10" {
+    description
+    "Added DataSyncEnabled, SyncState with State, LastSyncTime, DataStoreSyncState with Operational and Running syncstate";
+  }
+
+  revision "2022-02-10" {
+    description
+    "Added State, LockReason, LockReasonDetails to aid with cmHandle sync and timestamp to aid with retry/timeout scenarios";
+  }
+
+  revision "2021-12-13" {
+    description
+    "Added new list of public additional properties for a Cm-Handle which are exposed to clients of the NCMP interface";
+  }
+
+  revision "2021-10-20" {
+    description
+    "Added dmi-data-service-name & dmi-model-service-name to allow separate DMI instances for each responsibility";
+  }
+
+  revision "2021-05-20" {
+    description
+    "Initial Version";
+  }
+
+  grouping LockReason {
+    leaf reason {
+      type string;
+    }
+    leaf details {
+      type string;
+    }
+  }
+
+  grouping SyncState {
+   leaf sync-state {
+     type string;
+   }
+   leaf last-sync-time {
+     type string;
+   }
+  }
+
+  grouping Datastores {
+    container operational {
+      uses SyncState;
+    }
+    container running {
+      uses SyncState;
+    }
+  }
+
+  container dmi-registry {
+    list cm-handles {
+      key "id";
+      leaf id {
+        type string;
+      }
+      leaf dmi-service-name {
+        type string;
+      }
+      leaf dmi-data-service-name {
+        type string;
+      }
+      leaf dmi-model-service-name {
+        type string;
+      }
+      leaf module-set-tag {
+        type string;
+      }
+
+      list additional-properties {
+        key "name";
+        leaf name {
+          type string;
+        }
+        leaf value {
+          type string;
+        }
+      }
+
+      list public-properties {
+        key "name";
+        leaf name {
+          type string;
+        }
+        leaf value {
+          type string;
+        }
+      }
+
+      container state {
+        leaf cm-handle-state {
+          type string;
+        }
+
+        container lock-reason {
+          uses LockReason;
+        }
+
+        leaf last-update-time {
+          type string;
+        }
+
+        leaf data-sync-enabled {
+          type boolean;
+          default "false";
+        }
+
+        container datastores {
+          uses Datastores;
+        }
+      }
+    }
+  }
+}
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy
index a271ca4312..28eae8df14 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy
@@ -27,6 +27,7 @@ import org.onap.cps.api.CpsAdminService
 import org.onap.cps.api.CpsDataService
 import org.onap.cps.api.CpsModuleService
 import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException
+import org.onap.cps.spi.CascadeDeleteAllowed
 import org.onap.cps.spi.exceptions.AlreadyDefinedException
 import org.springframework.boot.SpringApplication
 import org.slf4j.LoggerFactory
@@ -114,10 +115,32 @@ class AbstractModelLoaderSpec extends Specification {
             assert thrown.details.contains('unable to read file')
     }
 
+    def 'Delete unused schema sets.'() {
+        when: 'several unused schemas are deleted '
+            objectUnderTest.deleteUnusedSchemaSets('some dataspace','schema set 1', 'schema set 2')
+        then: 'a request to delete each (without cascade) is delegated to the module service'
+            1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 1', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED)
+            1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 2', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED)
+
+    }
+
+    def 'Delete unused schema sets with exception.'() {
+        given: 'deleting the first schemaset causes an exception'
+            mockCpsModuleService.deleteSchemaSet(_, 'schema set 1', _) >> { throw new RuntimeException('test message')}
+        when: 'several unused schemas are deleted '
+            objectUnderTest.deleteUnusedSchemaSets('some dataspace','schema set 1', 'schema set 2')
+        then: 'the exception message is logged'
+            def logs = loggingListAppender.list.toString()
+            assert logs.contains('Deleting schema set failed')
+            assert logs.contains('test message')
+        and: 'the second schema set is still deleted'
+            1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 2', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED)
+    }
+
     def 'Create anchor.'() {
         when: 'creating an anchor'
             objectUnderTest.createAnchor('some dataspace','some schema set','new name')
-        then: 'thr operation is delegated to the admin service'
+        then: 'the operation is delegated to the admin service'
             1 * mockCpsAdminService.createAnchor('some dataspace','some schema set', 'new name')
     }
 
@@ -174,6 +197,24 @@ class AbstractModelLoaderSpec extends Specification {
             assert thrown.details.contains('test message')
     }
 
+    def 'Update anchor schema set.'() {
+        when: 'a schema set for an anchor is updated'
+            objectUnderTest.updateAnchorSchemaSet('some dataspace', 'anchor', 'new schema set')
+        then: 'the request is delegated to the admin service'
+            1 * mockCpsAdminService.updateAnchorSchemaSet('some dataspace', 'anchor', 'new schema set')
+    }
+
+    def 'Update anchor schema set with exception.'() {
+        given: 'the admin service throws an exception'
+            mockCpsAdminService.updateAnchorSchemaSet(*_) >> { throw new RuntimeException('test message') }
+        when: 'a schema set for an anchor is updated'
+            objectUnderTest.updateAnchorSchemaSet('some dataspace', 'anchor', 'new schema set')
+        then: 'a startup exception with correct message and details is thrown'
+            def thrown = thrown(NcmpStartUpException)
+            assert thrown.message.contains('Updating schema set failed')
+            assert thrown.details.contains('test message')
+    }
+
     class TestModelLoader extends AbstractModelLoader {
 
         TestModelLoader(final CpsAdminService cpsAdminService,
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy
new file mode 100644
index 0000000000..9195bc74ca
--- /dev/null
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy
@@ -0,0 +1,75 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2023 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.init
+
+import ch.qos.logback.classic.Level
+import ch.qos.logback.classic.Logger
+import ch.qos.logback.core.read.ListAppender
+import org.onap.cps.api.CpsAdminService
+import org.onap.cps.api.CpsDataService
+import org.onap.cps.api.CpsModuleService
+import org.onap.cps.spi.model.Dataspace
+import org.slf4j.LoggerFactory
+import org.springframework.boot.context.event.ApplicationReadyEvent
+import org.springframework.context.annotation.AnnotationConfigApplicationContext
+import spock.lang.Specification
+
+class InventoryModelLoaderSpec extends Specification {
+
+    def mockCpsAdminService = Mock(CpsAdminService)
+    def mockCpsModuleService = Mock(CpsModuleService)
+    def mockCpsDataService = Mock(CpsDataService)
+    def objectUnderTest = new InventoryModelLoader(mockCpsAdminService, mockCpsModuleService, mockCpsDataService)
+
+    def applicationContext = new AnnotationConfigApplicationContext()
+
+    def expectedYangResourceToContentMap
+    def logger = (Logger) LoggerFactory.getLogger(objectUnderTest.class)
+    def loggingListAppender
+
+    void setup() {
+        expectedYangResourceToContentMap = objectUnderTest.createYangResourceToContentMap('dmi-registry@2023-08-23.yang')
+        logger.setLevel(Level.DEBUG)
+        loggingListAppender = new ListAppender()
+        logger.addAppender(loggingListAppender)
+        loggingListAppender.start()
+        applicationContext.refresh()
+    }
+
+    void cleanup() {
+        ((Logger) LoggerFactory.getLogger(SubscriptionModelLoader.class)).detachAndStopAllAppenders()
+        applicationContext.close()
+    }
+
+    def 'Onboard subscription model via application ready event.'() {
+        given: 'dataspace is ready for use'
+            mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('')
+        when: 'the application is ready'
+            objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent))
+        then: 'the module service is used to create the new schema set from the correct resource'
+            1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'dmi-registry-2023-08-23', expectedYangResourceToContentMap)
+        and: 'the admin service is used to update the anchor'
+            1 * mockCpsAdminService.updateAnchorSchemaSet('NCMP-Admin', 'ncmp-dmi-registry', 'dmi-registry-2023-08-23')
+        and: 'No schema sets are being removed by the module service (yet)'
+            0 * mockCpsModuleService.deleteSchemaSet('NCMP-Admin', _, _)
+    }
+
+}
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy
index 305fe4c062..d99874ad6c 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy
@@ -41,12 +41,12 @@ class SubscriptionModelLoaderSpec extends Specification {
 
     def applicationContext = new AnnotationConfigApplicationContext()
 
-    def yangResourceToContentMap
+    def expectedYangResourceToContentMap
     def logger = (Logger) LoggerFactory.getLogger(objectUnderTest.class)
     def loggingListAppender
 
     void setup() {
-        yangResourceToContentMap = objectUnderTest.createYangResourceToContentMap('subscription.yang')
+        expectedYangResourceToContentMap = objectUnderTest.createYangResourceToContentMap('subscription.yang')
         logger.setLevel(Level.DEBUG)
         loggingListAppender = new ListAppender()
         logger.addAppender(loggingListAppender)
@@ -67,7 +67,7 @@ class SubscriptionModelLoaderSpec extends Specification {
         when: 'the application is ready'
             objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent))
         then: 'the module service to create schema set is called once'
-            1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'subscriptions', yangResourceToContentMap)
+            1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'subscriptions', expectedYangResourceToContentMap)
         and: 'the admin service to create an anchor set is called once'
             1 * mockCpsAdminService.createAnchor('NCMP-Admin', 'subscriptions', 'AVC-Subscriptions')
         and: 'the data service to create a top level datanode is called once'