Add retry mechanism on Subscription loader

Issue-ID: CPS-1568
Signed-off-by: emaclee <lee.anjella.macabuhay@est.tech>
Change-Id: I35697c260cc1a774f50d12552996b39f812fc2de

4 files changed, 264 insertions, 199 deletions

diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml
index 6d45835668..b95d9eb43b 100644
--- a/cps-application/src/main/resources/application.yml
+++ b/cps-application/src/main/resources/application.yml
@@ -1,197 +1,199 @@
-#  ============LICENSE_START=======================================================

-#  Copyright (C) 2021 Pantheon.tech

-#  Modifications Copyright (C) 2021-2022 Bell Canada

-#  Modifications Copyright (C) 2021-2023 Nordix Foundation

-#  ================================================================================

-#  Licensed under the Apache License, Version 2.0 (the "License");

-#  you may not use this file except in compliance with the License.

-#  You may obtain a copy of the License at

-#

-#        http://www.apache.org/licenses/LICENSE-2.0

-#

-#  Unless required by applicable law or agreed to in writing, software

-#  distributed under the License is distributed on an "AS IS" BASIS,

-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-#  See the License for the specific language governing permissions and

-#  limitations under the License.

-#

-#  SPDX-License-Identifier: Apache-2.0

-#  ============LICENSE_END=========================================================

-

-server:

-    port: 8080

-

-rest:

-    api:

-        cps-base-path: /cps/api

-        ncmp-base-path: /ncmp

-        ncmp-inventory-base-path: /ncmpInventory

-

-spring:

-    main:

-        banner-mode: "off"

-    application:

-        name: "cps-application"

-    jpa:

-        show-sql: false

-        ddl-auto: create

-        open-in-view: false

-        properties:

-            hibernate:

-                enable_lazy_load_no_trans: true

-                dialect: org.hibernate.dialect.PostgreSQLDialect

-

-    datasource:

-        url: jdbc:postgresql://${DB_HOST}:${DB_PORT:5432}/cpsdb

-        username: ${DB_USERNAME}

-        password: ${DB_PASSWORD}

-        driverClassName: org.postgresql.Driver

-        hikari:

-            minimumIdle: 5

-            maximumPoolSize: 80

-            idleTimeout: 60000

-            connectionTimeout: 120000

-            leakDetectionThreshold: 30000

-            pool-name: CpsDatabasePool

-

-    cache:

-        type: caffeine

-        cache-names: yangSchema

-        caffeine:

-            spec: maximumSize=10000,expireAfterAccess=10m

-

-    liquibase:

-        change-log: classpath:changelog/changelog-master.yaml

-        labels: ${LIQUIBASE_LABELS}

-

-    servlet:

-        multipart:

-            enabled: true

-            max-file-size: 100MB

-            max-request-size: 100MB

-

-    kafka:

-        bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVER:localhost:9092}

-        security:

-            protocol: PLAINTEXT

-        producer:

-            value-serializer: org.springframework.kafka.support.serializer.JsonSerializer

-            client-id: cps-core

-        consumer:

-            group-id: ${NCMP_CONSUMER_GROUP_ID:ncmp-group}

-            key-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer

-            value-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer

-            properties:

-                spring.deserializer.key.delegate.class: org.apache.kafka.common.serialization.StringDeserializer

-                spring.deserializer.value.delegate.class: org.springframework.kafka.support.serializer.JsonDeserializer

-                spring.json.use.type.headers: false

-

-    jackson:

-        default-property-inclusion: NON_NULL

-        serialization:

-            FAIL_ON_EMPTY_BEANS: false

-    sql:

-        init:

-            mode: ALWAYS

-app:

-    ncmp:

-        async-m2m:

-            topic: ${NCMP_ASYNC_M2M_TOPIC:ncmp-async-m2m}

-        avc:

-            subscription-topic: ${NCMP_CM_AVC_SUBSCRIPTION:cm-avc-subscription}

-            cm-events-topic: ${NCMP_CM_EVENTS_TOPIC:cm-events}

-    lcm:

-        events:

-            topic: ${LCM_EVENTS_TOPIC:ncmp-events}

-    dmi:

-        cm-events:

-            topic: ${DMI_CM_EVENTS_TOPIC:dmi-cm-events}

-

-

-notification:

-    enabled: true

-    data-updated:

-        topic: ${CPS_CHANGE_EVENT_TOPIC:cps.data-updated-events}

-        filters:

-            enabled-dataspaces: ${NOTIFICATION_DATASPACE_FILTER_PATTERNS:""}

-    async:

-        executor:

-            core-pool-size: 2

-            max-pool-size: 10

-            queue-capacity: 500

-            wait-for-tasks-to-complete-on-shutdown: true

-            thread-name-prefix: Async-

-            time-out-value-in-ms: 2000

-

-springdoc:

-    swagger-ui:

-        disable-swagger-default-url: true

-        urlsPrimaryName: cps-core

-        urls:

-            - name: cps-core

-              url: /api-docs/cps-core/openapi.yaml

-            - name: cps-ncmp

-              url: /api-docs/cps-ncmp/openapi.yaml

-            - name: cps-ncmp-inventory

-              url: /api-docs/cps-ncmp/openapi-inventory.yaml

-

-

-security:

-    # comma-separated uri patterns which do not require authorization

-    permit-uri: /manage/**,/swagger-ui.html,/swagger-ui/**,/swagger-resources/**,/api-docs/**

-    auth:

-        username: ${CPS_USERNAME}

-        password: ${CPS_PASSWORD}

-

-# Actuator

-management:

-    server:

-        port: 8081

-    endpoints:

-        web:

-            base-path: /manage

-            exposure:

-                include: info,health,loggers,prometheus

-    endpoint:

-        health:

-            show-details: always

-            # kubernetes probes: liveness and readiness

-            probes:

-                enabled: true

-

-logging:

-    format: json

-    level:

-        org:

-            springframework: INFO

-            onap:

-                cps: INFO

-ncmp:

-    dmi:

-        auth:

-            username: ${DMI_USERNAME}

-            password: ${DMI_PASSWORD}

-        api:

-            base-path: dmi

-

-    timers:

-        advised-modules-sync:

-            sleep-time-ms: 5000

-        locked-modules-sync:

-            sleep-time-ms: 300000

-        cm-handle-data-sync:

-            sleep-time-ms: 30000

-

-    modules-sync-watchdog:

-        async-executor:

-            parallelism-level: 10

-

-    model-loader:

-        subscription: false

-

-# Custom Hazelcast Config.

-hazelcast:

-  mode:

-    kubernetes:

-      enabled: ${HAZELCAST_MODE_KUBERNETES_ENABLED:false}

+#  ============LICENSE_START=======================================================
+#  Copyright (C) 2021 Pantheon.tech
+#  Modifications Copyright (C) 2021-2022 Bell Canada
+#  Modifications Copyright (C) 2021-2023 Nordix Foundation
+#  ================================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  SPDX-License-Identifier: Apache-2.0
+#  ============LICENSE_END=========================================================
+
+server:
+    port: 8080
+
+rest:
+    api:
+        cps-base-path: /cps/api
+        ncmp-base-path: /ncmp
+        ncmp-inventory-base-path: /ncmpInventory
+
+spring:
+    main:
+        banner-mode: "off"
+    application:
+        name: "cps-application"
+    jpa:
+        show-sql: false
+        ddl-auto: create
+        open-in-view: false
+        properties:
+            hibernate:
+                enable_lazy_load_no_trans: true
+                dialect: org.hibernate.dialect.PostgreSQLDialect
+
+    datasource:
+        url: jdbc:postgresql://${DB_HOST}:${DB_PORT:5432}/cpsdb
+        username: ${DB_USERNAME}
+        password: ${DB_PASSWORD}
+        driverClassName: org.postgresql.Driver
+        hikari:
+            minimumIdle: 5
+            maximumPoolSize: 80
+            idleTimeout: 60000
+            connectionTimeout: 120000
+            leakDetectionThreshold: 30000
+            pool-name: CpsDatabasePool
+
+    cache:
+        type: caffeine
+        cache-names: yangSchema
+        caffeine:
+            spec: maximumSize=10000,expireAfterAccess=10m
+
+    liquibase:
+        change-log: classpath:changelog/changelog-master.yaml
+        labels: ${LIQUIBASE_LABELS}
+
+    servlet:
+        multipart:
+            enabled: true
+            max-file-size: 100MB
+            max-request-size: 100MB
+
+    kafka:
+        bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVER:localhost:9092}
+        security:
+            protocol: PLAINTEXT
+        producer:
+            value-serializer: org.springframework.kafka.support.serializer.JsonSerializer
+            client-id: cps-core
+        consumer:
+            group-id: ${NCMP_CONSUMER_GROUP_ID:ncmp-group}
+            key-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer
+            value-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer
+            properties:
+                spring.deserializer.key.delegate.class: org.apache.kafka.common.serialization.StringDeserializer
+                spring.deserializer.value.delegate.class: org.springframework.kafka.support.serializer.JsonDeserializer
+                spring.json.use.type.headers: false
+
+    jackson:
+        default-property-inclusion: NON_NULL
+        serialization:
+            FAIL_ON_EMPTY_BEANS: false
+    sql:
+        init:
+            mode: ALWAYS
+app:
+    ncmp:
+        async-m2m:
+            topic: ${NCMP_ASYNC_M2M_TOPIC:ncmp-async-m2m}
+        avc:
+            subscription-topic: ${NCMP_CM_AVC_SUBSCRIPTION:cm-avc-subscription}
+            cm-events-topic: ${NCMP_CM_EVENTS_TOPIC:cm-events}
+    lcm:
+        events:
+            topic: ${LCM_EVENTS_TOPIC:ncmp-events}
+    dmi:
+        cm-events:
+            topic: ${DMI_CM_EVENTS_TOPIC:dmi-cm-events}
+
+
+notification:
+    enabled: true
+    data-updated:
+        topic: ${CPS_CHANGE_EVENT_TOPIC:cps.data-updated-events}
+        filters:
+            enabled-dataspaces: ${NOTIFICATION_DATASPACE_FILTER_PATTERNS:""}
+    async:
+        executor:
+            core-pool-size: 2
+            max-pool-size: 10
+            queue-capacity: 500
+            wait-for-tasks-to-complete-on-shutdown: true
+            thread-name-prefix: Async-
+            time-out-value-in-ms: 2000
+
+springdoc:
+    swagger-ui:
+        disable-swagger-default-url: true
+        urlsPrimaryName: cps-core
+        urls:
+            - name: cps-core
+              url: /api-docs/cps-core/openapi.yaml
+            - name: cps-ncmp
+              url: /api-docs/cps-ncmp/openapi.yaml
+            - name: cps-ncmp-inventory
+              url: /api-docs/cps-ncmp/openapi-inventory.yaml
+
+
+security:
+    # comma-separated uri patterns which do not require authorization
+    permit-uri: /manage/**,/swagger-ui.html,/swagger-ui/**,/swagger-resources/**,/api-docs/**
+    auth:
+        username: ${CPS_USERNAME}
+        password: ${CPS_PASSWORD}
+
+# Actuator
+management:
+    server:
+        port: 8081
+    endpoints:
+        web:
+            base-path: /manage
+            exposure:
+                include: info,health,loggers,prometheus
+    endpoint:
+        health:
+            show-details: always
+            # kubernetes probes: liveness and readiness
+            probes:
+                enabled: true
+
+logging:
+    format: json
+    level:
+        org:
+            springframework: INFO
+            onap:
+                cps: INFO
+ncmp:
+    dmi:
+        auth:
+            username: ${DMI_USERNAME}
+            password: ${DMI_PASSWORD}
+        api:
+            base-path: dmi
+
+    timers:
+        advised-modules-sync:
+            sleep-time-ms: 5000
+        locked-modules-sync:
+            sleep-time-ms: 300000
+        cm-handle-data-sync:
+            sleep-time-ms: 30000
+
+    modules-sync-watchdog:
+        async-executor:
+            parallelism-level: 10
+
+    model-loader:
+        subscription: false
+        maximumAttemptCount: 20
+        retryTimeMs: 1000
+
+# Custom Hazelcast Config.
+hazelcast:
+  mode:
+    kubernetes:
+      enabled: ${HAZELCAST_MODE_KUBERNETES_ENABLED:false}
       service-name: ${CPS_NCMP_SERVICE_NAME:"cps-and-ncmp-service"}
\ No newline at end of file
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java
index 231ba75b5d..5a418fd312 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java
@@ -31,6 +31,7 @@ import org.onap.cps.api.CpsDataService;
 import org.onap.cps.api.CpsModuleService;
 import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException;
 import org.onap.cps.spi.exceptions.AlreadyDefinedException;
+import org.onap.cps.spi.model.Dataspace;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
@@ -51,6 +52,12 @@ public class SubscriptionModelLoader implements ModelLoader {
     private static final String SUBSCRIPTION_SCHEMASET_NAME = "subscriptions";
     private static final String SUBSCRIPTION_REGISTRY_DATANODE_NAME = "subscription-registry";
 
+    @Value("${ncmp.model-loader.maximumAttemptCount:20}")
+    private int maximumAttemptCount;
+
+    @Value("${ncmp.model-loader.retryTimeMs:1000}")
+    private long retryTimeMs;
+
     @Value("${ncmp.model-loader.subscription:false}")
     private boolean subscriptionModelLoaderEnabled;
 
@@ -63,6 +70,7 @@ public class SubscriptionModelLoader implements ModelLoader {
     public void onApplicationEvent(final ApplicationReadyEvent applicationReadyEvent) {
         try {
             if (subscriptionModelLoaderEnabled) {
+                checkNcmpDataspaceExists();
                 onboardSubscriptionModel(createYangResourceToContentMap());
             } else {
                 log.info("Subscription Model Loader is disabled");
@@ -73,6 +81,29 @@ public class SubscriptionModelLoader implements ModelLoader {
         }
     }
 
+    private void checkNcmpDataspaceExists() {
+        boolean ncmpDataspaceExists = false;
+        int attemptCount = 0;
+        while (!ncmpDataspaceExists) {
+            final Dataspace ncmpDataspace = cpsAdminService.getDataspace(SUBSCRIPTION_DATASPACE_NAME);
+            if (ncmpDataspace != null) {
+                ncmpDataspaceExists = true;
+            }
+            if (attemptCount < maximumAttemptCount) {
+                try {
+                    Thread.sleep(attemptCount * retryTimeMs);
+                    attemptCount++;
+                    log.info("Retrieving NCMP dataspace... {} attempt(s) ", attemptCount);
+                } catch (final InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            } else {
+                throw new NcmpStartUpException("Retrieval of NCMP dataspace fails",
+                        "NCMP dataspace does not exist");
+            }
+        }
+    }
+
     /**
      * Method to onboard subscription model for NCMP.
      */
diff --git a/cps-ncmp-service/src/main/resources/model/subscription.yang b/cps-ncmp-service/src/main/resources/model/subscription.yang
index 8ae1be6646..e332a2898a 100644
--- a/cps-ncmp-service/src/main/resources/model/subscription.yang
+++ b/cps-ncmp-service/src/main/resources/model/subscription.yang
@@ -4,7 +4,7 @@ module subscription {
 
     prefix subs;
 
-    revision "2023-21-03" {
+    revision "2023-03-21" {
         description
             "NCMP subscription model";
     }
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy
index aa8bc53c9d..a14a0f286c 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy
@@ -32,6 +32,7 @@ import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException
 import org.onap.cps.spi.exceptions.AlreadyDefinedException
 import org.onap.cps.spi.exceptions.DataValidationException
 import org.onap.cps.spi.exceptions.SchemaSetNotFoundException
+import org.onap.cps.spi.model.Dataspace
 import org.springframework.boot.SpringApplication
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.event.ApplicationReadyEvent
@@ -73,9 +74,15 @@ class SubscriptionModelLoaderSpec extends Specification {
     }
 
     def 'Onboard subscription model successfully via application ready event'() {
-        when:'model loader is enabled'
+        given: 'dataspace is ready for use'
+            mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('NCMP-Admin')
+        and:'model loader is enabled'
             objectUnderTest.subscriptionModelLoaderEnabled = true
-        and: 'the application is ready'
+        and: 'maximum attempt count is set'
+            objectUnderTest.maximumAttemptCount = 20
+        and: 'retry time is set'
+            objectUnderTest.retryTimeMs = 100
+        when: 'the application is ready'
             objectUnderTest.onApplicationEvent(applicationReadyEvent)
         then: 'the module service to create schema set is called once'
             1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'subscriptions',sampleYangContentMap)
@@ -90,6 +97,8 @@ class SubscriptionModelLoaderSpec extends Specification {
             objectUnderTest.subscriptionModelLoaderEnabled = false
         and: 'application is ready'
             objectUnderTest.onApplicationEvent(applicationReadyEvent)
+        and: 'dataspace is ready for use'
+            mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('NCMP-Admin')
         then: 'the module service to create schema set was not called'
             0 * mockCpsModuleService.createSchemaSet(*_)
         and: 'the admin service to create an anchor set was not called'
@@ -98,11 +107,34 @@ class SubscriptionModelLoaderSpec extends Specification {
             0 * mockCpsDataService.saveData(*_)
     }
 
+    def 'Onboard subscription model fails as NCMP dataspace does not exist' () {
+        given: 'model loader is enabled'
+            objectUnderTest.subscriptionModelLoaderEnabled = true
+        and: 'maximum attempt count is set'
+            objectUnderTest.maximumAttemptCount = 20
+        and: 'retry time is set'
+            objectUnderTest.retryTimeMs = 100
+        when: 'the application is ready'
+            objectUnderTest.onApplicationEvent(applicationReadyEvent)
+        then: 'the module service to create schema set was not called'
+            0 * mockCpsModuleService.createSchemaSet(*_)
+        and: 'the admin service to create an anchor set was not called'
+            0 * mockCpsAdminService.createAnchor(*_)
+        and: 'the data service to create a top level datanode was not called'
+            0 * mockCpsDataService.saveData(*_)
+        and: 'the log message contains the correct exception message'
+            def logs = appender.list.toString()
+            assert logs.contains("Retrieval of NCMP dataspace fails")
+    }
+
+
     def 'Exception occurred while schema set creation' () {
         given: 'creating a schema set throws an exception'
             mockCpsModuleService.createSchemaSet(*_) >>  { throw new DataValidationException(*_) }
         and: 'model loader is enabled'
             objectUnderTest.subscriptionModelLoaderEnabled = true
+        and: 'dataspace is ready for use'
+            mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('NCMP-Admin')
         when: 'application is ready'
             objectUnderTest.onApplicationEvent(applicationReadyEvent)
         then: 'the admin service to create an anchor set was not called'