diff options
author | 2023-04-27 15:26:21 +0000 | |
---|---|---|
committer | 2023-04-27 15:26:21 +0000 | |
commit | a4cfede5bac589107ab81968f1d3a88c11a464d2 (patch) | |
tree | 3830ecd03ee767c863b3b5370107aa43587048ba | |
parent | 4031d43f06cb6ceb2e5bc39feb3e4a19ed3c6307 (diff) | |
parent | 9c126615d93ada36299fbac36dfb20aedbe2add1 (diff) |
Merge "Add retry mechanism on Subscription loader"
4 files changed, 264 insertions, 199 deletions
diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml index 6d4583566..b95d9eb43 100644 --- a/cps-application/src/main/resources/application.yml +++ b/cps-application/src/main/resources/application.yml @@ -1,197 +1,199 @@ -# ============LICENSE_START=======================================================
-# Copyright (C) 2021 Pantheon.tech
-# Modifications Copyright (C) 2021-2022 Bell Canada
-# Modifications Copyright (C) 2021-2023 Nordix Foundation
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# SPDX-License-Identifier: Apache-2.0
-# ============LICENSE_END=========================================================
-
-server:
- port: 8080
-
-rest:
- api:
- cps-base-path: /cps/api
- ncmp-base-path: /ncmp
- ncmp-inventory-base-path: /ncmpInventory
-
-spring:
- main:
- banner-mode: "off"
- application:
- name: "cps-application"
- jpa:
- show-sql: false
- ddl-auto: create
- open-in-view: false
- properties:
- hibernate:
- enable_lazy_load_no_trans: true
- dialect: org.hibernate.dialect.PostgreSQLDialect
-
- datasource:
- url: jdbc:postgresql://${DB_HOST}:${DB_PORT:5432}/cpsdb
- username: ${DB_USERNAME}
- password: ${DB_PASSWORD}
- driverClassName: org.postgresql.Driver
- hikari:
- minimumIdle: 5
- maximumPoolSize: 80
- idleTimeout: 60000
- connectionTimeout: 120000
- leakDetectionThreshold: 30000
- pool-name: CpsDatabasePool
-
- cache:
- type: caffeine
- cache-names: yangSchema
- caffeine:
- spec: maximumSize=10000,expireAfterAccess=10m
-
- liquibase:
- change-log: classpath:changelog/changelog-master.yaml
- labels: ${LIQUIBASE_LABELS}
-
- servlet:
- multipart:
- enabled: true
- max-file-size: 100MB
- max-request-size: 100MB
-
- kafka:
- bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVER:localhost:9092}
- security:
- protocol: PLAINTEXT
- producer:
- value-serializer: org.springframework.kafka.support.serializer.JsonSerializer
- client-id: cps-core
- consumer:
- group-id: ${NCMP_CONSUMER_GROUP_ID:ncmp-group}
- key-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer
- value-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer
- properties:
- spring.deserializer.key.delegate.class: org.apache.kafka.common.serialization.StringDeserializer
- spring.deserializer.value.delegate.class: org.springframework.kafka.support.serializer.JsonDeserializer
- spring.json.use.type.headers: false
-
- jackson:
- default-property-inclusion: NON_NULL
- serialization:
- FAIL_ON_EMPTY_BEANS: false
- sql:
- init:
- mode: ALWAYS
-app:
- ncmp:
- async-m2m:
- topic: ${NCMP_ASYNC_M2M_TOPIC:ncmp-async-m2m}
- avc:
- subscription-topic: ${NCMP_CM_AVC_SUBSCRIPTION:cm-avc-subscription}
- cm-events-topic: ${NCMP_CM_EVENTS_TOPIC:cm-events}
- lcm:
- events:
- topic: ${LCM_EVENTS_TOPIC:ncmp-events}
- dmi:
- cm-events:
- topic: ${DMI_CM_EVENTS_TOPIC:dmi-cm-events}
-
-
-notification:
- enabled: true
- data-updated:
- topic: ${CPS_CHANGE_EVENT_TOPIC:cps.data-updated-events}
- filters:
- enabled-dataspaces: ${NOTIFICATION_DATASPACE_FILTER_PATTERNS:""}
- async:
- executor:
- core-pool-size: 2
- max-pool-size: 10
- queue-capacity: 500
- wait-for-tasks-to-complete-on-shutdown: true
- thread-name-prefix: Async-
- time-out-value-in-ms: 2000
-
-springdoc:
- swagger-ui:
- disable-swagger-default-url: true
- urlsPrimaryName: cps-core
- urls:
- - name: cps-core
- url: /api-docs/cps-core/openapi.yaml
- - name: cps-ncmp
- url: /api-docs/cps-ncmp/openapi.yaml
- - name: cps-ncmp-inventory
- url: /api-docs/cps-ncmp/openapi-inventory.yaml
-
-
-security:
- # comma-separated uri patterns which do not require authorization
- permit-uri: /manage/**,/swagger-ui.html,/swagger-ui/**,/swagger-resources/**,/api-docs/**
- auth:
- username: ${CPS_USERNAME}
- password: ${CPS_PASSWORD}
-
-# Actuator
-management:
- server:
- port: 8081
- endpoints:
- web:
- base-path: /manage
- exposure:
- include: info,health,loggers,prometheus
- endpoint:
- health:
- show-details: always
- # kubernetes probes: liveness and readiness
- probes:
- enabled: true
-
-logging:
- format: json
- level:
- org:
- springframework: INFO
- onap:
- cps: INFO
-ncmp:
- dmi:
- auth:
- username: ${DMI_USERNAME}
- password: ${DMI_PASSWORD}
- api:
- base-path: dmi
-
- timers:
- advised-modules-sync:
- sleep-time-ms: 5000
- locked-modules-sync:
- sleep-time-ms: 300000
- cm-handle-data-sync:
- sleep-time-ms: 30000
-
- modules-sync-watchdog:
- async-executor:
- parallelism-level: 10
-
- model-loader:
- subscription: false
-
-# Custom Hazelcast Config.
-hazelcast:
- mode:
- kubernetes:
- enabled: ${HAZELCAST_MODE_KUBERNETES_ENABLED:false}
+# ============LICENSE_START======================================================= +# Copyright (C) 2021 Pantheon.tech +# Modifications Copyright (C) 2021-2022 Bell Canada +# Modifications Copyright (C) 2021-2023 Nordix Foundation +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +server: + port: 8080 + +rest: + api: + cps-base-path: /cps/api + ncmp-base-path: /ncmp + ncmp-inventory-base-path: /ncmpInventory + +spring: + main: + banner-mode: "off" + application: + name: "cps-application" + jpa: + show-sql: false + ddl-auto: create + open-in-view: false + properties: + hibernate: + enable_lazy_load_no_trans: true + dialect: org.hibernate.dialect.PostgreSQLDialect + + datasource: + url: jdbc:postgresql://${DB_HOST}:${DB_PORT:5432}/cpsdb + username: ${DB_USERNAME} + password: ${DB_PASSWORD} + driverClassName: org.postgresql.Driver + hikari: + minimumIdle: 5 + maximumPoolSize: 80 + idleTimeout: 60000 + connectionTimeout: 120000 + leakDetectionThreshold: 30000 + pool-name: CpsDatabasePool + + cache: + type: caffeine + cache-names: yangSchema + caffeine: + spec: maximumSize=10000,expireAfterAccess=10m + + liquibase: + change-log: classpath:changelog/changelog-master.yaml + labels: ${LIQUIBASE_LABELS} + + servlet: + multipart: + enabled: true + max-file-size: 100MB + max-request-size: 100MB + + kafka: + bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVER:localhost:9092} + security: + protocol: PLAINTEXT + producer: + value-serializer: org.springframework.kafka.support.serializer.JsonSerializer + client-id: cps-core + consumer: + group-id: ${NCMP_CONSUMER_GROUP_ID:ncmp-group} + key-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer + value-deserializer: org.springframework.kafka.support.serializer.ErrorHandlingDeserializer + properties: + spring.deserializer.key.delegate.class: org.apache.kafka.common.serialization.StringDeserializer + spring.deserializer.value.delegate.class: org.springframework.kafka.support.serializer.JsonDeserializer + spring.json.use.type.headers: false + + jackson: + default-property-inclusion: NON_NULL + serialization: + FAIL_ON_EMPTY_BEANS: false + sql: + init: + mode: ALWAYS +app: + ncmp: + async-m2m: + topic: ${NCMP_ASYNC_M2M_TOPIC:ncmp-async-m2m} + avc: + subscription-topic: ${NCMP_CM_AVC_SUBSCRIPTION:cm-avc-subscription} + cm-events-topic: ${NCMP_CM_EVENTS_TOPIC:cm-events} + lcm: + events: + topic: ${LCM_EVENTS_TOPIC:ncmp-events} + dmi: + cm-events: + topic: ${DMI_CM_EVENTS_TOPIC:dmi-cm-events} + + +notification: + enabled: true + data-updated: + topic: ${CPS_CHANGE_EVENT_TOPIC:cps.data-updated-events} + filters: + enabled-dataspaces: ${NOTIFICATION_DATASPACE_FILTER_PATTERNS:""} + async: + executor: + core-pool-size: 2 + max-pool-size: 10 + queue-capacity: 500 + wait-for-tasks-to-complete-on-shutdown: true + thread-name-prefix: Async- + time-out-value-in-ms: 2000 + +springdoc: + swagger-ui: + disable-swagger-default-url: true + urlsPrimaryName: cps-core + urls: + - name: cps-core + url: /api-docs/cps-core/openapi.yaml + - name: cps-ncmp + url: /api-docs/cps-ncmp/openapi.yaml + - name: cps-ncmp-inventory + url: /api-docs/cps-ncmp/openapi-inventory.yaml + + +security: + # comma-separated uri patterns which do not require authorization + permit-uri: /manage/**,/swagger-ui.html,/swagger-ui/**,/swagger-resources/**,/api-docs/** + auth: + username: ${CPS_USERNAME} + password: ${CPS_PASSWORD} + +# Actuator +management: + server: + port: 8081 + endpoints: + web: + base-path: /manage + exposure: + include: info,health,loggers,prometheus + endpoint: + health: + show-details: always + # kubernetes probes: liveness and readiness + probes: + enabled: true + +logging: + format: json + level: + org: + springframework: INFO + onap: + cps: INFO +ncmp: + dmi: + auth: + username: ${DMI_USERNAME} + password: ${DMI_PASSWORD} + api: + base-path: dmi + + timers: + advised-modules-sync: + sleep-time-ms: 5000 + locked-modules-sync: + sleep-time-ms: 300000 + cm-handle-data-sync: + sleep-time-ms: 30000 + + modules-sync-watchdog: + async-executor: + parallelism-level: 10 + + model-loader: + subscription: false + maximumAttemptCount: 20 + retryTimeMs: 1000 + +# Custom Hazelcast Config. +hazelcast: + mode: + kubernetes: + enabled: ${HAZELCAST_MODE_KUBERNETES_ENABLED:false} service-name: ${CPS_NCMP_SERVICE_NAME:"cps-and-ncmp-service"}
\ No newline at end of file diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java index 231ba75b5..5a418fd31 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java @@ -31,6 +31,7 @@ import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException; import org.onap.cps.spi.exceptions.AlreadyDefinedException; +import org.onap.cps.spi.model.Dataspace; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -51,6 +52,12 @@ public class SubscriptionModelLoader implements ModelLoader { private static final String SUBSCRIPTION_SCHEMASET_NAME = "subscriptions"; private static final String SUBSCRIPTION_REGISTRY_DATANODE_NAME = "subscription-registry"; + @Value("${ncmp.model-loader.maximumAttemptCount:20}") + private int maximumAttemptCount; + + @Value("${ncmp.model-loader.retryTimeMs:1000}") + private long retryTimeMs; + @Value("${ncmp.model-loader.subscription:false}") private boolean subscriptionModelLoaderEnabled; @@ -63,6 +70,7 @@ public class SubscriptionModelLoader implements ModelLoader { public void onApplicationEvent(final ApplicationReadyEvent applicationReadyEvent) { try { if (subscriptionModelLoaderEnabled) { + checkNcmpDataspaceExists(); onboardSubscriptionModel(createYangResourceToContentMap()); } else { log.info("Subscription Model Loader is disabled"); @@ -73,6 +81,29 @@ public class SubscriptionModelLoader implements ModelLoader { } } + private void checkNcmpDataspaceExists() { + boolean ncmpDataspaceExists = false; + int attemptCount = 0; + while (!ncmpDataspaceExists) { + final Dataspace ncmpDataspace = cpsAdminService.getDataspace(SUBSCRIPTION_DATASPACE_NAME); + if (ncmpDataspace != null) { + ncmpDataspaceExists = true; + } + if (attemptCount < maximumAttemptCount) { + try { + Thread.sleep(attemptCount * retryTimeMs); + attemptCount++; + log.info("Retrieving NCMP dataspace... {} attempt(s) ", attemptCount); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else { + throw new NcmpStartUpException("Retrieval of NCMP dataspace fails", + "NCMP dataspace does not exist"); + } + } + } + /** * Method to onboard subscription model for NCMP. */ diff --git a/cps-ncmp-service/src/main/resources/model/subscription.yang b/cps-ncmp-service/src/main/resources/model/subscription.yang index 8ae1be664..e332a2898 100644 --- a/cps-ncmp-service/src/main/resources/model/subscription.yang +++ b/cps-ncmp-service/src/main/resources/model/subscription.yang @@ -4,7 +4,7 @@ module subscription { prefix subs; - revision "2023-21-03" { + revision "2023-03-21" { description "NCMP subscription model"; } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy index aa8bc53c9..a14a0f286 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy @@ -32,6 +32,7 @@ import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException import org.onap.cps.spi.exceptions.AlreadyDefinedException import org.onap.cps.spi.exceptions.DataValidationException import org.onap.cps.spi.exceptions.SchemaSetNotFoundException +import org.onap.cps.spi.model.Dataspace import org.springframework.boot.SpringApplication import org.slf4j.LoggerFactory import org.springframework.boot.context.event.ApplicationReadyEvent @@ -73,9 +74,15 @@ class SubscriptionModelLoaderSpec extends Specification { } def 'Onboard subscription model successfully via application ready event'() { - when:'model loader is enabled' + given: 'dataspace is ready for use' + mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('NCMP-Admin') + and:'model loader is enabled' objectUnderTest.subscriptionModelLoaderEnabled = true - and: 'the application is ready' + and: 'maximum attempt count is set' + objectUnderTest.maximumAttemptCount = 20 + and: 'retry time is set' + objectUnderTest.retryTimeMs = 100 + when: 'the application is ready' objectUnderTest.onApplicationEvent(applicationReadyEvent) then: 'the module service to create schema set is called once' 1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'subscriptions',sampleYangContentMap) @@ -90,6 +97,8 @@ class SubscriptionModelLoaderSpec extends Specification { objectUnderTest.subscriptionModelLoaderEnabled = false and: 'application is ready' objectUnderTest.onApplicationEvent(applicationReadyEvent) + and: 'dataspace is ready for use' + mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('NCMP-Admin') then: 'the module service to create schema set was not called' 0 * mockCpsModuleService.createSchemaSet(*_) and: 'the admin service to create an anchor set was not called' @@ -98,11 +107,34 @@ class SubscriptionModelLoaderSpec extends Specification { 0 * mockCpsDataService.saveData(*_) } + def 'Onboard subscription model fails as NCMP dataspace does not exist' () { + given: 'model loader is enabled' + objectUnderTest.subscriptionModelLoaderEnabled = true + and: 'maximum attempt count is set' + objectUnderTest.maximumAttemptCount = 20 + and: 'retry time is set' + objectUnderTest.retryTimeMs = 100 + when: 'the application is ready' + objectUnderTest.onApplicationEvent(applicationReadyEvent) + then: 'the module service to create schema set was not called' + 0 * mockCpsModuleService.createSchemaSet(*_) + and: 'the admin service to create an anchor set was not called' + 0 * mockCpsAdminService.createAnchor(*_) + and: 'the data service to create a top level datanode was not called' + 0 * mockCpsDataService.saveData(*_) + and: 'the log message contains the correct exception message' + def logs = appender.list.toString() + assert logs.contains("Retrieval of NCMP dataspace fails") + } + + def 'Exception occurred while schema set creation' () { given: 'creating a schema set throws an exception' mockCpsModuleService.createSchemaSet(*_) >> { throw new DataValidationException(*_) } and: 'model loader is enabled' objectUnderTest.subscriptionModelLoaderEnabled = true + and: 'dataspace is ready for use' + mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('NCMP-Admin') when: 'application is ready' objectUnderTest.onApplicationEvent(applicationReadyEvent) then: 'the admin service to create an anchor set was not called' |