diff options
146 files changed, 4395 insertions, 1733 deletions
diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index d6fbcd98d5..ea79b718fc 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -26,7 +26,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>checkstyle</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <profiles> <profile> diff --git a/cps-application/pom.xml b/cps-application/pom.xml index 5ac2202e85..44d0677ead 100644 --- a/cps-application/pom.xml +++ b/cps-application/pom.xml @@ -28,7 +28,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> @@ -78,6 +78,10 @@ <artifactId>micrometer-tracing-bridge-otel</artifactId> </dependency> <dependency> + <groupId>io.github.mweirauch</groupId> + <artifactId>micrometer-jvm-extras</artifactId> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-xml</artifactId> </dependency> diff --git a/cps-application/src/main/java/org/onap/cps/Application.java b/cps-application/src/main/java/org/onap/cps/Application.java index 053139fcc8..62103bf368 100644 --- a/cps-application/src/main/java/org/onap/cps/Application.java +++ b/cps-application/src/main/java/org/onap/cps/Application.java @@ -22,9 +22,7 @@ package org.onap.cps; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.retry.annotation.EnableRetry;
-@EnableRetry
@SpringBootApplication
public class Application {
public static void main(final String[] args) {
diff --git a/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java index 39ed6ef5a7..b85f391b8e 100644 --- a/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java +++ b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java @@ -21,10 +21,14 @@ package org.onap.cps.config; import com.hazelcast.map.IMap; +import io.github.mweirauch.micrometer.jvm.extras.ProcessMemoryMetrics; +import io.github.mweirauch.micrometer.jvm.extras.ProcessThreadMetrics; import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -32,8 +36,8 @@ import org.springframework.context.annotation.Configuration; @RequiredArgsConstructor public class MicroMeterConfig { - private static final String TAG = "state"; - private static final String CMHANDLE_STATE_GAUGE = "cmHandlesByState"; + private static final String STATE_TAG = "state"; + private static final String CM_HANDLE_STATE_GAUGE = "cmHandlesByState"; final IMap<String, Integer> cmHandlesByState; @Bean @@ -41,6 +45,18 @@ public class MicroMeterConfig { return new TimedAspect(meterRegistry); } + @Bean + @ConditionalOnProperty("cps.monitoring.micrometer-jvm-extras") + public MeterBinder processMemoryMetrics() { + return new ProcessMemoryMetrics(); + } + + @Bean + @ConditionalOnProperty("cps.monitoring.micrometer-jvm-extras") + public MeterBinder processThreadMetrics() { + return new ProcessThreadMetrics(); + } + /** * Register gauge metric for cm handles with state 'advised'. * @@ -49,10 +65,10 @@ public class MicroMeterConfig { */ @Bean public Gauge advisedCmHandles(final MeterRegistry meterRegistry) { - return Gauge.builder(CMHANDLE_STATE_GAUGE, cmHandlesByState, + return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("advisedCmHandlesCount")) - .tag(TAG, "ADVISED") - .description("Current number of cmhandles in advised state") + .tag(STATE_TAG, "ADVISED") + .description("Current number of cm handles in advised state") .register(meterRegistry); } @@ -64,10 +80,10 @@ public class MicroMeterConfig { */ @Bean public Gauge readyCmHandles(final MeterRegistry meterRegistry) { - return Gauge.builder(CMHANDLE_STATE_GAUGE, cmHandlesByState, + return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("readyCmHandlesCount")) - .tag(TAG, "READY") - .description("Current number of cmhandles in ready state") + .tag(STATE_TAG, "READY") + .description("Current number of cm handles in ready state") .register(meterRegistry); } @@ -79,10 +95,10 @@ public class MicroMeterConfig { */ @Bean public Gauge lockedCmHandles(final MeterRegistry meterRegistry) { - return Gauge.builder(CMHANDLE_STATE_GAUGE, cmHandlesByState, + return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("lockedCmHandlesCount")) - .tag(TAG, "LOCKED") - .description("Current number of cmhandles in locked state") + .tag(STATE_TAG, "LOCKED") + .description("Current number of cm handles in locked state") .register(meterRegistry); } @@ -94,10 +110,10 @@ public class MicroMeterConfig { */ @Bean public Gauge deletingCmHandles(final MeterRegistry meterRegistry) { - return Gauge.builder(CMHANDLE_STATE_GAUGE, cmHandlesByState, + return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("deletingCmHandlesCount")) - .tag(TAG, "DELETING") - .description("Current number of cmhandles in deleting state") + .tag(STATE_TAG, "DELETING") + .description("Current number of cm handles in deleting state") .register(meterRegistry); } @@ -109,10 +125,10 @@ public class MicroMeterConfig { */ @Bean public Gauge deletedCmHandles(final MeterRegistry meterRegistry) { - return Gauge.builder(CMHANDLE_STATE_GAUGE, cmHandlesByState, + return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("deletedCmHandlesCount")) - .tag(TAG, "DELETED") - .description("Current number of cmhandles in deleted state") + .tag(STATE_TAG, "DELETED") + .description("Number of cm handles that have been deleted since the application started") .register(meterRegistry); } diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml index 573db1fb10..6b9c694cf2 100644 --- a/cps-application/src/main/resources/application.yml +++ b/cps-application/src/main/resources/application.yml @@ -152,6 +152,8 @@ security: password: ${CPS_PASSWORD:cpsr0cks!} cps: + monitoring: + micrometer-jvm-extras: false tracing: sampler: jaeger_remote: @@ -172,7 +174,7 @@ management: endpoints: web: exposure: - include: info,health,loggers,prometheus,metrics + include: info,health,loggers,prometheus,metrics,heapdump,threaddump endpoint: health: show-details: always @@ -207,6 +209,7 @@ ncmp: connectionTimeoutInSeconds: 30 readTimeoutInSeconds: 30 writeTimeoutInSeconds: 30 + responseTimeoutInSeconds: 60 dmi: httpclient: data-services: @@ -216,6 +219,7 @@ ncmp: connectionTimeoutInSeconds: 30 readTimeoutInSeconds: 30 writeTimeoutInSeconds: 30 + responseTimeoutInSeconds: 60 model-services: maximumInMemorySizeInMegabytes: 16 maximumConnectionsTotal: 100 @@ -223,6 +227,7 @@ ncmp: connectionTimeoutInSeconds: 30 readTimeoutInSeconds: 30 writeTimeoutInSeconds: 30 + responseTimeoutInSeconds: 60 auth: username: ${DMI_USERNAME:cpsuser} password: ${DMI_PASSWORD:cpsr0cks!} diff --git a/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy index 67ca64624a..b9302ccd72 100644 --- a/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy +++ b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy @@ -31,10 +31,17 @@ class MicroMeterConfigSpec extends Specification { def simpleMeterRegistry = new SimpleMeterRegistry() def 'Creating a timed aspect.'() { - expect: ' a timed aspect can be created' + expect: 'a timed aspect can be created' assert objectUnderTest.timedAspect(simpleMeterRegistry) != null } + def 'Creating JVM process metrics.'() { + expect: 'process memory metrics can be created' + assert objectUnderTest.processMemoryMetrics() != null + and: 'process thread metrics can be created' + assert objectUnderTest.processThreadMetrics() != null + } + def 'Creating gauges for cm handle states.'() { given: 'cache returns value for each state' cmHandlesByState.get(_) >> 1 @@ -45,9 +52,8 @@ class MicroMeterConfigSpec extends Specification { objectUnderTest.deletingCmHandles(simpleMeterRegistry) objectUnderTest.deletedCmHandles(simpleMeterRegistry) then: 'each state has the correct value when queried' - def states = ["ADVISED", "READY", "LOCKED", "DELETING", "DELETED"] - states.each { state -> - def gaugeValue = simpleMeterRegistry.get("cmHandlesByState").tag("state",state).gauge().value() + ['ADVISED', 'READY', 'LOCKED', 'DELETING', 'DELETED'].each { state -> + def gaugeValue = simpleMeterRegistry.get('cmHandlesByState').tag('state',state).gauge().value() assert gaugeValue == 1 } } diff --git a/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java b/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java index 1d39060024..c1d65758c7 100644 --- a/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java +++ b/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java @@ -36,6 +36,7 @@ public class ArchitectureTestBase { "lombok..", "org.apache..", "org.mapstruct..", + "org.opendaylight..", "org.slf4j..", "org.springframework..", "reactor.." diff --git a/cps-application/src/test/java/org/onap/cps/architecture/CpsArchitectureTest.java b/cps-application/src/test/java/org/onap/cps/architecture/CpsArchitectureTest.java index e68343b364..06ca632f36 100644 --- a/cps-application/src/test/java/org/onap/cps/architecture/CpsArchitectureTest.java +++ b/cps-application/src/test/java/org/onap/cps/architecture/CpsArchitectureTest.java @@ -43,11 +43,9 @@ public class CpsArchitectureTest extends ArchitectureTestBase { @ArchTest static final ArchRule cpsServiceImplShouldDependOnServiceAndEventsAndPathParserPackages = - // I think impl package should be moved from the api package. - // So in a way this whole rule is breaking our architecture goals - classes().that().resideInAPackage("org.onap.cps.api.impl..").should().onlyDependOnClassesThat() + classes().that().resideInAPackage("org.onap.cps.impl..").should().onlyDependOnClassesThat() .resideInAnyPackage(commonAndListedPackages("org.onap.cps.api..", - "org.onap.cps.api.impl..", + "org.onap.cps.impl..", "org.onap.cps.events..", "org.onap.cps.impl.utils..", "org.onap.cps.spi..", diff --git a/cps-bom/pom.xml b/cps-bom/pom.xml index 1bb8308e07..f01709f5d1 100644 --- a/cps-bom/pom.xml +++ b/cps-bom/pom.xml @@ -25,7 +25,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>cps-bom</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <packaging>pom</packaging> <description>This artifact contains dependencyManagement declarations of all published CPS components.</description> diff --git a/cps-dependencies/pom.xml b/cps-dependencies/pom.xml index 5cbefa1737..545476c628 100644 --- a/cps-dependencies/pom.xml +++ b/cps-dependencies/pom.xml @@ -27,7 +27,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>cps-dependencies</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <packaging>pom</packaging> <name>${project.groupId}:${project.artifactId}</name> @@ -86,19 +86,19 @@ <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-dependencies</artifactId> - <version>3.2.12</version> + <version>3.4.1</version> <type>pom</type> <scope>import</scope> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-cache</artifactId> - <version>3.2.12</version> + <version>3.4.1</version> </dependency> <dependency> <groupId>org.springframework.cloud</groupId> <artifactId>spring-cloud-dependencies</artifactId> - <version>2023.0.0</version> + <version>2024.0.0</version> <type>pom</type> <scope>import</scope> </dependency> @@ -116,12 +116,12 @@ <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> - <version>2.15.2</version> + <version>2.18.2</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-xml</artifactId> - <version>2.13.5</version> + <version>2.18.2</version> </dependency> <dependency> <groupId>com.github.spotbugs</groupId> @@ -136,7 +136,7 @@ <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> - <version>2.8.9</version> + <version>2.11.0</version> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -146,7 +146,7 @@ <dependency> <groupId>com.hazelcast</groupId> <artifactId>hazelcast-spring</artifactId> - <version>5.3.7</version> + <version>5.5.0</version> </dependency> <dependency> <groupId>com.squareup.okhttp3</groupId> @@ -173,6 +173,11 @@ <scope>import</scope> </dependency> <dependency> + <groupId>io.github.mweirauch</groupId> + <artifactId>micrometer-jvm-extras</artifactId> + <version>0.2.2</version> + </dependency> + <dependency> <groupId>io.gsonfire</groupId> <artifactId>gson-fire</artifactId> <version>1.9.0</version> diff --git a/cps-events/pom.xml b/cps-events/pom.xml index 9d49f181fb..50df9c8aa1 100644 --- a/cps-events/pom.xml +++ b/cps-events/pom.xml @@ -24,7 +24,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-events/pom.xml b/cps-ncmp-events/pom.xml index bb45493554..2667d0924b 100644 --- a/cps-ncmp-events/pom.xml +++ b/cps-ncmp-events/pom.xml @@ -23,7 +23,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml index 690ec01b8f..30a704f509 100644 --- a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml +++ b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml @@ -22,7 +22,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-ncmp-rest-stub</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> </parent> <artifactId>cps-ncmp-rest-stub-app</artifactId> diff --git a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml index bba80cdcf1..18d468e4f4 100644 --- a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml +++ b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml @@ -21,7 +21,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-ncmp-rest-stub</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> </parent> <artifactId>cps-ncmp-rest-stub-service</artifactId> diff --git a/cps-ncmp-rest-stub/pom.xml b/cps-ncmp-rest-stub/pom.xml index 056e52a4f0..b5654262b4 100644 --- a/cps-ncmp-rest-stub/pom.xml +++ b/cps-ncmp-rest-stub/pom.xml @@ -22,7 +22,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-rest/pom.xml b/cps-ncmp-rest/pom.xml index 4e2c48dd6f..2d8cb4dfd2 100644 --- a/cps-ncmp-rest/pom.xml +++ b/cps-ncmp-rest/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-service/pom.xml b/cps-ncmp-service/pom.xml index eb0aed1e7c..70e0b4f8f4 100644 --- a/cps-ncmp-service/pom.xml +++ b/cps-ncmp-service/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/config/ServiceConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/config/ServiceConfig.java index f1fce0c7c6..775e9d7b14 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/config/ServiceConfig.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/config/ServiceConfig.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation. + * Copyright (C) 2024-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,4 +33,5 @@ public abstract class ServiceConfig { private Integer connectionTimeoutInSeconds = 1; private long readTimeoutInSeconds = 1; private long writeTimeoutInSeconds = 1; + private long responseTimeoutInSeconds = 60; } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/AdminCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/AdminCacheConfig.java index a29799b13f..fe933d766f 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/AdminCacheConfig.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/AdminCacheConfig.java @@ -28,7 +28,7 @@ import org.springframework.context.annotation.Configuration; @Configuration public class AdminCacheConfig extends HazelcastCacheConfig { - private static final MapConfig adminCacheMapConfig = createMapConfig("adminCacheMapConfig"); + private static final MapConfig cmHandleStateCacheMapConfig = createMapConfig("cmHandleStateCacheMapConfig"); /** * Distributed instance admin cache map for cm handles by state for use of gauge metrics. @@ -37,13 +37,7 @@ public class AdminCacheConfig extends HazelcastCacheConfig { */ @Bean public IMap<String, Integer> cmHandlesByState() { - final IMap<String, Integer> cmHandlesByState = getOrCreateHazelcastInstance(adminCacheMapConfig).getMap( + return getOrCreateHazelcastInstance(cmHandleStateCacheMapConfig).getMap( "cmHandlesByState"); - cmHandlesByState.putIfAbsent("advisedCmHandlesCount", 0); - cmHandlesByState.putIfAbsent("readyCmHandlesCount", 0); - cmHandlesByState.putIfAbsent("lockedCmHandlesCount", 0); - cmHandlesByState.putIfAbsent("deletingCmHandlesCount", 0); - cmHandlesByState.putIfAbsent("deletedCmHandlesCount", 0); - return cmHandlesByState; } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java index 75007e2e35..1a7ef758d8 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================== - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import com.hazelcast.config.MapConfig; import com.hazelcast.config.NamedConfig; import com.hazelcast.config.NearCacheConfig; import com.hazelcast.config.QueueConfig; -import com.hazelcast.config.RestEndpointGroup; import com.hazelcast.config.SetConfig; import com.hazelcast.core.Hazelcast; import com.hazelcast.core.HazelcastInstance; @@ -61,7 +60,6 @@ public class HazelcastCacheConfig { config.setClusterName(clusterName); config.setClassLoader(Dataspace.class.getClassLoader()); configureDataStructures(namedConfig, config); - exposeClusterInformation(config); updateDiscoveryMode(config); return config; } @@ -130,9 +128,4 @@ public class HazelcastCacheConfig { } } - protected void exposeClusterInformation(final Config config) { - config.getNetworkConfig().getRestApiConfig().setEnabled(true) - .enableGroups(RestEndpointGroup.HEALTH_CHECK, RestEndpointGroup.CLUSTER_READ); - } - } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandler.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandler.java index a118d53e7e..d74863a710 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandler.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandler.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -79,8 +79,11 @@ public class DmiSubJobRequestHandler { jsonObjectMapper.asJsonString(subJobWriteRequest), OperationType.CREATE, authorization); - final SubJobWriteResponse subJobWriteResponse = jsonObjectMapper - .convertToValueType(responseEntity.getBody(), SubJobWriteResponse.class); + final Map<String, String> responseAsKeyValuePairs = jsonObjectMapper + .convertToValueType(responseEntity.getBody(), Map.class); + final String subJobId = responseAsKeyValuePairs.get("subJobId"); + final SubJobWriteResponse subJobWriteResponse = new SubJobWriteResponse(subJobId, + producerKey.dmiServiceName(), producerKey.dataProducerIdentifier()); log.debug("Sub job write response: {}", subJobWriteResponse); subJobWriteResponses.add(subJobWriteResponse); }); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java index 59d0f9704e..ae913ddfe7 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java @@ -38,7 +38,6 @@ import org.onap.cps.api.CpsQueryService; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.cpspath.parser.CpsPathUtil; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.ncmp.api.inventory.DataStoreSyncState; import org.onap.cps.ncmp.api.inventory.models.CmHandleState; import org.onap.cps.ncmp.api.inventory.models.TrustLevel; @@ -46,6 +45,7 @@ import org.onap.cps.ncmp.impl.inventory.models.ModelledDmiServiceLeaves; import org.onap.cps.ncmp.impl.inventory.models.PropertyType; import org.onap.cps.ncmp.impl.inventory.trustlevel.TrustLevelCacheConfig; import org.onap.cps.ncmp.impl.utils.YangDataConverter; +import org.onap.cps.utils.CpsValidator; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java index fc215c9c01..75c52f3c60 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2021-2022 Bell Canada * Modifications Copyright (C) 2023 TechMahindra Ltd. @@ -23,7 +23,6 @@ package org.onap.cps.ncmp.impl.inventory; -import static org.onap.cps.ncmp.api.NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_FOUND; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_READY; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST; @@ -138,17 +137,20 @@ public class CmHandleRegistrationService { protected void processRemovedCmHandles(final DmiPluginRegistration dmiPluginRegistration, final DmiPluginRegistrationResponse dmiPluginRegistrationResponse) { - final List<String> tobeRemovedCmHandleIds = dmiPluginRegistration.getRemovedCmHandles(); + final List<String> toBeRemovedCmHandleIds = dmiPluginRegistration.getRemovedCmHandles(); + if (toBeRemovedCmHandleIds.isEmpty()) { + return; + } final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = - new ArrayList<>(tobeRemovedCmHandleIds.size()); + new ArrayList<>(toBeRemovedCmHandleIds.size()); final Collection<YangModelCmHandle> yangModelCmHandles = - inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandleIds); + inventoryPersistence.getYangModelCmHandles(toBeRemovedCmHandleIds); updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING); final Set<String> notDeletedCmHandles = new HashSet<>(); - for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandleIds, DELETE_BATCH_SIZE)) { + for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(toBeRemovedCmHandleIds, DELETE_BATCH_SIZE)) { try { - batchDeleteCmHandlesFromDbAndCaches(tobeRemovedCmHandleBatch); + deleteCmHandlesFromDbAndCaches(tobeRemovedCmHandleBatch); tobeRemovedCmHandleBatch.forEach(cmHandleId -> cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId))); @@ -201,8 +203,9 @@ public class CmHandleRegistrationService { protected void processUpdatedCmHandles(final DmiPluginRegistration dmiPluginRegistration, final DmiPluginRegistrationResponse dmiPluginRegistrationResponse) { - dmiPluginRegistrationResponse.setUpdatedCmHandles(cmHandleRegistrationServicePropertyHandler - .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles())); + final List<CmHandleRegistrationResponse> updatedCmHandles = cmHandleRegistrationServicePropertyHandler + .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles()); + dmiPluginRegistrationResponse.setUpdatedCmHandles(updatedCmHandles); } protected void processUpgradedCmHandles( @@ -275,7 +278,7 @@ public class CmHandleRegistrationService { private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) { try { - deleteCmHandleFromDbAndCaches(cmHandleId); + deleteCmHandlesFromDbAndCaches(Collections.singletonList(cmHandleId)); return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId); } catch (final DataNodeNotFoundException dataNodeNotFoundException) { log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}", @@ -298,16 +301,9 @@ public class CmHandleRegistrationService { lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle); } - private void deleteCmHandleFromDbAndCaches(final String cmHandleId) { - inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId); - inventoryPersistence.deleteDataNode(NCMP_DMI_REGISTRY_PARENT + "/cm-handles[@id='" + cmHandleId + "']"); - trustLevelManager.removeCmHandles(Collections.singleton(cmHandleId)); - removeDeletedCmHandleFromModuleSyncMap(cmHandleId); - } - - private void batchDeleteCmHandlesFromDbAndCaches(final Collection<String> cmHandleIds) { - inventoryPersistence.deleteSchemaSetsWithCascade(cmHandleIds); + private void deleteCmHandlesFromDbAndCaches(final Collection<String> cmHandleIds) { inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(cmHandleIds)); + inventoryPersistence.deleteAnchors(cmHandleIds); trustLevelManager.removeCmHandles(cmHandleIds); cmHandleIds.forEach(this::removeDeletedCmHandleFromModuleSyncMap); } @@ -326,8 +322,11 @@ public class CmHandleRegistrationService { private List<CmHandleRegistrationResponse> upgradeCmHandles(final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle) { + if (cmHandleStatePerCmHandle.isEmpty()) { + return Collections.emptyList(); + } final List<String> cmHandleIds = getCmHandleIds(cmHandleStatePerCmHandle); - log.info("Moving cm handles : {} into locked (for upgrade) state.", cmHandleIds); + log.info("Moving {} cm handles into locked (for upgrade) state: {} ", cmHandleIds.size(), cmHandleIds); try { lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle); return CmHandleRegistrationResponse.createSuccessResponses(cmHandleIds); @@ -347,7 +346,7 @@ public class CmHandleRegistrationService { final Collection<String> rejectedCmHandleIds = alternateIdChecker .getIdsOfCmHandlesWithRejectedAlternateId(cmHandlesToBeCreated, AlternateIdChecker.Operation.CREATE); cmHandleRegistrationResponses.addAll(CmHandleRegistrationResponse.createFailureResponses( - rejectedCmHandleIds, ALTERNATE_ID_ALREADY_ASSOCIATED)); + rejectedCmHandleIds, CM_HANDLE_ALREADY_EXIST)); return rejectedCmHandleIds; } @@ -384,5 +383,4 @@ public class CmHandleRegistrationService { ncmpServiceCmHandle.getDataProducerIdentifier()); } - } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java index b7a13d9989..3415793478 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ @@ -22,8 +22,8 @@ package org.onap.cps.ncmp.impl.inventory; -import static org.onap.cps.ncmp.api.NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_FOUND; +import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_INVALID_ID; import static org.onap.cps.ncmp.impl.inventory.CmHandleRegistrationServicePropertyHandler.PropertyType.DMI_PROPERTY; import static org.onap.cps.ncmp.impl.inventory.CmHandleRegistrationServicePropertyHandler.PropertyType.PUBLIC_PROPERTY; @@ -49,7 +49,7 @@ import org.onap.cps.api.CpsDataService; import org.onap.cps.api.exceptions.DataNodeNotFoundException; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; +import org.onap.cps.impl.DataNodeBuilder; import org.onap.cps.ncmp.api.inventory.models.CmHandleRegistrationResponse; import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; @@ -81,7 +81,7 @@ public class CmHandleRegistrationServicePropertyHandler { final Collection<String> rejectedCmHandleIds = alternateIdChecker .getIdsOfCmHandlesWithRejectedAlternateId(updatedNcmpServiceCmHandles, AlternateIdChecker.Operation.UPDATE); final List<CmHandleRegistrationResponse> failureResponses = - CmHandleRegistrationResponse.createFailureResponses(rejectedCmHandleIds, ALTERNATE_ID_ALREADY_ASSOCIATED); + CmHandleRegistrationResponse.createFailureResponses(rejectedCmHandleIds, CM_HANDLE_ALREADY_EXIST); final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>(failureResponses); for (final NcmpServiceCmHandle updatedNcmpServiceCmHandle : updatedNcmpServiceCmHandles) { final String cmHandleId = updatedNcmpServiceCmHandle.getCmHandleId(); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java index d566ae43cb..e145c62921 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ @@ -45,13 +45,13 @@ import org.onap.cps.api.model.DataNode; import org.onap.cps.api.model.ModuleDefinition; import org.onap.cps.api.model.ModuleReference; import org.onap.cps.api.parameters.FetchDescendantsOption; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException; import org.onap.cps.ncmp.api.inventory.models.CompositeState; import org.onap.cps.ncmp.api.inventory.models.CompositeStateBuilder; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; import org.onap.cps.ncmp.impl.utils.YangDataConverter; import org.onap.cps.utils.ContentType; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.utils.JsonObjectMapper; import org.springframework.stereotype.Component; @@ -62,26 +62,27 @@ public class InventoryPersistenceImpl extends NcmpPersistenceImpl implements Inv private static final int CMHANDLE_BATCH_SIZE = 100; private final CpsModuleService cpsModuleService; - private final CpsAnchorService cpsAnchorService; private final CpsValidator cpsValidator; private final CmHandleQueryService cmHandleQueryService; /** * initialize an inventory persistence object. * - * @param jsonObjectMapper json mapper object - * @param cpsDataService cps data service instance - * @param cpsModuleService cps module service instance - * @param cpsValidator cps validation service instance - * @param cpsAnchorService cps anchor service instance + * @param cpsValidator cps validation service instance + * @param jsonObjectMapper json mapper object + * @param cpsAnchorService cps anchor service instance + * @param cpsModuleService cps module service instance + * @param cpsDataService cps data service instance + * @param cmHandleQueryService cm handle query service instance */ - public InventoryPersistenceImpl(final JsonObjectMapper jsonObjectMapper, final CpsDataService cpsDataService, - final CpsModuleService cpsModuleService, final CpsValidator cpsValidator, + public InventoryPersistenceImpl(final CpsValidator cpsValidator, + final JsonObjectMapper jsonObjectMapper, final CpsAnchorService cpsAnchorService, + final CpsModuleService cpsModuleService, + final CpsDataService cpsDataService, final CmHandleQueryService cmHandleQueryService) { - super(jsonObjectMapper, cpsDataService, cpsModuleService, cpsValidator); + super(jsonObjectMapper, cpsAnchorService, cpsDataService); this.cpsModuleService = cpsModuleService; - this.cpsAnchorService = cpsAnchorService; this.cpsValidator = cpsValidator; this.cmHandleQueryService = cmHandleQueryService; } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistence.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistence.java index 714a7ca12f..f327edab17 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistence.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistence.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,9 +25,6 @@ import java.util.Collection; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.parameters.FetchDescendantsOption; -/** - * DmiRegistryConstants class to be strictly used for DMI Related constants only. - */ public interface NcmpPersistence { String NCMP_DATASPACE_NAME = "NCMP-Admin"; @@ -44,20 +41,6 @@ public interface NcmpPersistence { void deleteListOrListElement(String listElementXpath); /** - * Method to delete a schema set. - * - * @param schemaSetName schema set name - */ - void deleteSchemaSetWithCascade(String schemaSetName); - - /** - * Method to delete multiple schema sets. - * - * @param schemaSetNames schema set names - */ - void deleteSchemaSetsWithCascade(Collection<String> schemaSetNames); - - /** * Get data node via xpath. * * @param xpath xpath @@ -113,4 +96,12 @@ public interface NcmpPersistence { * @param dataNodeXpaths data node xpaths */ void deleteDataNodes(Collection<String> dataNodeXpaths); + + /** + * Deletes multiple anchors identified by their IDs. + * + * @param anchorIds ids of the anchors to be deleted + */ + void deleteAnchors(Collection<String> anchorIds); + } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistenceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistenceImpl.java index 6092d8b3b9..2232d7ce12 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistenceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/NcmpPersistenceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,31 +20,25 @@ package org.onap.cps.ncmp.impl.inventory; -import static org.onap.cps.api.parameters.CascadeDeleteAllowed.CASCADE_DELETE_ALLOWED; import static org.onap.cps.api.parameters.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS; import io.micrometer.core.annotation.Timed; import java.util.Collection; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.CpsDataService; -import org.onap.cps.api.CpsModuleService; -import org.onap.cps.api.exceptions.SchemaSetNotFoundException; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.parameters.FetchDescendantsOption; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.utils.JsonObjectMapper; import org.springframework.stereotype.Component; -@Slf4j @RequiredArgsConstructor @Component public class NcmpPersistenceImpl implements NcmpPersistence { protected final JsonObjectMapper jsonObjectMapper; + protected final CpsAnchorService cpsAnchorService; protected final CpsDataService cpsDataService; - private final CpsModuleService cpsModuleService; - private final CpsValidator cpsValidator; @Override public void deleteListOrListElement(final String listElementXpath) { @@ -53,27 +47,6 @@ public class NcmpPersistenceImpl implements NcmpPersistence { } @Override - @Timed(value = "cps.ncmp.inventory.persistence.schemaset.delete", - description = "Time taken to delete a schemaset") - public void deleteSchemaSetWithCascade(final String schemaSetName) { - try { - cpsValidator.validateNameCharacters(schemaSetName); - cpsModuleService.deleteSchemaSet(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetName, - CASCADE_DELETE_ALLOWED); - } catch (final SchemaSetNotFoundException schemaSetNotFoundException) { - log.warn("Schema set {} does not exist or already deleted", schemaSetName); - } - } - - @Override - @Timed(value = "cps.ncmp.inventory.persistence.schemaset.delete.batch", - description = "Time taken to delete multiple schemaset") - public void deleteSchemaSetsWithCascade(final Collection<String> schemaSetNames) { - cpsValidator.validateNameCharacters(schemaSetNames); - cpsModuleService.deleteSchemaSetsWithCascade(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetNames); - } - - @Override @Timed(value = "cps.ncmp.inventory.persistence.datanode.get", description = "Time taken to get a data node (from ncmp dmi registry)") public Collection<DataNode> getDataNode(final String xpath) { @@ -116,4 +89,9 @@ public class NcmpPersistenceImpl implements NcmpPersistence { cpsDataService.deleteDataNodes(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpaths, NO_TIMESTAMP); } + @Override + public void deleteAnchors(final Collection<String> anchorIds) { + cpsAnchorService.deleteAnchors(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, anchorIds); + } + } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java index 1e24671f8d..2cc4375447 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,13 +61,14 @@ public class DmiModelOperations { * Retrieves module references. * * @param yangModelCmHandle the yang model cm handle + * @param targetModuleSetTag module set tag to send to dmi * @return module references */ @Timed(value = "cps.ncmp.inventory.module.references.from.dmi", description = "Time taken to get all module references for a cm handle from dmi") - public List<ModuleReference> getModuleReferences(final YangModelCmHandle yangModelCmHandle) { - final DmiRequestBody dmiRequestBody = DmiRequestBody.builder() - .moduleSetTag(yangModelCmHandle.getModuleSetTag()).build(); + public List<ModuleReference> getModuleReferences(final YangModelCmHandle yangModelCmHandle, + final String targetModuleSetTag) { + final DmiRequestBody dmiRequestBody = DmiRequestBody.builder().moduleSetTag(targetModuleSetTag).build(); dmiRequestBody.asDmiProperties(yangModelCmHandle.getDmiProperties()); final ResponseEntity<Object> dmiFetchModulesResponseEntity = getResourceFromDmiWithJsonData( yangModelCmHandle.resolveDmiServiceName(MODEL), @@ -79,18 +80,20 @@ public class DmiModelOperations { * Retrieve yang resources from dmi for any modules that CPS-NCMP hasn't cached before. * * @param yangModelCmHandle the yangModelCmHandle + * @param targetModuleSetTag module set tag to send to dmi * @param newModuleReferences the unknown module references * @return yang resources as map of module name to yang(re)source */ @Timed(value = "cps.ncmp.inventory.yang.resources.from.dmi", description = "Time taken to get list of yang resources from dmi") public Map<String, String> getNewYangResourcesFromDmi(final YangModelCmHandle yangModelCmHandle, + final String targetModuleSetTag, final Collection<ModuleReference> newModuleReferences) { if (newModuleReferences.isEmpty()) { return Collections.emptyMap(); } final String jsonWithDataAndDmiProperties = getRequestBodyToFetchYangResources(newModuleReferences, - yangModelCmHandle.getDmiProperties(), yangModelCmHandle.getModuleSetTag()); + yangModelCmHandle.getDmiProperties(), targetModuleSetTag); final ResponseEntity<Object> responseEntity = getResourceFromDmiWithJsonData( yangModelCmHandle.resolveDmiServiceName(MODEL), jsonWithDataAndDmiProperties, @@ -123,13 +126,13 @@ public class DmiModelOperations { private static String getRequestBodyToFetchYangResources(final Collection<ModuleReference> newModuleReferences, final List<YangModelCmHandle.Property> dmiProperties, - final String moduleSetTag) { + final String targetModuleSetTag) { final JsonArray moduleReferencesAsJson = getModuleReferencesAsJson(newModuleReferences); final JsonObject data = new JsonObject(); data.add("modules", moduleReferencesAsJson); final JsonObject jsonRequestObject = new JsonObject(); - if (!moduleSetTag.isEmpty()) { - jsonRequestObject.addProperty("moduleSetTag", moduleSetTag); + if (!targetModuleSetTag.isEmpty()) { + jsonRequestObject.addProperty("moduleSetTag", targetModuleSetTag); } jsonRequestObject.add("data", data); jsonRequestObject.add("cmHandleProperties", toJsonObject(dmiProperties)); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java index 994ca80287..80e41652ee 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -189,7 +189,12 @@ public class ModuleOperationsUtils { .getLockReasonCategory())); } - public static String getUpgradedModuleSetTagFromLockReason(final CompositeState.LockReason lockReason) { + public static String getTargetModuleSetTagForUpgrade(final YangModelCmHandle yangModelCmHandle) { + final CompositeState.LockReason lockReason = yangModelCmHandle.getCompositeState().getLockReason(); + return getTargetModuleSetTagFromLockReason(lockReason); + } + + private static String getTargetModuleSetTagFromLockReason(final CompositeState.LockReason lockReason) { return getLockedCompositeStateDetails(lockReason).getOrDefault(MODULE_SET_TAG_KEY, ""); } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java index 3f92dc73f0..f929d6708c 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,24 +26,17 @@ import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT; import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME; -import com.hazelcast.collection.ISet; import java.time.OffsetDateTime; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import lombok.AllArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.util.Strings; import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; -import org.onap.cps.api.exceptions.SchemaSetNotFoundException; +import org.onap.cps.api.exceptions.AlreadyDefinedException; import org.onap.cps.api.model.ModuleReference; -import org.onap.cps.api.parameters.CascadeDeleteAllowed; -import org.onap.cps.ncmp.api.exceptions.NcmpException; -import org.onap.cps.ncmp.api.inventory.models.CmHandleState; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; import org.onap.cps.utils.ContentType; import org.onap.cps.utils.JsonObjectMapper; @@ -54,15 +47,11 @@ import org.springframework.stereotype.Service; @RequiredArgsConstructor public class ModuleSyncService { - private static final Map<String, String> NO_NEW_MODULES = Collections.emptyMap(); - private final DmiModelOperations dmiModelOperations; private final CpsModuleService cpsModuleService; private final CpsDataService cpsDataService; private final CpsAnchorService cpsAnchorService; private final JsonObjectMapper jsonObjectMapper; - private final ISet<String> moduleSetTagsBeingProcessed; - private final Map<String, ModuleDelta> privateModuleSetCache = new HashMap<>(); @AllArgsConstructor private static final class ModuleDelta { @@ -71,41 +60,20 @@ public class ModuleSyncService { } /** - * This method creates a cm handle and initiates modules sync. + * Creates a CM handle and initiates the synchronization of modules to create a schema set and anchor. * * @param yangModelCmHandle the yang model of cm handle. */ public void syncAndCreateSchemaSetAndAnchor(final YangModelCmHandle yangModelCmHandle) { - final String moduleSetTag = yangModelCmHandle.getModuleSetTag(); - final ModuleDelta moduleDelta; - boolean isNewModuleSetTag = Strings.isNotBlank(moduleSetTag); + final String cmHandleId = yangModelCmHandle.getId(); + final String targetModuleSetTag = yangModelCmHandle.getModuleSetTag(); + final String schemaSetName = getSchemaSetNameForModuleSetTag(cmHandleId, targetModuleSetTag); + syncAndCreateSchemaSet(yangModelCmHandle, schemaSetName, targetModuleSetTag); try { - if (privateModuleSetCache.containsKey(moduleSetTag)) { - moduleDelta = privateModuleSetCache.get(moduleSetTag); - } else { - if (isNewModuleSetTag) { - if (moduleSetTagsBeingProcessed.add(moduleSetTag)) { - log.info("Processing new module set tag {}", moduleSetTag); - } else { - isNewModuleSetTag = false; - throw new NcmpException("Concurrent processing of module set tag " + moduleSetTag, - moduleSetTag + " already being processed for cm handle " + yangModelCmHandle.getId()); - } - } - moduleDelta = getModuleDelta(yangModelCmHandle, moduleSetTag); - } - final String cmHandleId = yangModelCmHandle.getId(); - cpsModuleService.createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, - moduleDelta.newModuleNameToContentMap, moduleDelta.allModuleReferences); - cpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, cmHandleId); - if (isNewModuleSetTag) { - final ModuleDelta noModuleDelta = new ModuleDelta(moduleDelta.allModuleReferences, NO_NEW_MODULES); - privateModuleSetCache.put(moduleSetTag, noModuleDelta); - } - } finally { - if (isNewModuleSetTag) { - moduleSetTagsBeingProcessed.remove(moduleSetTag); - } + cpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetName, cmHandleId); + } catch (final AlreadyDefinedException alreadyDefinedException) { + log.warn("Ignoring (Anchor) already exists exception for {}. Exception details: {}", cmHandleId, + alreadyDefinedException.getDetails()); } } @@ -115,55 +83,63 @@ public class ModuleSyncService { * @param yangModelCmHandle the yang model of cm handle. */ public void syncAndUpgradeSchemaSet(final YangModelCmHandle yangModelCmHandle) { - final String upgradedModuleSetTag = ModuleOperationsUtils.getUpgradedModuleSetTagFromLockReason( - yangModelCmHandle.getCompositeState().getLockReason()); - final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle, upgradedModuleSetTag); - cpsModuleService.upgradeSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, - yangModelCmHandle.getId(), moduleDelta.newModuleNameToContentMap, moduleDelta.allModuleReferences); - setCmHandleModuleSetTag(yangModelCmHandle, upgradedModuleSetTag); + final String cmHandleId = yangModelCmHandle.getId(); + final String sourceModuleSetTag = yangModelCmHandle.getModuleSetTag(); + final String targetModuleSetTag = ModuleOperationsUtils.getTargetModuleSetTagForUpgrade(yangModelCmHandle); + final String schemaSetName = getSchemaSetNameForModuleSetTag(cmHandleId, targetModuleSetTag); + if (sourceModuleSetTag.isEmpty() && targetModuleSetTag.isEmpty()) { + final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle, targetModuleSetTag); + cpsModuleService.upgradeSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, + schemaSetName, moduleDelta.newModuleNameToContentMap, moduleDelta.allModuleReferences); + } else { + syncAndCreateSchemaSet(yangModelCmHandle, schemaSetName, targetModuleSetTag); + cpsAnchorService.updateAnchorSchemaSet(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, schemaSetName); + setCmHandleModuleSetTag(yangModelCmHandle, targetModuleSetTag); + log.info("Upgrading schema set for CM handle ID: {}, Source Tag: {}, Target Tag: {}", + cmHandleId, sourceModuleSetTag, targetModuleSetTag); + } } - /** - * Deletes the SchemaSet for schema set id if the SchemaSet Exists. - * - * @param schemaSetId the schema set id to be deleted - */ - public void deleteSchemaSetIfExists(final String schemaSetId) { - try { - cpsModuleService.deleteSchemaSet(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetId, - CascadeDeleteAllowed.CASCADE_DELETE_ALLOWED); - log.debug("SchemaSet for {} has been deleted. Ready to be recreated.", schemaSetId); - } catch (final SchemaSetNotFoundException e) { - log.debug("No SchemaSet for {}. Assuming CmHandle has not been previously Module Synced.", schemaSetId); + private void syncAndCreateSchemaSet(final YangModelCmHandle yangModelCmHandle, + final String schemaSetName, + final String targetModuleSetTag) { + if (isNewSchemaSet(schemaSetName)) { + final String cmHandleId = yangModelCmHandle.getId(); + final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle, targetModuleSetTag); + try { + log.info("Creating Schema Set {} for CM Handle {}", schemaSetName, cmHandleId); + cpsModuleService.createSchemaSetFromModules( + NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, + schemaSetName, + moduleDelta.newModuleNameToContentMap, + moduleDelta.allModuleReferences + ); + log.info("Successfully created Schema Set {} for CM Handle {}", schemaSetName, + yangModelCmHandle.getId()); + } catch (final AlreadyDefinedException alreadyDefinedException) { + log.warn("Ignoring (Schema Set) already exists exception for {}. Exception details: {}", cmHandleId, + alreadyDefinedException.getDetails()); + } } } - public void clearPrivateModuleSetCache() { - privateModuleSetCache.clear(); + private boolean isNewSchemaSet(final String schemaSetName) { + return !cpsModuleService.schemaSetExists(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetName); } - private ModuleDelta getModuleDelta(final YangModelCmHandle yangModelCmHandle, final String targetModuleSetTag) { - final Map<String, String> newYangResources; - Collection<ModuleReference> allModuleReferences = getModuleReferencesByModuleSetTag(targetModuleSetTag); - if (allModuleReferences.isEmpty()) { - allModuleReferences = dmiModelOperations.getModuleReferences(yangModelCmHandle); - newYangResources = dmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, - cpsModuleService.identifyNewModuleReferences(allModuleReferences)); - } else { - log.info("Found other cm handle having same module set tag: {}", targetModuleSetTag); - newYangResources = NO_NEW_MODULES; - } + private ModuleDelta getModuleDelta(final YangModelCmHandle yangModelCmHandle, + final String targetModuleSetTag) { + final Collection<ModuleReference> allModuleReferences = + dmiModelOperations.getModuleReferences(yangModelCmHandle, targetModuleSetTag); + final Collection<ModuleReference> newModuleReferences = + cpsModuleService.identifyNewModuleReferences(allModuleReferences); + final Map<String, String> newYangResources = dmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, + targetModuleSetTag, newModuleReferences); + log.debug("Module delta calculated for CM handle ID: {}. All references: {}. New modules: {}", + yangModelCmHandle.getId(), allModuleReferences, newYangResources.keySet()); return new ModuleDelta(allModuleReferences, newYangResources); } - private Collection<ModuleReference> getModuleReferencesByModuleSetTag(final String moduleSetTag) { - if (Strings.isBlank(moduleSetTag)) { - return Collections.emptyList(); - } - return cpsModuleService.getModuleReferencesByAttribute(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, - Map.of("module-set-tag", moduleSetTag), Map.of("cm-handle-state", CmHandleState.READY.name())); - } - private void setCmHandleModuleSetTag(final YangModelCmHandle yangModelCmHandle, final String newModuleSetTag) { final String jsonForUpdate = jsonObjectMapper.asJsonString(Map.of( "cm-handles", Map.of("id", yangModelCmHandle.getId(), "module-set-tag", newModuleSetTag))); @@ -171,4 +147,8 @@ public class ModuleSyncService { jsonForUpdate, OffsetDateTime.now(), ContentType.JSON); } + private static String getSchemaSetNameForModuleSetTag(final String cmHandleId, final String moduleSetTag) { + return moduleSetTag.isEmpty() ? cmHandleId : moduleSetTag; + } + } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java index 5f289c2c01..f039cf3c02 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -109,13 +109,12 @@ public class ModuleSyncTasks { if (inUpgrade) { moduleSyncService.syncAndUpgradeSchemaSet(yangModelCmHandle); } else { - moduleSyncService.deleteSchemaSetIfExists(yangModelCmHandle.getId()); moduleSyncService.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle); } compositeState.setLockReason(null); return CmHandleState.READY; } catch (final Exception e) { - log.warn("Processing of {} module failed due to reason {}.", yangModelCmHandle.getId(), e.getMessage()); + log.warn("Processing of {} failed, reason: {}.", yangModelCmHandle.getId(), e.getMessage()); final LockReasonCategory lockReasonCategory = inUpgrade ? LockReasonCategory.MODULE_UPGRADE_FAILED : LockReasonCategory.MODULE_SYNC_FAILED; diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java index d6ac242b30..c05944f66e 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================== - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,13 +20,10 @@ package org.onap.cps.ncmp.impl.inventory.sync; -import com.hazelcast.collection.ISet; import com.hazelcast.config.MapConfig; import com.hazelcast.config.QueueConfig; -import com.hazelcast.config.SetConfig; import com.hazelcast.map.IMap; import java.util.concurrent.BlockingQueue; -import lombok.extern.slf4j.Slf4j; import org.onap.cps.ncmp.impl.cache.HazelcastCacheConfig; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -34,7 +31,6 @@ import org.springframework.context.annotation.Configuration; /** * Core infrastructure of the hazelcast distributed caches for Module Sync and Data Sync use cases. */ -@Slf4j @Configuration public class SynchronizationCacheConfig extends HazelcastCacheConfig { @@ -45,8 +41,6 @@ public class SynchronizationCacheConfig extends HazelcastCacheConfig { private static final MapConfig moduleSyncStartedConfig = createMapConfigWithTimeToLiveInSeconds("moduleSyncStartedConfig", MODULE_SYNC_STARTED_TTL_SECS); private static final MapConfig dataSyncSemaphoresConfig = createMapConfig("dataSyncSemaphoresConfig"); - private static final SetConfig moduleSetTagsBeingProcessedConfig - = createSetConfig("moduleSetTagsBeingProcessedConfig"); /** * Module Sync Distributed Queue Instance. @@ -78,14 +72,4 @@ public class SynchronizationCacheConfig extends HazelcastCacheConfig { return getOrCreateHazelcastInstance(dataSyncSemaphoresConfig).getMap("dataSyncSemaphores"); } - /** - * Collection of (new) module set tags being processed. - * To prevent processing on multiple threads of same tag - * - * @return set of module set tags being processed - */ - @Bean - public ISet<String> moduleSetTagsBeingProcessed() { - return getOrCreateHazelcastInstance(moduleSetTagsBeingProcessedConfig).getSet("moduleSetTagsBeingProcessed"); - } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitor.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitor.java index 31270ca9fc..708508e9d8 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitor.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitor.java @@ -25,18 +25,42 @@ import com.hazelcast.map.IMap; import java.util.Collection; import java.util.Map; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.onap.cps.ncmp.api.inventory.models.CmHandleState; import org.onap.cps.ncmp.api.inventory.models.CompositeState; +import org.onap.cps.ncmp.impl.inventory.CmHandleQueryService; import org.onap.cps.ncmp.impl.inventory.sync.lcm.LcmEventsCmHandleStateHandlerImpl.CmHandleTransitionPair; +import org.onap.cps.ncmp.utils.events.NcmpInventoryModelOnboardingFinishedEvent; +import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; @Component @RequiredArgsConstructor +@Slf4j public class CmHandleStateMonitor { - private static final String METRIC_POSTFIX = "CmHandlesCount"; - final IMap<String, Integer> cmHandlesByState; + + private final CmHandleQueryService cmHandleQueryService; + private final IMap<String, Integer> cmHandlesByState; + + /** + * Method to initialise cm handle state monitor by querying the current state counts + * and storing them in the provided map. This method is triggered by NcmpInventoryModelOnboardingFinishedEvent. + * + * @param ncmpInventoryModelOnboardingFinishedEvent the event that triggers the initialization + */ + @EventListener + public void initialiseCmHandleStateMonitor( + final NcmpInventoryModelOnboardingFinishedEvent ncmpInventoryModelOnboardingFinishedEvent) { + for (final CmHandleState cmHandleState : CmHandleState.values()) { + final String cmHandleStateAsString = cmHandleState.name().toLowerCase(); + final String stateMetricKey = cmHandleStateAsString + METRIC_POSTFIX; + final int cmHandleCountForState = cmHandleQueryService.queryCmHandleIdsByState(cmHandleState).size(); + cmHandlesByState.putIfAbsent(stateMetricKey, cmHandleCountForState); + log.info("Cm handle state monitor has set " + stateMetricKey + " to " + cmHandleCountForState); + } + } /** * Asynchronously update the cm handle state metrics. diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/http/WebClientConfiguration.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/http/WebClientConfiguration.java index eefabd1079..0214c0c2a9 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/http/WebClientConfiguration.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/http/WebClientConfiguration.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation. + * Copyright (C) 2024-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,8 +41,6 @@ import reactor.netty.resources.ConnectionProvider; */ public class WebClientConfiguration { - private static final Duration DEFAULT_RESPONSE_TIMEOUT = Duration.ofSeconds(30); - protected WebClient configureWebClient(final WebClient.Builder webClientBuilder, final ServiceConfig serviceConfig) { final ConnectionProvider connectionProvider = getConnectionProvider(serviceConfig); @@ -53,7 +51,7 @@ public class WebClientConfiguration { private static HttpClient createHttpClient(final ServiceConfig serviceConfig, final ConnectionProvider connectionProvider) { return HttpClient.create(connectionProvider) - .responseTimeout(DEFAULT_RESPONSE_TIMEOUT) + .responseTimeout(Duration.ofSeconds(serviceConfig.getResponseTimeoutInSeconds())) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, serviceConfig.getConnectionTimeoutInSeconds() * 1000) .doOnConnected(connection -> connection.addHandlerLast(new ReadTimeoutHandler( serviceConfig.getReadTimeoutInSeconds(), TimeUnit.SECONDS)).addHandlerLast( diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java index 58a5f553af..514d9b8fe4 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java @@ -30,11 +30,15 @@ import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsDataspaceService; import org.onap.cps.api.CpsModuleService; import org.onap.cps.init.AbstractModelLoader; +import org.onap.cps.ncmp.utils.events.NcmpInventoryModelOnboardingFinishedEvent; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @Slf4j @Service public class InventoryModelLoader extends AbstractModelLoader { + + private final ApplicationEventPublisher applicationEventPublisher; private static final String NEW_MODEL_FILE_NAME = "dmi-registry@2024-02-23.yang"; private static final String NEW_SCHEMA_SET_NAME = "dmi-registry-2024-02-23"; private static final String REGISTRY_DATANODE_NAME = "dmi-registry"; @@ -42,14 +46,17 @@ public class InventoryModelLoader extends AbstractModelLoader { public InventoryModelLoader(final CpsDataspaceService cpsDataspaceService, final CpsModuleService cpsModuleService, final CpsAnchorService cpsAnchorService, - final CpsDataService cpsDataService) { + final CpsDataService cpsDataService, + final ApplicationEventPublisher applicationEventPublisher) { super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService); + this.applicationEventPublisher = applicationEventPublisher; } @Override public void onboardOrUpgradeModel() { updateInventoryModel(); log.info("Inventory Model updated successfully"); + applicationEventPublisher.publishEvent(new NcmpInventoryModelOnboardingFinishedEvent(this)); } private void updateInventoryModel() { diff --git a/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/utils/events/NcmpInventoryModelOnboardingFinishedEvent.java index 5106d29fa5..92d5e8241d 100644 --- a/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/utils/events/NcmpInventoryModelOnboardingFinishedEvent.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,21 +18,22 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.init +package org.onap.cps.ncmp.utils.events; -import org.onap.cps.api.CpsModuleService -import spock.lang.Specification +import org.springframework.context.ApplicationEvent; -class DbCleanerSpec extends Specification { - - def mockCpsModuleService = Mock(CpsModuleService) - - def objectUnderTest = new DbCleaner(mockCpsModuleService) +/** + * Custom event triggered when the NCMP inventory model onboarding process is completed. + * Extends Spring's {@link ApplicationEvent} to be used within Spring's event-driven architecture. + */ +public class NcmpInventoryModelOnboardingFinishedEvent extends ApplicationEvent { - def 'DB clean up.'() { - when: 'scheduled method is triggered' - objectUnderTest.cleanDbAtStartUp() - then: 'the unused yang resource modules are deleted' - 1 * mockCpsModuleService.deleteUnusedYangResourceModules() + /** + * Creates a new instance of NcmpModelOnboardingFinishedEvent. + * + * @param source the object that is the source of the event (i.e. the source that completed the onboarding process) + */ + public NcmpInventoryModelOnboardingFinishedEvent(final Object source) { + super(source); } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy index 055a6e7448..c49af0f01b 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2022 Bell Canada - * Modifications Copyright (C) 2023-2024 Nordix Foundation + * Modifications Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import spock.lang.Specification import java.util.stream.Collectors -import static org.onap.cps.ncmp.api.NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNKNOWN_ERROR @@ -89,14 +88,14 @@ class CmHandleRegistrationResponseSpec extends Specification { } def 'Failed cm-handle registration based on cm handle id and registration error'() { - when: 'the failure response is created with "alternate id already associated" error code for 1 cm handle' + when: 'the failure response is created with "cm-handle already exists" error code for 1 cm handle' def cmHandleRegistrationResponses = - CmHandleRegistrationResponse.createFailureResponses(['ch 1'], ALTERNATE_ID_ALREADY_ASSOCIATED) + CmHandleRegistrationResponse.createFailureResponses(['ch 1'], CM_HANDLE_ALREADY_EXIST) then: 'the response with expected values' assert cmHandleRegistrationResponses[0].cmHandle == 'ch 1' assert cmHandleRegistrationResponses[0].status == Status.FAILURE - assert cmHandleRegistrationResponses[0].ncmpResponseStatus == ALTERNATE_ID_ALREADY_ASSOCIATED - assert cmHandleRegistrationResponses[0].errorText == 'alternate id already associated' + assert cmHandleRegistrationResponses[0].ncmpResponseStatus == CM_HANDLE_ALREADY_EXIST + assert cmHandleRegistrationResponses[0].errorText == 'cm-handle already exists' } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy index 4d42e62025..8b04568239 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy @@ -24,7 +24,7 @@ package org.onap.cps.ncmp.api.inventory.models import org.onap.cps.ncmp.api.inventory.DataStoreSyncState import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import spock.lang.Specification import java.time.OffsetDateTime diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/DmiHttpClientConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/DmiHttpClientConfigSpec.groovy index 23f5edd890..387252e163 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/DmiHttpClientConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/DmiHttpClientConfigSpec.groovy @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation. + * Copyright (C) 2023-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,7 @@ class DmiHttpClientConfigSpec extends Specification { assert connectionTimeoutInSeconds == 4 assert readTimeoutInSeconds == 5 assert writeTimeoutInSeconds == 6 + assert responseTimeoutInSeconds == 60 } } @@ -56,6 +57,7 @@ class DmiHttpClientConfigSpec extends Specification { assert connectionTimeoutInSeconds == 14 assert readTimeoutInSeconds == 15 assert writeTimeoutInSeconds == 16 + assert responseTimeoutInSeconds == 60 } } @@ -68,6 +70,7 @@ class DmiHttpClientConfigSpec extends Specification { assert connectionTimeoutInSeconds == 24 assert readTimeoutInSeconds == 25 assert writeTimeoutInSeconds == 26 + assert responseTimeoutInSeconds == 60 } } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/PolicyExecutorHttpClientConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/PolicyExecutorHttpClientConfigSpec.groovy index b988f9e171..3df910322c 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/PolicyExecutorHttpClientConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/config/PolicyExecutorHttpClientConfigSpec.groovy @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation. + * Copyright (C) 2024-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,7 @@ class PolicyExecutorHttpClientConfigSpec extends Specification { assert pendingAcquireMaxCount == 33 assert connectionTimeoutInSeconds == 34 assert writeTimeoutInSeconds == 36 + assert responseTimeoutInSeconds == 60 } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/AdminCacheConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/AdminCacheConfigSpec.groovy index 9b9603369d..a576865262 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/AdminCacheConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/AdminCacheConfigSpec.groovy @@ -45,13 +45,5 @@ class AdminCacheConfigSpec extends Specification { assert Hazelcast.allHazelcastInstances.size() > 0 and: 'Hazelcast cache instance for cm handle by state is present' assert Hazelcast.getHazelcastInstanceByName('cps-and-ncmp-hazelcast-instance-test-config').getMap('cmHandlesByState') != null - and: 'the cache already contains 5 entries, an entry for each state' - def cmHandleByState = Hazelcast.getHazelcastInstanceByName('cps-and-ncmp-hazelcast-instance-test-config').getMap('cmHandlesByState') - assert cmHandleByState.size() == 5 - assert cmHandleByState.containsKey('advisedCmHandlesCount') - assert cmHandleByState.containsKey('lockedCmHandlesCount') - assert cmHandleByState.containsKey('readyCmHandlesCount') - assert cmHandleByState.containsKey('deletingCmHandlesCount') - assert cmHandleByState.containsKey('deletedCmHandlesCount') } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy index c08ff75a44..0026d7c4e6 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,7 @@ package org.onap.cps.ncmp.impl.cache -import com.hazelcast.config.Config -import com.hazelcast.config.RestEndpointGroup + import com.hazelcast.core.Hazelcast import spock.lang.Specification @@ -60,17 +59,4 @@ class HazelcastCacheConfigSpec extends Specification { 'Set Config' | HazelcastCacheConfig.createSetConfig('my set config') || false | false | true } - def 'Verify Hazelcast Cluster Information'() { - given: 'a test configuration' - def testConfig = new Config() - when: 'cluster information is exposed' - objectUnderTest.exposeClusterInformation(testConfig) - then: 'REST api configs are enabled' - assert testConfig.networkConfig.restApiConfig.enabled - and: 'only health check and cluster read are enabled' - def enabledGroups = testConfig.networkConfig.restApiConfig.enabledGroups - assert enabledGroups.size() == 2 - assert enabledGroups.containsAll([RestEndpointGroup.CLUSTER_READ, RestEndpointGroup.HEALTH_CHECK]) - } - } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandlerSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandlerSpec.groovy index 041fbd95ee..93362f23be 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandlerSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/datajobs/DmiSubJobRequestHandlerSpec.groovy @@ -29,12 +29,13 @@ class DmiSubJobRequestHandlerSpec extends Specification { def dmiWriteOperationsPerProducerKey = [new ProducerKey('dmi1', 'prod1'): [dmiWriteOperation]] def authorization = 'my authorization header' and: 'the dmi rest client will return a response (for the correct parameters)' - def responseEntity = new ResponseEntity<>(new SubJobWriteResponse('my-sub-job-id', 'dmi1', 'prod1'), HttpStatus.OK) + def responseAsKeyValuePairs = [subJobId:'my-sub-job-id'] + def responseEntity = new ResponseEntity<>(responseAsKeyValuePairs, HttpStatus.OK) def expectedJson = '{"destination":"d1","dataAcceptType":"t1","dataContentType":"t2","dataProducerId":"prod1","dataJobId":"some-job-id","data":[{"path":"p","op":"operation","moduleSetTag":"tag","value":null,"operationId":"o1","privateProperties":{}}]}' mockDmiRestClient.synchronousPostOperationWithJsonData(RequiredDmiService.DATA, _, expectedJson, OperationType.CREATE, authorization) >> responseEntity when: 'sending request to DMI invoked' objectUnderTest.sendRequestsToDmi(authorization, dataJobId, dataJobMetadata, dmiWriteOperationsPerProducerKey) then: 'the result contains the expected sub-job id' - assert responseEntity.body.subJobId == 'my-sub-job-id' + assert responseEntity.body.get('subJobId') == 'my-sub-job-id' } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy index 811e4ea526..1cbdc7beca 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy @@ -26,7 +26,7 @@ import com.hazelcast.core.Hazelcast import com.hazelcast.instance.impl.HazelcastInstanceFactory import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsQueryService -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.ncmp.api.inventory.DataStoreSyncState import org.onap.cps.ncmp.api.inventory.models.TrustLevel import org.onap.cps.ncmp.api.inventory.models.CmHandleState diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy index b600d02be5..70bd418026 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy @@ -32,7 +32,7 @@ import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle import org.onap.cps.api.exceptions.DataNodeNotFoundException import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import org.slf4j.LoggerFactory diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy index ff190cc1ca..953e1c7d0e 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,10 @@ package org.onap.cps.ncmp.impl.inventory import com.hazelcast.map.IMap import org.onap.cps.api.CpsDataService -import org.onap.cps.api.CpsModuleService +import org.onap.cps.api.exceptions.AlreadyDefinedException +import org.onap.cps.api.exceptions.CpsException +import org.onap.cps.api.exceptions.DataNodeNotFoundException +import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.ncmp.api.exceptions.DmiRequestException import org.onap.cps.ncmp.api.inventory.DataStoreSyncState import org.onap.cps.ncmp.api.inventory.models.CmHandleRegistrationResponse @@ -36,11 +39,6 @@ import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle import org.onap.cps.ncmp.impl.inventory.sync.lcm.LcmEventsCmHandleStateHandler import org.onap.cps.ncmp.impl.inventory.trustlevel.TrustLevelManager -import org.onap.cps.api.exceptions.AlreadyDefinedException -import org.onap.cps.api.exceptions.CpsException -import org.onap.cps.api.exceptions.DataNodeNotFoundException -import org.onap.cps.api.exceptions.DataValidationException -import org.onap.cps.api.exceptions.SchemaSetNotFoundException import spock.lang.Specification import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_FOUND @@ -53,7 +51,6 @@ import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NFP_OPERATIONAL_D class CmHandleRegistrationServiceSpec extends Specification { def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: 'some-cm-handle-id') - def mockCpsModuleService = Mock(CpsModuleService) def mockNetworkCmProxyDataServicePropertyHandler = Mock(CmHandleRegistrationServicePropertyHandler) def mockInventoryPersistence = Mock(InventoryPersistence) def mockCmHandleQueries = Mock(CmHandleQueryService) @@ -80,33 +77,43 @@ class CmHandleRegistrationServiceSpec extends Specification { def dmiRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server') dmiRegistration.setCreatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-1', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])]) dmiRegistration.setUpdatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-2', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])]) - dmiRegistration.setRemovedCmHandles(['cmhandle-2']) - dmiRegistration.setUpgradedCmHandles(new UpgradedCmHandles(cmHandles: ['cmhandle-3'], moduleSetTag: 'some-module-set-tag')) - and: 'cm handles are persisted' + dmiRegistration.setRemovedCmHandles(['cmhandle-3']) + dmiRegistration.setUpgradedCmHandles(new UpgradedCmHandles(cmHandles: ['cmhandle-4', 'cmhandle-5'], moduleSetTag: moduleSetTagForUpgrade)) + and: 'cm handles 2,3 and 4 already exist in the inventory' mockInventoryPersistence.getYangModelCmHandles(['cmhandle-2']) >> [new YangModelCmHandle()] - mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> new YangModelCmHandle(id: 'cmhandle-3', moduleSetTag: '', compositeState: new CompositeState(cmHandleState: CmHandleState.READY)) - and: 'cm handle is in READY state' - mockCmHandleQueries.cmHandleHasState('cmhandle-3', CmHandleState.READY) >> true - and: 'cm handles is present in in-progress map' - mockModuleSyncStartedOnCmHandles.containsKey('cmhandle-2') >> true + mockInventoryPersistence.getYangModelCmHandles(['cmhandle-3']) >> [new YangModelCmHandle()] + mockInventoryPersistence.getYangModelCmHandle('cmhandle-4') >> new YangModelCmHandle(id: 'cmhandle-4', moduleSetTag: '', compositeState: new CompositeState(cmHandleState: CmHandleState.READY)) + and: 'cm handle 5 also exist but already has the new module set tag (upgrade to)' + mockInventoryPersistence.getYangModelCmHandle('cmhandle-5') >> new YangModelCmHandle(id: 'cmhandle-5', moduleSetTag: moduleSetTagForUpgrade , compositeState: new CompositeState(cmHandleState: CmHandleState.READY)) + and: 'all cm handles are in READY state' + mockCmHandleQueries.cmHandleHasState(_, CmHandleState.READY) >> true + and: 'cm handle to be removed is in progress map' + mockModuleSyncStartedOnCmHandles.containsKey('cmhandle-3') >> true when: 'registration is processed' - objectUnderTest.updateDmiRegistration(dmiRegistration) + def result = objectUnderTest.updateDmiRegistration(dmiRegistration) then: 'cm-handles are removed first' 1 * objectUnderTest.processRemovedCmHandles(*_) and: 'de-registered cm handle entry is removed from in progress map' - 1 * mockModuleSyncStartedOnCmHandles.removeAsync('cmhandle-2') - then: 'cm-handles are updated' + 1 * mockModuleSyncStartedOnCmHandles.removeAsync('cmhandle-3') + then: 'updated cm handles are processed by the property handler service' 1 * objectUnderTest.processUpdatedCmHandles(*_) - 1 * mockNetworkCmProxyDataServicePropertyHandler.updateCmHandleProperties(*_) >> [] + 1 * mockNetworkCmProxyDataServicePropertyHandler.updateCmHandleProperties(*_) >> [CmHandleRegistrationResponse.createSuccessResponse('cmhandle-2')] then: 'cm-handles are upgraded' 1 * objectUnderTest.processUpgradedCmHandles(*_) + and: 'result contains the correct cm handles for each operation' + assert result.createdCmHandles.cmHandle == ['cmhandle-1'] + assert result.updatedCmHandles.cmHandle == ['cmhandle-2'] + assert result.removedCmHandles.cmHandle == ['cmhandle-3'] + assert result.upgradedCmHandles.cmHandle as Set == ['cmhandle-4', 'cmhandle-5'] as Set + where: 'upgrade with and without module set tag' + moduleSetTagForUpgrade << ['some tag', ''] } def 'DMI Registration upgrade operation with upgrade node state #scenario'() { given: 'a registration with upgrade operation' def dmiRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server') dmiRegistration.setUpgradedCmHandles(new UpgradedCmHandles(cmHandles: ['cmhandle-3'], moduleSetTag: 'some-module-set-tag')) - and: 'exception while checking cm handle state' + and: 'cm handle has the state #cmHandleState' mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> new YangModelCmHandle(id: 'cmhandle-3', moduleSetTag: '', compositeState: new CompositeState(cmHandleState: cmHandleState)) when: 'registration is processed' def result = objectUnderTest.updateDmiRegistration(dmiRegistration) @@ -134,6 +141,21 @@ class CmHandleRegistrationServiceSpec extends Specification { 'cm handle is invalid' | new DataValidationException('some error message', 'some error details') || '110' } + def 'DMI Registration upgrade with exception while updating CM-handle state'() { + given: 'a registration with upgrade operation' + def dmiRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server') + dmiRegistration.setUpgradedCmHandles(new UpgradedCmHandles(cmHandles: ['cmhandle-3'], moduleSetTag: 'some-module-set-tag')) + and: 'cm handle has the state READY' + mockInventoryPersistence.getYangModelCmHandle('cmhandle-3') >> new YangModelCmHandle(id: 'cmhandle-3', moduleSetTag: '', compositeState: new CompositeState(cmHandleState: CmHandleState.READY)) + and: 'exception will occur while updating cm handle state to LOCKED for upgrade' + mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { throw new RuntimeException() } + when: 'registration is processed' + def result = objectUnderTest.updateDmiRegistration(dmiRegistration) + then: 'upgrade operation contains expected error code' + assert result.upgradedCmHandles[0].status == Status.FAILURE + assert result.upgradedCmHandles[0].ncmpResponseStatus == UNKNOWN_ERROR + } + def 'Create CM-handle Validation: Registration with valid Service names: #scenario'() { given: 'a registration ' def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: dmiPlugin, dmiModelPlugin: dmiModelPlugin, @@ -276,18 +298,15 @@ class CmHandleRegistrationServiceSpec extends Specification { assert response.updatedCmHandles.containsAll(updateOperationResponse) } - def 'Remove CmHandle Successfully: #scenario'() { - given: 'a registration' + def 'Remove CmHandle Successfully'() { + given: 'a registration update to delete a cm handle' def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) - and: '#scenario' - mockCpsModuleService.deleteSchemaSetsWithCascade(_, ['cmhandle']) >> { if (!schemaSetExist) { throw new SchemaSetNotFoundException('', '') } } - when: 'registration is updated to delete cmhandle' + when: 'the registration is updated' def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration) - then: 'the cmHandle state is updated to "DELETING"' - 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> - { args -> args[0].values()[0] == CmHandleState.DELETING } - then: 'method to delete relevant schema set is called once' - 1 * mockInventoryPersistence.deleteSchemaSetsWithCascade(_) + then: 'the cmHandle state is set to "DELETING"' + 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args -> args[0].values()[0] == CmHandleState.DELETING } + then: 'method to delete anchors is called once' + 1 * mockInventoryPersistence.deleteAnchors(_) and: 'method to delete relevant list/list element is called once' 1 * mockInventoryPersistence.deleteDataNodes(_) and: 'successful response is received' @@ -297,14 +316,7 @@ class CmHandleRegistrationServiceSpec extends Specification { assert it.cmHandle == 'cmhandle' } and: 'the cmHandle state is updated to "DELETED"' - 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> - { args -> args[0].values()[0] == CmHandleState.DELETED } - and: 'No cm handles state updates for "upgraded cm handles"' - 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch([:]) - where: - scenario | schemaSetExist - 'schema-set exists and can be deleted successfully' | true - 'schema-set does not exist' | false + 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args -> args[0].values()[0] == CmHandleState.DELETED } } def 'Remove CmHandle: Partial Success'() { @@ -314,10 +326,11 @@ class CmHandleRegistrationServiceSpec extends Specification { and: 'cm handles to be deleted in the progress map' mockModuleSyncStartedOnCmHandles.containsKey("cmhandle1") >> true mockModuleSyncStartedOnCmHandles.containsKey("cmhandle3") >> true - and: 'cm-handle deletion fails on batch' - mockInventoryPersistence.deleteDataNodes(_) >> { throw new RuntimeException("Failed") } - and: 'cm-handle deletion is successful for 1st and 3rd; failed for 2nd' - mockInventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='cmhandle2']") >> { throw new RuntimeException("Failed") } + and: 'delete fails for batch. Retry only fails for and cm handle 2' + mockInventoryPersistence.deleteDataNodes(_) >> { throw new RuntimeException("Batch Failed") } + >> { /* cm handle 1 is OK */ } + >> { throw new RuntimeException("Cm handle 2 Failed")} + >> { /* cm handle 3 is OK */ } when: 'registration is updated to delete cmhandles' def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration) then: 'the cmHandle states are all updated to "DELETING"' @@ -343,7 +356,7 @@ class CmHandleRegistrationServiceSpec extends Specification { with(response.removedCmHandles[1]) { assert it.status == Status.FAILURE assert it.ncmpResponseStatus == UNKNOWN_ERROR - assert it.errorText == 'Failed' + assert it.errorText == 'Cm handle 2 Failed' assert it.cmHandle == 'cmhandle2' } and: 'the cmHandle state is updated to DELETED for 1st and 3rd' @@ -351,40 +364,11 @@ class CmHandleRegistrationServiceSpec extends Specification { assert it.size() == 2 assert it.every { entry -> entry.value == CmHandleState.DELETED } }) - and: 'No cm handles state updates for "upgraded cm handles"' - 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch([:]) - } - - def 'Remove CmHandle Error Handling: Schema Set Deletion failed'() { - given: 'a registration' - def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', - removedCmHandles: ['cmhandle']) - and: 'schema set batch deletion failed with unknown error' - mockInventoryPersistence.deleteSchemaSetsWithCascade(_) >> { throw new RuntimeException('Failed') } - and: 'schema set single deletion failed with unknown error' - mockInventoryPersistence.deleteSchemaSetWithCascade(_) >> { throw new RuntimeException('Failed') } - when: 'registration is updated to delete cmhandle' - def response = objectUnderTest.updateDmiRegistration(dmiPluginRegistration) - then: 'no exception is thrown' - noExceptionThrown() - and: 'cm-handle is not deleted' - 0 * mockInventoryPersistence.deleteDataNodes(_) - and: 'the cmHandle state is not updated to "DELETED"' - 0 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch([yangModelCmHandle: CmHandleState.DELETED]) - and: 'a failure response is received' - assert response.removedCmHandles.size() == 1 - with(response.removedCmHandles[0]) { - assert it.status == Status.FAILURE - assert it.cmHandle == 'cmhandle' - assert it.errorText == 'Failed' - assert it.ncmpResponseStatus == UNKNOWN_ERROR - } } def 'Remove CmHandle Error Handling: #scenario'() { given: 'a registration' - def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', - removedCmHandles: ['cmhandle']) + def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: 'cm-handle deletion fails on batch' mockInventoryPersistence.deleteDataNodes(_) >> { throw deleteListElementException } and: 'cm-handle deletion fails on individual delete' @@ -408,7 +392,7 @@ class CmHandleRegistrationServiceSpec extends Specification { 'an unexpected exception' | new RuntimeException('Failed') || UNKNOWN_ERROR | 'Failed' } - def 'Set Cm Handle Data Sync Enabled Flag where data sync flag is #scenario'() { + def 'Set Cm Handle Data Sync Enabled Flag where data sync flag is #scenario'() { given: 'an existing cm handle composite state' def compositeState = new CompositeState(cmHandleState: CmHandleState.READY, dataSyncEnabled: initialDataSyncEnabledFlag, dataStores: CompositeState.DataStores.builder() @@ -429,7 +413,7 @@ class CmHandleRegistrationServiceSpec extends Specification { saveCmHandleStateExpectedNumberOfInvocations * mockInventoryPersistence.saveCmHandleState('some-cm-handle-id', compositeState) where: 'the following data sync enabled flag is used' scenario | dataSyncEnabledFlag | initialDataSyncEnabledFlag | initialDataSyncState || expectedDataStoreSyncState | deleteDataNodeExpectedNumberOfInvocation | saveCmHandleStateExpectedNumberOfInvocations - 'enabled' | true | false | DataStoreSyncState.NONE_REQUESTED || DataStoreSyncState.UNSYNCHRONIZED | 0 | 1 + 'enabled' | true | false | DataStoreSyncState.NONE_REQUESTED || DataStoreSyncState.UNSYNCHRONIZED | 0 | 1 'disabled' | false | true | DataStoreSyncState.UNSYNCHRONIZED || DataStoreSyncState.NONE_REQUESTED | 0 | 1 'disabled where sync-state is currently SYNCHRONIZED' | false | true | DataStoreSyncState.SYNCHRONIZED || DataStoreSyncState.NONE_REQUESTED | 1 | 1 'is set to existing flag state' | true | true | DataStoreSyncState.UNSYNCHRONIZED || DataStoreSyncState.UNSYNCHRONIZED | 0 | 0 @@ -448,6 +432,4 @@ class CmHandleRegistrationServiceSpec extends Specification { 0 * mockInventoryPersistence.saveCmHandleState(_, _) } - - } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy index 619da70bf2..0ed9dd8aae 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ @@ -23,38 +23,36 @@ package org.onap.cps.ncmp.impl.inventory import com.fasterxml.jackson.databind.ObjectMapper +import java.time.OffsetDateTime +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsModuleService import org.onap.cps.api.exceptions.DataNodeNotFoundException import org.onap.cps.api.exceptions.DataValidationException -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.api.model.DataNode +import org.onap.cps.api.model.ModuleDefinition +import org.onap.cps.api.model.ModuleReference +import org.onap.cps.api.parameters.FetchDescendantsOption +import org.onap.cps.utils.CpsValidator import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException import org.onap.cps.ncmp.api.inventory.models.CompositeState import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle import org.onap.cps.ncmp.impl.utils.YangDataConverter -import org.onap.cps.api.parameters.CascadeDeleteAllowed -import org.onap.cps.api.parameters.FetchDescendantsOption -import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.ModuleDefinition -import org.onap.cps.api.model.ModuleReference import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import spock.lang.Shared import spock.lang.Specification -import java.time.OffsetDateTime -import java.time.ZoneOffset -import java.time.format.DateTimeFormatter - +import static org.onap.cps.api.parameters.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS +import static org.onap.cps.api.parameters.FetchDescendantsOption.OMIT_DESCENDANTS import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DATASPACE_NAME import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_ANCHOR import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NO_TIMESTAMP -import static org.onap.cps.api.parameters.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS -import static org.onap.cps.api.parameters.FetchDescendantsOption.OMIT_DESCENDANTS class InventoryPersistenceImplSpec extends Specification { @@ -72,8 +70,7 @@ class InventoryPersistenceImplSpec extends Specification { def mockYangDataConverter = Mock(YangDataConverter) - def objectUnderTest = new InventoryPersistenceImpl(spiedJsonObjectMapper, mockCpsDataService, mockCpsModuleService, - mockCpsValidator, mockCpsAnchorService, mockCmHandleQueries) + def objectUnderTest = new InventoryPersistenceImpl(mockCpsValidator, spiedJsonObjectMapper, mockCpsAnchorService, mockCpsModuleService, mockCpsDataService, mockCmHandleQueries) def formattedDateAndTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ") .format(OffsetDateTime.of(2022, 12, 31, 20, 30, 40, 1, ZoneOffset.UTC)) @@ -294,24 +291,6 @@ class InventoryPersistenceImplSpec extends Specification { 1 * mockCpsDataService.deleteListOrListElement(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR,'sample xPath',null) } - def 'Delete schema set with a valid schema set name'() { - when: 'the method to delete schema set is called with valid schema set name' - objectUnderTest.deleteSchemaSetWithCascade('validSchemaSetName') - then: 'the module service to delete schemaSet is invoked once' - 1 * mockCpsModuleService.deleteSchemaSet(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'validSchemaSetName', CascadeDeleteAllowed.CASCADE_DELETE_ALLOWED) - and: 'the schema set name is validated' - 1 * mockCpsValidator.validateNameCharacters('validSchemaSetName') - } - - def 'Delete multiple schema sets with valid schema set names'() { - when: 'the method to delete schema sets is called with valid schema set names' - objectUnderTest.deleteSchemaSetsWithCascade(['validSchemaSetName1', 'validSchemaSetName2']) - then: 'the module service to delete schema sets is invoked once' - 1 * mockCpsModuleService.deleteSchemaSetsWithCascade(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, ['validSchemaSetName1', 'validSchemaSetName2']) - and: 'the schema set names are validated' - 1 * mockCpsValidator.validateNameCharacters(['validSchemaSetName1', 'validSchemaSetName2']) - } - def 'Get data node via xPath'() { when: 'the method to get data nodes is called' objectUnderTest.getDataNode('sample xPath') diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy index 714555958a..302e43f170 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,14 +21,11 @@ package org.onap.cps.ncmp.impl.inventory.sync -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.ObjectMapper import org.onap.cps.ncmp.impl.dmi.DmiOperationsBaseSpec import org.onap.cps.ncmp.impl.dmi.DmiProperties import org.onap.cps.ncmp.impl.utils.http.UrlTemplateParameters import org.onap.cps.api.model.ModuleReference import org.onap.cps.utils.JsonObjectMapper -import org.spockframework.spring.SpringBean import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus @@ -40,9 +37,12 @@ import static org.onap.cps.ncmp.api.data.models.OperationType.READ import static org.onap.cps.ncmp.impl.models.RequiredDmiService.MODEL @SpringBootTest -@ContextConfiguration(classes = [DmiProperties, DmiModelOperations]) +@ContextConfiguration(classes = [DmiProperties, DmiModelOperations, JsonObjectMapper]) class DmiModelOperationsSpec extends DmiOperationsBaseSpec { + def NO_AUTH_HEADER = null + def NO_MODULE_SET_TAG = '' + def expectedModulesUrlTemplateWithVariables = new UrlTemplateParameters('myServiceName/dmi/v1/ch/{cmHandleId}/modules', ['cmHandleId': cmHandleId]) def expectedModuleResourcesUrlTemplateWithVariables = new UrlTemplateParameters('myServiceName/dmi/v1/ch/{cmHandleId}/moduleResources', ['cmHandleId': cmHandleId]) @@ -52,11 +52,6 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { @Autowired DmiModelOperations objectUnderTest - @SpringBean - JsonObjectMapper spiedJsonObjectMapper = Spy(new JsonObjectMapper(new ObjectMapper())) - - def NO_AUTH_HEADER = null - def 'Retrieving module references.'() { given: 'a cm handle' mockYangModelCmHandleRetrieval([]) @@ -65,7 +60,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity([schemas: moduleReferencesAsLisOfMaps], HttpStatus.OK) mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModulesUrlTemplateWithVariables, '{"cmHandleProperties":{},"moduleSetTag":""}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get module references is called' - def result = objectUnderTest.getModuleReferences(yangModelCmHandle) + def result = objectUnderTest.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) then: 'the result consists of expected module references' assert result == [new ModuleReference(moduleName: 'mod1', revision: 'A'), new ModuleReference(moduleName: 'mod2', revision: 'X')] } @@ -78,7 +73,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity(bodyAsMap, HttpStatus.NO_CONTENT) mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> responseFromDmi when: 'get module references is called' - def result = objectUnderTest.getModuleReferences(yangModelCmHandle) + def result = objectUnderTest.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) then: 'the result is empty' assert result == [] where: 'the DMI response body has the following content' @@ -97,7 +92,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModulesUrlTemplateWithVariables, '{"cmHandleProperties":' + expectedAdditionalPropertiesInRequest + ',"moduleSetTag":""}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'a get module references is called' - def result = objectUnderTest.getModuleReferences(yangModelCmHandle) + def result = objectUnderTest.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) then: 'the result is the response from DMI service' assert result == [] where: 'the following DMI properties are used' @@ -116,7 +111,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModuleResourcesUrlTemplateWithVariables, '{"data":{"modules":[' + expectedModuleReferencesInRequest + ']},"cmHandleProperties":{}}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, newModuleReferences) then: 'the result has the 2 expected yang (re)sources (order is not guaranteed)' assert result.size() == 2 assert result.get('mod1') == 'some yang source' @@ -131,7 +126,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity(responseFromDmiBody, HttpStatus.NO_CONTENT) mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, newModuleReferences) then: 'the result is empty' assert result == [:] where: 'the DMI response body has the following content' @@ -149,7 +144,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { '{"data":{"modules":[{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}]},"cmHandleProperties":' + expectedAdditionalPropertiesInRequest + '}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, newModuleReferences) then: 'the result is the response from DMI service' assert result == [mod1:'some yang source'] where: 'the following DMI properties are used' @@ -166,7 +161,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModuleResourcesUrlTemplateWithVariables, '{' + expectedModuleSetTagInRequest + '"data":{"modules":[{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}]},"cmHandleProperties":{}}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, moduleSetTag, newModuleReferences) then: 'the result is the response from DMI service' assert result == [mod1:'some yang source'] where: 'the following Module Set Tags are used' @@ -180,7 +175,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { given: 'a cm handle' mockYangModelCmHandleRetrieval([]) when: 'a get new yang resources from DMI is called with no module references' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, []) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, []) then: 'no resources are returned' assert result == [:] and: 'no request is sent to DMI' @@ -191,21 +186,35 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { given: 'a cm handle' mockYangModelCmHandleRetrieval(null) when: 'a get new yang resources from DMI is called' - objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, [new ModuleReference('mod1', 'A')]) + objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, [new ModuleReference('mod1', 'A')]) then: 'a null pointer is thrown (we might need to address this later)' thrown(NullPointerException) } - def 'Retrieving module references with Json processing exception.'() { - given: 'a cm handle' - mockYangModelCmHandleRetrieval([]) - and: 'a Json processing exception occurs' - spiedJsonObjectMapper.asJsonString(_) >> {throw (new JsonProcessingException('parsing error'))} - when: 'a DMI operation is executed' - objectUnderTest.getModuleReferences(yangModelCmHandle) - then: 'an ncmp exception is thrown' - def exceptionThrown = thrown(JsonProcessingException) - and: 'the message indicates a parsing error' - exceptionThrown.message.toLowerCase().contains('parsing error') + def 'Retrieving module references forwards the new module set tag to DMI during CM-handle upgrade.'() { + given: 'a cm handle with an existing module set tag' + mockYangModelCmHandleRetrieval([], 'OLD-TAG') + when: 'get module references is called' + objectUnderTest.getModuleReferences(yangModelCmHandle, 'NEW-TAG') + then: 'a request was sent to DMI with the NEW module set tag in the body' + 1 * mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> { args -> + def requestBodyAsJson = args[2] as String + assert requestBodyAsJson.contains('"moduleSetTag":"NEW-TAG"') + return new ResponseEntity([schemas: [[moduleName: 'mod1', revision: 'A'], [moduleName: 'mod2', revision: 'X']]], HttpStatus.OK) + } + } + + def 'Retrieving yang resources forwards the new module set tag to DMI during CM-handle upgrade.'() { + given: 'a cm handle with an existing module set tag' + mockYangModelCmHandleRetrieval([], 'OLD-TAG') + when: 'get new yang resources from DMI service' + objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, 'NEW-TAG', newModuleReferences) + then: 'a request was sent to DMI with the NEW module set tag in the body' + 1 * mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> { args -> + def requestBodyAsJson = args[2] as String + assert requestBodyAsJson.contains('"moduleSetTag":"NEW-TAG"') + return new ResponseEntity([[moduleName: 'mod1', revision: 'A', yangSource: 'some yang source'], + [moduleName: 'mod2', revision: 'X', yangSource: 'other yang source']], HttpStatus.OK) + } } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy index 67cc4edd83..b4837f7bab 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,20 +20,17 @@ package org.onap.cps.ncmp.impl.inventory.sync -import com.hazelcast.collection.ISet import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsModuleService -import org.onap.cps.ncmp.api.exceptions.NcmpException +import org.onap.cps.api.exceptions.AlreadyDefinedException +import org.onap.cps.api.exceptions.DuplicatedYangResourceException +import org.onap.cps.api.model.ModuleReference import org.onap.cps.ncmp.api.inventory.models.CompositeStateBuilder import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle import org.onap.cps.ncmp.impl.inventory.CmHandleQueryService import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle -import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncService.ModuleDelta -import org.onap.cps.api.parameters.CascadeDeleteAllowed -import org.onap.cps.api.exceptions.SchemaSetNotFoundException -import org.onap.cps.api.model.ModuleReference import org.onap.cps.utils.JsonObjectMapper import spock.lang.Specification @@ -42,55 +39,45 @@ import static org.onap.cps.ncmp.api.inventory.models.LockReasonCategory.MODULE_U class ModuleSyncServiceSpec extends Specification { + def NO_MODULE_SET_TAG = '' + def mockCpsModuleService = Mock(CpsModuleService) def mockDmiModelOperations = Mock(DmiModelOperations) def mockCpsAnchorService = Mock(CpsAnchorService) def mockCmHandleQueries = Mock(CmHandleQueryService) def mockCpsDataService = Mock(CpsDataService) def mockJsonObjectMapper = Mock(JsonObjectMapper) - def mockModuleSetTagsBeingProcessed = Mock(ISet<String>); - - def objectUnderTest = new ModuleSyncService(mockDmiModelOperations, mockCpsModuleService, mockCpsDataService, mockCpsAnchorService, mockJsonObjectMapper, mockModuleSetTagsBeingProcessed) - def expectedDataspaceName = NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME - - def setup() { - // Allow tags for al test except 'duplicate-processing-tag' to be added to processing semaphore - mockModuleSetTagsBeingProcessed.add('new-tag') >> true - mockModuleSetTagsBeingProcessed.add('same-tag') >> true - mockModuleSetTagsBeingProcessed.add('cached-tag') >> true - } + def objectUnderTest = new ModuleSyncService(mockDmiModelOperations, mockCpsModuleService, mockCpsDataService, mockCpsAnchorService, mockJsonObjectMapper) def 'Sync models for a NEW cm handle using module set tags: #scenario.'() { given: 'a cm handle to be synced' def yangModelCmHandle = createAdvisedCmHandle(moduleSetTag) and: 'DMI operations returns some module references' def moduleReferences = [ new ModuleReference('module1','1'), new ModuleReference('module2','2') ] - mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences + mockDmiModelOperations.getModuleReferences(yangModelCmHandle, moduleSetTag) >> moduleReferences and: 'DMI-Plugin returns resource(s) for "new" module(s)' - mockDmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, identifiedNewModuleReferences) >> newModuleNameContentToMap + mockDmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, moduleSetTag, identifiedNewModuleReferences) >> newModuleNameContentToMap and: 'the module service identifies #identifiedNewModuleReferences.size() new modules' mockCpsModuleService.identifyNewModuleReferences(moduleReferences) >> identifiedNewModuleReferences - and: 'the service returns a list of module references when queried with the specified attributes' - mockCpsModuleService.getModuleReferencesByAttribute(*_) >> existingModuleReferences when: 'module sync is triggered' objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) then: 'create schema set from module is invoked with correct parameters' - 1 * mockCpsModuleService.createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'ch-1', newModuleNameContentToMap, moduleReferences) + 1 * mockCpsModuleService.createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, expectedSchemaSetName, newModuleNameContentToMap, moduleReferences) and: 'anchor is created with the correct parameters' - 1 * mockCpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'ch-1', 'ch-1') + 1 * mockCpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, expectedSchemaSetName, 'ch-1') where: 'the following parameters are used' - scenario | identifiedNewModuleReferences | newModuleNameContentToMap | moduleSetTag | existingModuleReferences - 'one new module, new tag' | [new ModuleReference('module1', '1')] | [module1: 'some yang source'] | '' | [] - 'no new module, new tag' | [] | [:] | 'new-tag' | [] - 'same tag' | [] | [:] | 'same-tag' | [new ModuleReference('module1', '1'), new ModuleReference('module2', '2')] + scenario | identifiedNewModuleReferences | newModuleNameContentToMap | moduleSetTag | existingModuleReferences || expectedSchemaSetName + 'one new module, new tag' | [new ModuleReference('module1', '1')] | [module1: 'some yang source'] | '' | [] || 'ch-1' + 'no new module, new tag' | [] | [:] | 'new-tag' | [] || 'new-tag' + 'same tag' | [] | [:] | 'same-tag' | [new ModuleReference('module1', '1'), new ModuleReference('module2', '2')] || 'same-tag' } - def 'Attempt Sync models for a cm handle with exception and #scenario module set tag'() { + def 'Attempt Sync models for a cm handle with exception and #scenario module set tag.'() { given: 'a cm handle to be synced' def yangModelCmHandle = createAdvisedCmHandle(moduleSetTag) - and: 'the service returns a list of module references when queried with the specified attributes' - mockCpsModuleService.getModuleReferencesByAttribute(*_) >> [new ModuleReference('module1', '1')] + and: 'dmi returns no new yang resources' + mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] and: 'exception occurs when trying to store result' def testException = new RuntimeException('test') mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw testException } @@ -99,130 +86,100 @@ class ModuleSyncServiceSpec extends Specification { then: 'the same exception is thrown up' def exceptionThrown = thrown(Exception) assert testException == exceptionThrown - and: 'module set tag is removed from processing semaphores only when needed' - expectedCallsToRemoveTag * mockModuleSetTagsBeingProcessed.remove('new-tag') where: 'following module set tags are used' - scenario | moduleSetTag || expectedCallsToRemoveTag - 'with' | 'new-tag' || 1 - 'without' | ' ' || 0 + scenario | moduleSetTag + 'with' | 'new-tag' + 'without' | '' } - def 'Sync models for a cm handle with previously cached module set tag.'() { + def 'Sync models for a cm handle with already defined exception upon schema set creation.'() { given: 'a cm handle to be synced' - def yangModelCmHandle = createAdvisedCmHandle('cached-tag') - and: 'The module set tag exists in the private cache' - def moduleReferences = [ new ModuleReference('module1','1') ] - def cachedModuleDelta = new ModuleDelta(moduleReferences, [:]) - objectUnderTest.privateModuleSetCache.put('cached-tag', cachedModuleDelta) + def yangModelCmHandle = createAdvisedCmHandle('existing tag') + and: 'dmi returns no new yang resources' + mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] + and: 'already defined exception occurs when creating schema (existing)' + mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw AlreadyDefinedException.forSchemaSet('', '', null) } when: 'module sync is triggered' objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) - then: 'create schema set from module is invoked with correct parameters' - 1 * mockCpsModuleService.createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'ch-1', [:], moduleReferences) - and: 'anchor is created with the correct parameters' - 1 * mockCpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'ch-1', 'ch-1') + then: 'the exception is ignored' + noExceptionThrown() + } + + def 'Sync models for a cm handle with already defined exception upon anchor set creation.'() { + given: 'a cm handle to be synced' + def yangModelCmHandle = createAdvisedCmHandle('existing tag') + and: 'dmi returns no new yang resources' + mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] + and: 'already defined exception occurs when creating schema (existing)' + mockCpsAnchorService.createAnchor(*_) >> { throw AlreadyDefinedException.forAnchor('', '', null) } + when: 'module sync is triggered' + objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) + then: 'the exception is ignored' + noExceptionThrown() } - def 'Attempt to sync using a module set tag already being processed by a different instance or thread.'() { + def 'Attempt Sync models for a cm handle with duplicate yang resources exception).'() { given: 'a cm handle to be synced' - def yangModelCmHandle = createAdvisedCmHandle('duplicateTag') - and: 'The module set tag already exists in the processing semaphore set' - mockModuleSetTagsBeingProcessed.add('duplicate-processing-tag') > false + def yangModelCmHandle = createAdvisedCmHandle('existing tag') + and: 'dmi returns no new yang resources' + mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] + and: 'duplicate yang resource exception occurs when creating schema' + def originalException = new DuplicatedYangResourceException('', '', null) + mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw originalException } when: 'module sync is triggered' objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) - then: 'a ncmp exception is thrown with the relevant details' - def exceptionThrown = thrown(NcmpException) - assert exceptionThrown.message.contains('duplicateTag') - assert exceptionThrown.details.contains('duplicateTag') - assert exceptionThrown.details.contains('ch-1') + then: 'same exception is thrown up' + def thrownException = thrown(Exception) + assert thrownException == originalException } - def 'Upgrade model for an existing cm handle with Module Set Tag where the modules are #scenario'() { - given: 'a cm handle being upgraded to module set tag: tag-1' + def 'Model upgrade without using Module Set Tags (legacy) where the modules are in database.'() { + given: 'a cm handle being upgraded without using module set tags' def ncmpServiceCmHandle = new NcmpServiceCmHandle() - ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'Upgrade to ModuleSetTag: tag-1').build()) + ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, '').build()) def dmiServiceName = 'some service name' ncmpServiceCmHandle.cmHandleId = 'upgraded-ch' - def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(dmiServiceName, '', '', ncmpServiceCmHandle,'tag-1', '', '') - and: 'some module references' - def moduleReferences = [ new ModuleReference('module1','1') ] + def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(dmiServiceName, '', '', ncmpServiceCmHandle,'', '', '') and: 'DMI operations returns some module references for upgraded cm handle' - mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences - mockDmiModelOperations.getNewYangResourcesFromDmi(_, []) >> [:] - and: 'none of these module references are new (unknown to the system)' + def moduleReferences = [ new ModuleReference('module1','1') ] + mockDmiModelOperations.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) >> moduleReferences + mockDmiModelOperations.getNewYangResourcesFromDmi(_, NO_MODULE_SET_TAG, []) >> [:] + and: 'none of these module references are new (all already known to the system)' mockCpsModuleService.identifyNewModuleReferences(_) >> [] - and: 'CPS-Core returns list of existing module resources for TBD' - mockCpsModuleService.getYangResourcesModuleReferences(*_) >> [ new ModuleReference('module1','1') ] - and: 'the service returns a list of module references when queried with the specified attributes' - mockCpsModuleService.getModuleReferencesByAttribute(*_) >> existingModuleReferences - and: 'the other cm handle is a state ready' - mockCmHandleQueries.cmHandleHasState('otherId', CmHandleState.READY) >> true when: 'module sync is triggered' objectUnderTest.syncAndUpgradeSchemaSet(yangModelCmHandle) then: 'update schema set from module is invoked for the upgraded cm handle' 1 * mockCpsModuleService.upgradeSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'upgraded-ch', [:], moduleReferences) - and: 'create schema set from module is not invoked for the upgraded cm handle' - 0 * mockCpsModuleService.createSchemaSetFromModules(*_) and: 'No anchor is created for the upgraded cm handle' 0 * mockCpsAnchorService.createAnchor(*_) - where: 'the following parameters are used' - scenario | existingModuleReferences - 'new' | [] - 'in database' | [new ModuleReference('module1', '1')] } - def 'upgrade model for an existing cm handle'() { + def 'Model upgrade using to existing Module Set Tag'() { given: 'a cm handle that is ready but locked for upgrade' def ncmpServiceCmHandle = new NcmpServiceCmHandle() - ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder() - .withLockReason(MODULE_UPGRADE, 'Upgrade to ModuleSetTag: targetModuleSetTag').build()) + ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withLockReason(MODULE_UPGRADE, 'Upgrade to ModuleSetTag: ' + tagTo).build()) ncmpServiceCmHandle.setCmHandleId('cmHandleId-1') - def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle('some service name', '', '', ncmpServiceCmHandle, 'targetModuleSetTag', '', '') + def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle('some service name', '', '', ncmpServiceCmHandle, tagFrom, '', '') mockCmHandleQueries.cmHandleHasState('cmHandleId-1', CmHandleState.READY) >> true - and: 'the module service returns some module references' - def moduleReferences = [new ModuleReference('module1', '1'), new ModuleReference('module2', '2')] - mockCpsModuleService.getYangResourcesModuleReferences(*_)>> moduleReferences - and: 'the service returns a list of module references when queried with the specified attributes' - mockCpsModuleService.getModuleReferencesByAttribute(*_) >> moduleReferences + and: 'the module tag (schemaset) exists is #schemaExists' + mockCpsModuleService.schemaSetExists(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, tagTo) >> schemaExists + and: 'DMI operations returns some module references for upgraded cm handle' + def moduleReferences = [ new ModuleReference('module1','1') ] + expectedCallsToDmi * mockDmiModelOperations.getModuleReferences(yangModelCmHandle, tagTo) >> moduleReferences + and: 'dmi returns no new yang resources' + mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] + and: 'none of these module references are new (all already known to the system)' + expectedCallsToModuleService * mockCpsModuleService.identifyNewModuleReferences(_) >> [] when: 'module upgrade is triggered' objectUnderTest.syncAndUpgradeSchemaSet(yangModelCmHandle) - then: 'the upgrade is delegated to the module service (with the correct parameters)' - 1 * mockCpsModuleService.upgradeSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'cmHandleId-1', Collections.emptyMap(), moduleReferences) - } - - def 'Delete Schema Set for CmHandle'() { - when: 'delete schema set if exists is called' - objectUnderTest.deleteSchemaSetIfExists('some-cmhandle-id') - then: 'the module service is invoked to delete the correct schema set' - 1 * mockCpsModuleService.deleteSchemaSet(expectedDataspaceName, 'some-cmhandle-id', CascadeDeleteAllowed.CASCADE_DELETE_ALLOWED) - } - - def 'Delete a non-existing Schema Set for CmHandle' () { - given: 'the DB throws an exception because its Schema Set does not exist' - mockCpsModuleService.deleteSchemaSet(*_) >> { throw new SchemaSetNotFoundException('some-dataspace-name', 'some-cmhandle-id') } - when: 'delete schema set if exists is called' - objectUnderTest.deleteSchemaSetIfExists('some-cmhandle-id') - then: 'the exception from the DB is ignored; there are no exceptions' - noExceptionThrown() - } - - def 'Delete Schema Set for CmHandle with other exception' () { - given: 'an exception other than SchemaSetNotFoundException is thrown' - UnsupportedOperationException unsupportedOperationException = new UnsupportedOperationException(); - 1 * mockCpsModuleService.deleteSchemaSet(*_) >> { throw unsupportedOperationException } - when: 'delete schema set if exists is called' - objectUnderTest.deleteSchemaSetIfExists('some-cmhandle-id') - then: 'an exception is thrown' - def result = thrown(UnsupportedOperationException) - result == unsupportedOperationException - } - - def 'Clear module set cache.'() { - given: 'something in the module set cache' - objectUnderTest.privateModuleSetCache.put('test',new ModuleDelta([],[:])) - when: 'the cache is cleared' - objectUnderTest.clearPrivateModuleSetCache() - then: 'the cache is empty' - objectUnderTest.privateModuleSetCache.isEmpty() + then: 'the upgrade is delegated to the anchor service (with the correct parameters) except when new tag is blank' + expectedCallsToAnchorService * mockCpsAnchorService.updateAnchorSchemaSet(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'cmHandleId-1', tagTo) + where: 'with or without from tag' + scenario | schemaExists | tagFrom | tagTo || expectedCallsToDmi | expectedCallsToModuleService | expectedCallsToAnchorService + 'from no tag to existing tag' | true | '' | 'tagTo'|| 0 | 0 | 1 + 'from tag to other existing tag' | true | 'oldTag' | 'tagTo'|| 0 | 0 | 1 + 'to new tag' | false | 'oldTag' | 'tagTo'|| 1 | 1 | 1 + 'to NO tag' | true | 'oldTag' | '' || 1 | 1 | 0 } def createAdvisedCmHandle(moduleSetTag) { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy index 556ed0b63c..92f4b38f31 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,15 +29,16 @@ import com.hazelcast.config.Config import com.hazelcast.core.Hazelcast import com.hazelcast.instance.impl.HazelcastInstanceFactory import com.hazelcast.map.IMap +import org.onap.cps.api.exceptions.DataNodeNotFoundException +import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.api.inventory.models.CompositeState import org.onap.cps.ncmp.api.inventory.models.CompositeStateBuilder import org.onap.cps.ncmp.impl.inventory.InventoryPersistence -import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle import org.onap.cps.ncmp.impl.inventory.sync.lcm.LcmEventsCmHandleStateHandler -import org.onap.cps.api.exceptions.DataNodeNotFoundException import org.slf4j.LoggerFactory import spock.lang.Specification + import java.util.concurrent.atomic.AtomicInteger import static org.onap.cps.ncmp.api.inventory.models.LockReasonCategory.MODULE_SYNC_FAILED @@ -87,10 +88,7 @@ class ModuleSyncTasksSpec extends Specification { mockInventoryPersistence.getYangModelCmHandle('cm-handle-2') >> cmHandle2 when: 'module sync poll is executed' objectUnderTest.performModuleSync(['cm-handle-1', 'cm-handle-2'], batchCount) - then: 'module sync service deletes schemas set of each cm handle if it already exists' - 1 * mockModuleSyncService.deleteSchemaSetIfExists('cm-handle-1') - 1 * mockModuleSyncService.deleteSchemaSetIfExists('cm-handle-2') - and: 'module sync service is invoked for each cm handle' + then: 'module sync service is invoked for each cm handle' 1 * mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(_) >> { args -> assert args[0].id == 'cm-handle-1' } 1 * mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(_) >> { args -> assert args[0].id == 'cm-handle-2' } and: 'the state handler is called for the both cm handles' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy index 3213e5d442..7db9e5a870 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================== - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ package org.onap.cps.ncmp.impl.inventory.sync -import com.hazelcast.collection.ISet import com.hazelcast.config.Config import com.hazelcast.core.Hazelcast import com.hazelcast.map.IMap @@ -45,9 +44,6 @@ class SynchronizationCacheConfigSpec extends Specification { @Autowired IMap<String, Boolean> dataSyncSemaphores - @Autowired - ISet<String> moduleSetTagsBeingProcessed - def cleanupSpec() { Hazelcast.getHazelcastInstanceByName('cps-and-ncmp-hazelcast-instance-test-config').shutdown() } @@ -59,8 +55,6 @@ class SynchronizationCacheConfigSpec extends Specification { assert null != moduleSyncStartedOnCmHandles and: 'system is able to create an instance of a map to hold data sync semaphores' assert null != dataSyncSemaphores - and: 'system is able to create an instance of a set to hold module set tags being processed' - assert null != moduleSetTagsBeingProcessed and: 'there is only one instance with the correct name' assert Hazelcast.allHazelcastInstances.size() == 1 assert Hazelcast.allHazelcastInstances.name[0] == 'cps-and-ncmp-hazelcast-instance-test-config' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitorSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitorSpec.groovy index 9fd40b9605..4d7c22a200 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitorSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/lcm/CmHandleStateMonitorSpec.groovy @@ -26,16 +26,19 @@ import static org.onap.cps.ncmp.api.inventory.models.CmHandleState.READY import com.hazelcast.core.Hazelcast import com.hazelcast.map.IMap import org.onap.cps.ncmp.api.inventory.models.CompositeState +import org.onap.cps.ncmp.impl.inventory.CmHandleQueryService import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle import org.onap.cps.ncmp.impl.inventory.sync.lcm.CmHandleStateMonitor.DecreasingEntryProcessor import org.onap.cps.ncmp.impl.inventory.sync.lcm.CmHandleStateMonitor.IncreasingEntryProcessor +import org.onap.cps.ncmp.utils.events.NcmpInventoryModelOnboardingFinishedEvent import spock.lang.Shared import spock.lang.Specification; class CmHandleStateMonitorSpec extends Specification { - def cmHandlesByState = Mock(IMap) - def objectUnderTest = new CmHandleStateMonitor(cmHandlesByState) + def mockCmHandlesByState = Mock(IMap) + def mockCmHandleQueryService = Mock(CmHandleQueryService) + def objectUnderTest = new CmHandleStateMonitor(mockCmHandleQueryService, mockCmHandlesByState) @Shared def entryProcessingMap = Hazelcast.newHazelcastInstance().getMap('entryProcessingMap') @@ -49,6 +52,25 @@ class CmHandleStateMonitorSpec extends Specification { Hazelcast.shutdownAll() } + def 'Initialise cm handle state monitor: #scenario'() { + given: 'the query service returns a list of cm-handle ids for the given state' + mockCmHandleQueryService.queryCmHandleIdsByState(_) >> queryResult + and: 'a mocked NcmpModelOnboardingFinishedEvent is triggered' + def mockNcmpModelOnboardingFinishedEvent = Mock(NcmpInventoryModelOnboardingFinishedEvent) + when: 'the method to initialise cm handle state monitor is triggered by onboarding event' + objectUnderTest.initialiseCmHandleStateMonitor(mockNcmpModelOnboardingFinishedEvent) + then: 'metrics map is called correct number of times for each state except DELETED, with expected value' + 1 * mockCmHandlesByState.putIfAbsent("advisedCmHandlesCount", expectedValue) + 1 * mockCmHandlesByState.putIfAbsent("readyCmHandlesCount", expectedValue) + 1 * mockCmHandlesByState.putIfAbsent("lockedCmHandlesCount", expectedValue) + 1 * mockCmHandlesByState.putIfAbsent("deletingCmHandlesCount", expectedValue) + where: + scenario | queryResult || expectedValue + 'query service returns zero cm handle id'| [] || 0 + 'query service returns 1 cm handle id' | ['someId'] || 1 + } + + def 'Update cm handle state metric'() { given: 'a collection of cm handle state pair' def cmHandleTransitionPair = new LcmEventsCmHandleStateHandlerImpl.CmHandleTransitionPair() @@ -57,19 +79,19 @@ class CmHandleStateMonitorSpec extends Specification { when: 'method to update cm handle state metrics is called' objectUnderTest.updateCmHandleStateMetrics([cmHandleTransitionPair]) then: 'cm handle by state cache map is called once for current and target state for entry processing' - 1 * cmHandlesByState.executeOnKey('advisedCmHandlesCount', _) - 1 * cmHandlesByState.executeOnKey('readyCmHandlesCount', _) + 1 * mockCmHandlesByState.executeOnKey('advisedCmHandlesCount', _) + 1 * mockCmHandlesByState.executeOnKey('readyCmHandlesCount', _) } - def 'Updating cm handle state metric with no previous state'() { + def 'Update cm handle state metric with no previous state'() { given: 'a collection of cm handle state pair wherein current state is null' def cmHandleTransitionPair = new LcmEventsCmHandleStateHandlerImpl.CmHandleTransitionPair() cmHandleTransitionPair.currentYangModelCmHandle = new YangModelCmHandle(compositeState: null) cmHandleTransitionPair.targetYangModelCmHandle = new YangModelCmHandle(compositeState: new CompositeState(cmHandleState: ADVISED)) - when: 'method to update cm handle state metrics is called' + when: 'updating cm handle state metrics' objectUnderTest.updateCmHandleStateMetrics([cmHandleTransitionPair]) then: 'cm handle by state cache map is called only once' - 1 * cmHandlesByState.executeOnKey(_, _) + 1 * mockCmHandlesByState.executeOnKey(_, _) } def 'Applying decreasing entry processor to a key on map where #scenario'() { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy index ffd1d89fe1..dc6ec4120b 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy @@ -30,6 +30,7 @@ import org.onap.cps.api.CpsModuleService import org.onap.cps.api.model.Dataspace import org.slf4j.LoggerFactory import org.springframework.boot.context.event.ApplicationStartedEvent +import org.springframework.context.ApplicationEventPublisher import org.springframework.context.annotation.AnnotationConfigApplicationContext import spock.lang.Specification @@ -42,7 +43,8 @@ class InventoryModelLoaderSpec extends Specification { def mockCpsModuleService = Mock(CpsModuleService) def mockCpsDataService = Mock(CpsDataService) def mockCpsAnchorService = Mock(CpsAnchorService) - def objectUnderTest = new InventoryModelLoader(mockCpsAdminService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService) + def mockApplicationEventPublisher = Mock(ApplicationEventPublisher) + def objectUnderTest = new InventoryModelLoader(mockCpsAdminService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService, mockApplicationEventPublisher) def applicationContext = new AnnotationConfigApplicationContext() @@ -75,6 +77,8 @@ class InventoryModelLoaderSpec extends Specification { 1 * mockCpsAnchorService.updateAnchorSchemaSet(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, 'dmi-registry-2024-02-23') and: 'No schema sets are being removed by the module service (yet)' 0 * mockCpsModuleService.deleteSchemaSet(NCMP_DATASPACE_NAME, _, _) + and: 'application event publisher is called once' + 1 * mockApplicationEventPublisher.publishEvent(_) } } diff --git a/cps-parent/pom.xml b/cps-parent/pom.xml index 366685ad18..88097f219f 100644 --- a/cps-parent/pom.xml +++ b/cps-parent/pom.xml @@ -27,7 +27,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <packaging>pom</packaging> <properties> @@ -55,12 +55,12 @@ <maven.resources.plugin.version>3.3.1</maven.resources.plugin.version> <maven.site.plugin.version>4.0.0-M13</maven.site.plugin.version> <maven.surefire.plugin.version>3.3.1</maven.surefire.plugin.version> - <postgres.version>42.5.1</postgres.version> + <postgres.version>42.5.5</postgres.version> <slf4j.simple.version>2.0.6</slf4j.simple.version> <sonar.version>4.0.0.4121</sonar.version> <spotbugs.plugin.version>4.8.6.4</spotbugs.plugin.version> <spotbugs.version>4.8.6</spotbugs.version> - <spring.boot.maven.plugin.version>3.3.7</spring.boot.maven.plugin.version> + <spring.boot.maven.plugin.version>3.4.1</spring.boot.maven.plugin.version> <swagger.codegen.version>1.2.1</swagger.codegen.version> <!-- Reporting paths and coverage --> diff --git a/cps-path-parser/pom.xml b/cps-path-parser/pom.xml index 469357462e..e30c728053 100644 --- a/cps-path-parser/pom.xml +++ b/cps-path-parser/pom.xml @@ -23,7 +23,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-rest/docs/openapi/cpsAdmin.yml b/cps-rest/docs/openapi/cpsAdmin.yml index f394270dd5..6cfffa48f3 100644 --- a/cps-rest/docs/openapi/cpsAdmin.yml +++ b/cps-rest/docs/openapi/cpsAdmin.yml @@ -1,6 +1,6 @@ # ============LICENSE_START======================================================= # Copyright (c) 2021 Bell Canada. -# Modifications Copyright (C) 2021-2022 Nordix Foundation +# Modifications Copyright (C) 2021-2025 Nordix Foundation # Modifications Copyright (C) 2022 TechMahindra Ltd. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); @@ -230,3 +230,23 @@ adminDataspace: $ref: 'components.yml#/components/responses/Forbidden' '500': $ref: 'components.yml#/components/responses/InternalServerError' + +adminCleanDataspace: + post: + description: Clean the dataspace (remove orphaned schema sets and modules) + tags: + - cps-admin + summary: Clean the dataspace + operationId: cleanDataspace + parameters: + - $ref: 'components.yml#/components/parameters/apiVersionInPath' + - $ref: 'components.yml#/components/parameters/dataspaceNameInPath' + responses: + '204': + $ref: 'components.yml#/components/responses/NoContent' + '400': + $ref: 'components.yml#/components/responses/BadRequest' + '403': + $ref: 'components.yml#/components/responses/Forbidden' + '500': + $ref: 'components.yml#/components/responses/InternalServerError' diff --git a/cps-rest/docs/openapi/openapi.yml b/cps-rest/docs/openapi/openapi.yml index f4eab61875..c85bf7cac7 100644 --- a/cps-rest/docs/openapi/openapi.yml +++ b/cps-rest/docs/openapi/openapi.yml @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (C) 2021-2024 Nordix Foundation +# Copyright (C) 2021-2025 Nordix Foundation # Modifications Copyright (C) 2021 Pantheon.tech # Modifications Copyright (C) 2021 Bell Canada. # Modifications Copyright (C) 2022-2024 TechMahindra Ltd. @@ -61,6 +61,9 @@ paths: /{apiVersion}/admin/dataspaces/{dataspace-name}: $ref: 'cpsAdmin.yml#/adminDataspace' + /{apiVersion}/admin/dataspaces/{dataspace-name}/actions/clean: + $ref: 'cpsAdmin.yml#/adminCleanDataspace' + /v1/dataspaces/{dataspace-name}/anchors: $ref: 'cpsAdminV1Deprecated.yml#/anchorsByDataspace' diff --git a/cps-rest/pom.xml b/cps-rest/pom.xml index b04daf03bd..4e52b1b794 100644 --- a/cps-rest/pom.xml +++ b/cps-rest/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> @@ -62,10 +62,6 @@ <artifactId>spring-boot-starter-jetty</artifactId> </dependency> <dependency> - <groupId>org.springframework.retry</groupId> - <artifactId>spring-retry</artifactId> - </dependency> - <dependency> <groupId>org.springframework</groupId> <artifactId>spring-aspects</artifactId> </dependency> diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java index 675c0eaec4..4c6bd6cdc5 100755 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2023 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2021 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -176,6 +176,20 @@ public class AdminRestController implements CpsAdminApi { } /** + * Clean the given dataspace of any orphaned (module) data. + * + * @param apiVersion api version + * @param dataspaceName dataspace name + * + * @return a {@Link ResponseEntity} of {@link HttpStatus} NO_CONTENT + */ + @Override + public ResponseEntity<Void> cleanDataspace(final String apiVersion, final String dataspaceName) { + cpsModuleService.deleteAllUnusedYangModuleData(dataspaceName); + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + /** * Create a new anchor. * * @param dataspaceName dataspace name diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy index 2335a5e770..0d189783fd 100755 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2020-2021 Bell Canada. - * Modifications Copyright (C) 2021-2022 Nordix Foundation + * Modifications Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,15 +23,8 @@ package org.onap.cps.rest.controller -import org.onap.cps.api.CpsAnchorService - -import static org.onap.cps.api.parameters.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post - import org.mapstruct.factory.Mappers +import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataspaceService import org.onap.cps.api.CpsModuleService import org.onap.cps.api.exceptions.AlreadyDefinedException @@ -51,6 +44,12 @@ import org.springframework.util.LinkedMultiValueMap import org.springframework.util.MultiValueMap import spock.lang.Specification +import static org.onap.cps.api.parameters.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post + @WebMvcTest(AdminRestController) class AdminRestControllerSpec extends Specification { @@ -79,7 +78,7 @@ class AdminRestControllerSpec extends Specification { def dataspace = new Dataspace(name: dataspaceName) def 'Create new dataspace with #scenario.'() { - when: 'post is invoked' + when: 'post is invoked on endpoint for creating a dataspace' def response = mvc.perform( post("/cps/api/${apiVersion}/dataspaces") @@ -97,7 +96,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create dataspace over existing with same name.'() { - given: 'an endpoint' + given: 'the endpoint to create a dataspace' def createDataspaceEndpoint = "$basePath/v1/dataspaces" and: 'the service method throws an exception indicating the dataspace is already defined' def thrownException = new AlreadyDefinedException(dataspaceName, new RuntimeException()) @@ -115,7 +114,7 @@ class AdminRestControllerSpec extends Specification { def 'Get a dataspace.'() { given: 'service method returns a dataspace' mockCpsDataspaceService.getDataspace(dataspaceName) >> dataspace - and: 'an endpoint' + and: 'the endpoint for getting a dataspace by name' def getDataspaceEndpoint = "$basePath/v1/admin/dataspaces/$dataspaceName" when: 'get dataspace API is invoked' def response = mvc.perform(get(getDataspaceEndpoint)).andReturn().response @@ -124,6 +123,17 @@ class AdminRestControllerSpec extends Specification { response.getContentAsString().contains(dataspaceName) } + def 'Clean a dataspace.'() { + given: 'service method returns a dataspace' + mockCpsDataspaceService.getDataspace(dataspaceName) >> dataspace + and: 'the endpoint for cleaning a dataspace' + def postCleanDataspaceEndpoint = "$basePath/v1/admin/dataspaces/$dataspaceName/actions/clean" + when: 'post is invoked on the clean dataspace endpoint' + def response = mvc.perform(post(postCleanDataspaceEndpoint)).andReturn().response + then: 'no content is returned' + response.status == HttpStatus.NO_CONTENT.value() + } + def 'Get all dataspaces.'() { given: 'service method returns all dataspace' mockCpsDataspaceService.getAllDataspaces() >> [dataspace, new Dataspace(name: "dataspace-test2")] @@ -173,8 +183,7 @@ class AdminRestControllerSpec extends Specification { .param('schema-set-name', schemaSetName)) .andReturn().response then: 'associated service method is invoked with expected parameters' - 1 * mockCpsModuleService.createSchemaSet(dataspaceName, schemaSetName, _) >> - { args -> yangResourceMapCapture = args[2] } + 1 * mockCpsModuleService.createSchemaSet(dataspaceName, schemaSetName, _) >> { args -> yangResourceMapCapture = args[2] } yangResourceMapCapture['assembly.yang'] == "fake assembly content 1\n" yangResourceMapCapture['component.yang'] == "fake component content 1\n" and: 'response code indicates success' @@ -208,7 +217,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create schema set from zip archive having #caseDescriptor.'() { - given: 'an endpoint' + given: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'zip archive having #caseDescriptor is uploaded with create schema set request' def response = @@ -228,7 +237,7 @@ class AdminRestControllerSpec extends Specification { def 'Create schema set from file with unsupported filename extension.'() { given: 'file with unsupported filename extension (.doc)' def multipartFile = createMultipartFile("filename.doc", "content") - and: 'an endpoint' + and: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'file uploaded with schema set create request' def response = @@ -242,7 +251,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create schema set from #fileType file with IOException occurrence on processing.'() { - given: 'an endpoint' + given: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'file uploaded with schema set create request' def multipartFile = createMultipartFileForIOException(fileType) @@ -259,7 +268,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete schema set.'() { - given: 'an endpoint' + given: 'the endpoint for deleting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'delete schema set endpoint is invoked' def response = mvc.perform(delete(schemaSetEndpoint)).andReturn().response @@ -274,7 +283,7 @@ class AdminRestControllerSpec extends Specification { def thrownException = new SchemaSetInUseException(dataspaceName, schemaSetName) mockCpsModuleService.deleteSchemaSet(dataspaceName, schemaSetName, CASCADE_DELETE_PROHIBITED) >> { throw thrownException } - and: 'an endpoint' + and: 'the endpoint for deleting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'delete schema set endpoint is invoked' def response = mvc.perform(delete(schemaSetEndpoint)).andReturn().response @@ -286,7 +295,7 @@ class AdminRestControllerSpec extends Specification { given: 'service method returns a new schema set' mockCpsModuleService.getSchemaSet(dataspaceName, schemaSetName) >> new SchemaSet(name: schemaSetName, dataspaceName: dataspaceName) - and: 'an endpoint' + and: 'the endpoint for getting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'get schema set API is invoked' def response = mvc.perform(get(schemaSetEndpoint)).andReturn().response @@ -300,7 +309,7 @@ class AdminRestControllerSpec extends Specification { mockCpsModuleService.getSchemaSets(dataspaceName) >> [new SchemaSet(name: schemaSetName, dataspaceName: dataspaceName), new SchemaSet(name: "test-schemaset", dataspaceName: dataspaceName)] - and: 'an endpoint' + and: 'the endpoint for getting all schema sets' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'get schema sets API is invoked' def response = mvc.perform(get(schemaSetEndpoint)).andReturn().response @@ -315,7 +324,7 @@ class AdminRestControllerSpec extends Specification { def requestParams = new LinkedMultiValueMap<>() requestParams.add('schema-set-name', schemaSetName) requestParams.add('anchor-name', anchorName) - when: 'post is invoked' + when: 'post is invoked on the create anchors endpoint' def response = mvc.perform( post("/cps/api/${apiVersion}/dataspaces/my_dataspace/anchors") @@ -332,10 +341,10 @@ class AdminRestControllerSpec extends Specification { 'V2 API' | 'v2' || '' } - def 'Get existing anchor.'() { - given: 'service method returns a list of anchors' + def 'Get existing anchors.'() { + given: 'service method returns a list of (one) anchors' mockCpsAnchorService.getAnchors(dataspaceName) >> [anchor] - and: 'an endpoint' + and: 'the endpoint for getting all anchors' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors" when: 'get all anchors API is invoked' def response = mvc.perform(get(anchorEndpoint)).andReturn().response @@ -348,7 +357,7 @@ class AdminRestControllerSpec extends Specification { given: 'service method returns an anchor' mockCpsAnchorService.getAnchor(dataspaceName, anchorName) >> new Anchor(name: anchorName, dataspaceName: dataspaceName, schemaSetName: schemaSetName) - and: 'an endpoint' + and: 'the endpoint for getting an anchor' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors/$anchorName" when: 'get anchor API is invoked' def response = mvc.perform(get(anchorEndpoint)).andReturn().response @@ -361,7 +370,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete anchor.'() { - given: 'an endpoint' + given: 'the endpoint for deleting an anchor' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors/$anchorName" when: 'delete method is invoked on anchor endpoint' def response = mvc.perform(delete(anchorEndpoint)).andReturn().response @@ -372,7 +381,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete dataspace.'() { - given: 'an endpoint' + given: 'the endpoint for deleting a dataspace' def dataspaceEndpoint = "$basePath/v1/dataspaces" when: 'delete dataspace endpoint is invoked' def response = mvc.perform(delete(dataspaceEndpoint) diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy index ca89fafe83..f2f962422f 100755 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy @@ -30,8 +30,8 @@ import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder -import org.onap.cps.api.model.DeltaReportBuilder +import org.onap.cps.impl.DataNodeBuilder +import org.onap.cps.impl.DeltaReportBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.DateTimeUtility import org.onap.cps.utils.JsonObjectMapper diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy index f29654c99f..2b5c471287 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsQueryService import org.onap.cps.api.parameters.PaginationOption -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.JsonObjectMapper import org.onap.cps.utils.PrefixResolver import org.spockframework.spring.SpringBean diff --git a/cps-ri/pom.xml b/cps-ri/pom.xml index 2492cb837e..7ae85df851 100644 --- a/cps-ri/pom.xml +++ b/cps-ri/pom.xml @@ -26,7 +26,7 @@ <parent>
<groupId>org.onap.cps</groupId>
<artifactId>cps-parent</artifactId>
- <version>3.6.0-SNAPSHOT</version>
+ <version>3.6.1-SNAPSHOT</version>
<relativePath>../cps-parent/pom.xml</relativePath>
</parent>
@@ -56,10 +56,6 @@ <artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<dependency>
- <groupId>org.springframework.retry</groupId>
- <artifactId>spring-retry</artifactId>
- </dependency>
- <dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aspects</artifactId>
</dependency>
diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java index c43c8e2999..52fd7f2be1 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java @@ -50,12 +50,12 @@ import org.onap.cps.api.exceptions.CpsPathException; import org.onap.cps.api.exceptions.DataNodeNotFoundException; import org.onap.cps.api.exceptions.DataNodeNotFoundExceptionBatch; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.api.parameters.PaginationOption; import org.onap.cps.cpspath.parser.CpsPathQuery; import org.onap.cps.cpspath.parser.CpsPathUtil; import org.onap.cps.cpspath.parser.PathParsingException; +import org.onap.cps.impl.DataNodeBuilder; import org.onap.cps.ri.models.AnchorEntity; import org.onap.cps.ri.models.DataspaceEntity; import org.onap.cps.ri.models.FragmentEntity; diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java index 023e76ef89..aaf6165471 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2024 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -70,10 +70,6 @@ import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource; import org.opendaylight.yangtools.yang.parser.api.YangSyntaxErrorException; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.YangModelDependencyInfo; import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.retry.RetryContext; -import org.springframework.retry.annotation.Backoff; -import org.springframework.retry.annotation.Retryable; -import org.springframework.retry.support.RetrySynchronizationManager; import org.springframework.stereotype.Component; @Slf4j @@ -154,10 +150,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - // A retry is made to store the schema set if it fails because of duplicated yang resource exception that - // can occur in case of specific concurrent requests. - @Retryable(retryFor = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = - @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) public void storeSchemaSet(final String dataspaceName, final String schemaSetName, final Map<String, String> moduleReferenceNameToContentMap) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); @@ -174,6 +166,12 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ } @Override + public boolean schemaSetExists(final String dataspaceName, final String schemaSetName) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + return schemaSetRepository.existsByDataspaceAndName(dataspaceEntity, schemaSetName); + } + + @Override public Collection<SchemaSet> getSchemaSetsByDataspaceName(final String dataspaceName) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); final List<SchemaSetEntity> schemaSetEntities = schemaSetRepository.findByDataspace(dataspaceEntity); @@ -183,12 +181,8 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - // A retry is made to store the schema set if it fails because of duplicated yang resource exception that - // can occur in case of specific concurrent requests. - @Retryable(retryFor = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = - @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) @Timed(value = "cps.module.persistence.schemaset.store", - description = "Time taken to store a schemaset (list of module references") + description = "Time taken to store a schemaset (list of module references)") public void storeSchemaSetFromModules(final String dataspaceName, final String schemaSetName, final Map<String, String> newModuleNameToContentMap, final Collection<ModuleReference> allModuleReferences) { @@ -217,7 +211,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ schemaSetRepository.deleteByDataspaceAndNameIn(dataspaceEntity, schemaSetNames); } - @Override @Transactional public void updateSchemaSetFromModules(final String dataspaceName, final String schemaSetName, @@ -232,8 +225,10 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - public void deleteUnusedYangResourceModules() { - yangResourceRepository.deleteOrphans(); + public void deleteAllUnusedYangModuleData(final String dataspaceName) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + schemaSetRepository.deleteOrphanedSchemaSets(dataspaceEntity.getId()); + yangResourceRepository.deleteOrphanedYangResources(); } @Override @@ -242,15 +237,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ return moduleReferenceRepository.identifyNewModuleReferences(moduleReferencesToCheck); } - @Override - public Collection<ModuleReference> getModuleReferencesByAttribute(final String dataspaceName, - final String anchorName, - final Map<String, String> parentAttributes, - final Map<String, String> childAttributes) { - return moduleReferenceRepository.findModuleReferences(dataspaceName, anchorName, parentAttributes, - childAttributes); - } - private Set<YangResourceEntity> synchronizeYangResources( final Map<String, String> moduleReferenceNameToContentMap) { final Map<String, YangResourceEntity> checksumToEntityMap = moduleReferenceNameToContentMap.entrySet().stream() @@ -281,18 +267,10 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ yangResourceRepository.saveAll(newYangResourceEntities); } catch (final DataIntegrityViolationException dataIntegrityViolationException) { // Throw a CPS duplicated Yang resource exception if the cause of the error is a yang checksum - // database constraint violation. - // If it is not, then throw the original exception + // database constraint violation. If it is not, then throw the original exception final Optional<DuplicatedYangResourceException> convertedException = convertToDuplicatedYangResourceException( dataIntegrityViolationException, newYangResourceEntities); - convertedException.ifPresent( - e -> { - final RetryContext retryContext = RetrySynchronizationManager.getContext(); - int retryCount = retryContext == null ? 0 : retryContext.getRetryCount(); - log.warn("Cannot persist duplicated yang resource. System will attempt this method " - + "up to 5 times. Current retry count : {}", ++retryCount, e); - }); throw convertedException.isPresent() ? convertedException.get() : dataIntegrityViolationException; } } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceQuery.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceQuery.java index 85d0e438cb..c91e8de48d 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceQuery.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceQuery.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation. + * Copyright (C) 2022-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ package org.onap.cps.ri.repository; import java.util.Collection; -import java.util.Map; import org.onap.cps.api.model.ModuleReference; /** @@ -31,7 +30,4 @@ public interface ModuleReferenceQuery { Collection<ModuleReference> identifyNewModuleReferences(final Collection<ModuleReference> moduleReferencesToCheck); - Collection<ModuleReference> findModuleReferences(final String dataspaceName, final String anchorName, - final Map<String, String> parentAttributes, - final Map<String, String> childAttributes); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceRepositoryImpl.java index b98696ca72..7611dcd8a5 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceRepositoryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/ModuleReferenceRepositoryImpl.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation. + * Copyright (C) 2022-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,24 +20,18 @@ package org.onap.cps.ri.repository; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; -import jakarta.persistence.Query; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.model.ModuleReference; import org.springframework.transaction.annotation.Transactional; -@Slf4j @Transactional @RequiredArgsConstructor public class ModuleReferenceRepositoryImpl implements ModuleReferenceQuery { @@ -70,104 +64,14 @@ public class ModuleReferenceRepositoryImpl implements ModuleReferenceQuery { return identifyNewModuleReferencesForCmHandle(tempTableName); } - /** - * Finds module references based on specified dataspace, anchor, and attribute filters. - * This method constructs and executes a SQL query to retrieve module references. The query applies filters to - * parent and child fragments using the provided attribute maps. The `parentAttributes` are used to filter - * parent fragments, while `childAttributes` filter child fragments. - * - * @param dataspaceName the name of the dataspace to filter on. - * @param anchorName the name of the anchor to filter on. - * @param parentAttributes a map of attributes for filtering parent fragments. - * @param childAttributes a map of attributes for filtering child fragments. - * @return a collection of {@link ModuleReference} objects that match the specified filters. - */ - @Transactional - @SuppressWarnings("unchecked") - @Override - public Collection<ModuleReference> findModuleReferences(final String dataspaceName, final String anchorName, - final Map<String, String> parentAttributes, - final Map<String, String> childAttributes) { - - final String parentFragmentWhereClause = buildWhereClause(childAttributes, "parentFragment"); - final String childFragmentWhereClause = buildWhereClause(parentAttributes, "childFragment"); - - final String moduleReferencesSqlQuery = buildModuleReferencesSqlQuery(parentFragmentWhereClause, - childFragmentWhereClause); - - final Query query = entityManager.createNativeQuery(moduleReferencesSqlQuery); - setQueryParameters(query, parentAttributes, childAttributes, anchorName, dataspaceName); - return processQueryResults(query.getResultList()); - } - - private String buildWhereClause(final Map<String, String> attributes, final String alias) { - return attributes.keySet().stream() - .map(attributeName -> String.format("%s.attributes->>'%s' = ?", alias, attributeName)) - .collect(Collectors.joining(" AND ")); - } - - private void setQueryParameters(final Query query, final Map<String, String> parentAttributes, - final Map<String, String> childAttributes, final String anchorName, - final String dataspaceName) { - final String childAttributeValue = childAttributes.entrySet().iterator().next().getValue(); - query.setParameter(1, childAttributeValue); - - final String parentAttributeValue = parentAttributes.entrySet().iterator().next().getValue(); - query.setParameter(2, parentAttributeValue); - - query.setParameter(3, anchorName); - query.setParameter(4, dataspaceName); - } - - @SuppressFBWarnings(value = "VA_FORMAT_STRING_USES_NEWLINE", justification = "no \n in string just in file format") - private String buildModuleReferencesSqlQuery(final String parentFragmentClause, final String childFragmentClause) { - return """ - WITH Fragment AS ( - SELECT childFragment.attributes->>'id' AS schema_set_name - FROM fragment parentFragment - JOIN fragment childFragment ON parentFragment.parent_id = childFragment.id - JOIN anchor anchorInfo ON parentFragment.anchor_id = anchorInfo.id - JOIN dataspace dataspaceInfo ON anchorInfo.dataspace_id = dataspaceInfo.id - WHERE %s - AND %s - AND anchorInfo.name = ? - AND dataspaceInfo.name = ? - LIMIT 1 - ), - SchemaSet AS ( - SELECT id - FROM schema_set - WHERE name = (SELECT schema_set_name FROM Fragment) - ) - SELECT yangResource.module_name, yangResource.revision - FROM yang_resource yangResource - JOIN schema_set_yang_resources schemaSetYangResources - ON yangResource.id = schemaSetYangResources.yang_resource_id - WHERE schemaSetYangResources.schema_set_id = (SELECT id FROM SchemaSet); - """.formatted(parentFragmentClause, childFragmentClause); - } - - private Collection<ModuleReference> processQueryResults(final List<Object[]> queryResults) { - if (queryResults.isEmpty()) { - log.info("No module references found for the provided attributes."); - return Collections.emptyList(); - } - return queryResults.stream() - .map(queryResult -> { - final String name = (String) queryResult[0]; - final String revision = (String) queryResult[1]; - return new ModuleReference(name, revision); - }) - .collect(Collectors.toList()); - } - private Collection<ModuleReference> identifyNewModuleReferencesForCmHandle(final String tempTableName) { - final String sql = String.format( - "SELECT %1$s.module_name, %1$s.revision" - + " FROM %1$s LEFT JOIN yang_resource" - + " ON yang_resource.module_name=%1$s.module_name" - + " AND yang_resource.revision=%1$s.revision" - + " WHERE yang_resource.module_name IS NULL;", tempTableName); + final String sql = """ + SELECT %1$s.module_name, %1$s.revision + FROM %1$s + LEFT JOIN yang_resource + ON yang_resource.module_name=%1$s.module_name AND yang_resource.revision=%1$s.revision + WHERE yang_resource.module_name IS NULL; + """.formatted(tempTableName); @SuppressWarnings("unchecked") final List<Object[]> resultsAsObjects = entityManager.createNativeQuery(sql).getResultList(); diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java index 4e4948e601..fdd72624ba 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2020 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. - * Modifications Copyright (C) 2023-2024 Nordix Foundation + * Modifications Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,6 +36,8 @@ import org.springframework.stereotype.Repository; @Repository public interface SchemaSetRepository extends JpaRepository<SchemaSetEntity, Integer> { + boolean existsByDataspaceAndName(DataspaceEntity dataspaceEntity, String schemaSetName); + Optional<SchemaSetEntity> findByDataspaceAndName(DataspaceEntity dataspaceEntity, String schemaSetName); /** @@ -76,4 +78,13 @@ public interface SchemaSetRepository extends JpaRepository<SchemaSetEntity, Inte deleteByDataspaceIdAndNameIn(dataspaceEntity.getId(), schemaSetNames); } + /** + * Delete any schema set no longer used by any anchor. + */ + @Modifying + @Query(value = """ + DELETE FROM schema_set WHERE schema_set.dataspace_id = :dataspaceId AND + NOT EXISTS (SELECT 1 FROM anchor WHERE anchor.schema_set_id = schema_set.id) + """, nativeQuery = true) + void deleteOrphanedSchemaSets(@Param("dataspaceId") final int dataspaceId); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepository.java index 8350d5728c..410dcc2e26 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepository.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2021-2023 Nordix Foundation. + * Copyright (C) 2021-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,13 @@ import java.util.List; public interface SchemaSetYangResourceRepository { + + /** + * Link yang resources (ids) with a schema set (id). + * + * @param schemaSetId the schema set id + * @param yangResourceIds list of yang resource ids + */ void insertSchemaSetIdYangResourceId(final Integer schemaSetId, final List<Integer> yangResourceIds); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java index 831766cc9a..628502f846 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2020 Pantheon.tech - * Modifications Copyright (C) 2021-2024 Nordix Foundation + * Modifications Copyright (C) 2021-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,7 +92,9 @@ public interface YangResourceRepository extends JpaRepository<YangResourceEntity void deleteSchemaSetYangResourceForSchemaSetId(@Param("schemaSetId") int schemaSetId); @Modifying - @Query(value = "DELETE FROM yang_resource yr WHERE NOT EXISTS " - + "(SELECT 1 FROM schema_set_yang_resources ssyr WHERE ssyr.yang_resource_id = yr.id)", nativeQuery = true) - void deleteOrphans(); + @Query(value = """ + DELETE FROM yang_resource WHERE NOT EXISTS (SELECT 1 FROM schema_set_yang_resources + WHERE schema_set_yang_resources.yang_resource_id = yang_resource.id) + """, nativeQuery = true) + void deleteOrphanedYangResources(); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java index fa9feee1e7..9e89c8aed9 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java @@ -27,7 +27,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.parameters.PaginationOption; -import org.onap.cps.impl.utils.CpsValidator; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Component; @Slf4j diff --git a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy index c818f3ba1f..e927922acf 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy @@ -33,7 +33,7 @@ import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.exceptions.ConcurrencyException import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.JsonObjectMapper import org.springframework.dao.DataIntegrityViolationException import spock.lang.Specification diff --git a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy deleted file mode 100644 index 28a615b0e8..0000000000 --- a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy +++ /dev/null @@ -1,145 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022 Bell Canada. - * Modifications Copyright (C) 2021-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the 'License'); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ -package org.onap.cps.ri - -import org.hibernate.exception.ConstraintViolationException -import org.onap.cps.ri.models.DataspaceEntity -import org.onap.cps.ri.models.SchemaSetEntity -import org.onap.cps.ri.repository.DataspaceRepository -import org.onap.cps.ri.repository.ModuleReferenceRepository -import org.onap.cps.ri.repository.SchemaSetRepository -import org.onap.cps.ri.repository.YangResourceRepository -import org.onap.cps.spi.CpsAdminPersistenceService -import org.onap.cps.spi.CpsModulePersistenceService -import org.onap.cps.api.exceptions.DuplicatedYangResourceException -import org.onap.cps.api.model.ModuleReference -import org.spockframework.spring.SpringBean -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest -import org.springframework.dao.DataIntegrityViolationException -import org.springframework.retry.annotation.EnableRetry -import spock.lang.Specification - -import java.sql.SQLException - -@SpringBootTest(classes=[CpsModulePersistenceServiceImpl]) -@EnableRetry -class CpsModulePersistenceServiceConcurrencySpec extends Specification { - - @Autowired - CpsModulePersistenceService objectUnderTest - - @SpringBean - DataspaceRepository dataspaceRepository = Mock() - - @SpringBean - YangResourceRepository yangResourceRepository = Mock() - - @SpringBean - SchemaSetRepository schemaSetRepository = Mock() - - @SpringBean - CpsAdminPersistenceService cpsAdminPersistenceService = Mock() - - @SpringBean - ModuleReferenceRepository moduleReferenceRepository = Mock() - - def NEW_RESOURCE_NAME = 'some new resource' - def NEW_RESOURCE_CONTENT = 'module stores {\n' + - ' yang-version 1.1;\n' + - ' namespace "org:onap:ccsdk:sample";\n' + - '}' - - def newYangResourcesNameToContentMap = [(NEW_RESOURCE_NAME):NEW_RESOURCE_CONTENT] - - def yangResourceChecksum = 'b13faef573ed1374139d02c40d8ce09c80ea1dc70e63e464c1ed61568d48d539' - - def yangResourceChecksumDbConstraint = 'yang_resource_checksum_key' - - def sqlExceptionMessage = String.format('(checksum)=(%s)', yangResourceChecksum) - - def checksumIntegrityException = new DataIntegrityViolationException("checksum integrity exception", - new ConstraintViolationException('', new SQLException(sqlExceptionMessage), yangResourceChecksumDbConstraint)) - - def 'Store new schema set, maximum retries.'() { - given: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored' - objectUnderTest.storeSchemaSet('some dataspace', 'some new schema set', newYangResourcesNameToContentMap) - then: 'a duplicated yang resource exception is thrown ' - thrown(DuplicatedYangResourceException) - and: 'the system will attempt to save the data 5 times (because checksum integrity exception is thrown each time)' - 5 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - } - - def 'Store new schema set, succeed on third attempt.'() { - given: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored' - objectUnderTest.storeSchemaSet('some dataspace', 'some new schema set', newYangResourcesNameToContentMap) - then: 'no exception is thrown ' - noExceptionThrown() - and: 'the system will attempt to save the data 2 times with checksum integrity exception but then succeed' - 2 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - 1 * yangResourceRepository.saveAll(_) >> [] - } - - def 'Store schema set using modules, maximum retries.'() { - given: 'map of new modules, a list of existing modules, module reference' - def mapOfNewModules = [newModule1: 'module newmodule { yang-version 1.1; revision "2021-10-12" { } }'] - def moduleReferenceForExistingModule = new ModuleReference("test","2021-10-12") - def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule] - and: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored from a module' - objectUnderTest.storeSchemaSetFromModules('some dataspace', 'some new schema set' , mapOfNewModules, listOfExistingModulesModuleReference) - then: 'a duplicated yang resource exception is thrown ' - thrown(DuplicatedYangResourceException) - and: 'the system will attempt to save the data 5 times (because checksum integrity exception is thrown each time)' - 5 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - } - - def 'Store schema set using modules, succeed on third attempt.'() { - given: 'map of new modules, a list of existing modules, module reference' - def mapOfNewModules = [newModule1: 'module newmodule { yang-version 1.1; revision "2021-10-12" { } }'] - def moduleReferenceForExistingModule = new ModuleReference("test","2021-10-12") - def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule] - and: 'no pre-existing schemaset in database' - def dataspaceEntity = new DataspaceEntity() - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - yangResourceRepository.getResourceIdsByModuleReferences(_) >> [] - and: 'can retrieve schemaset details after storing it' - def schemaSetEntity = new SchemaSetEntity() - schemaSetRepository.getByDataspaceAndName(dataspaceEntity, 'new schema set') >> schemaSetEntity - when: 'a new schemaset is stored from a module' - objectUnderTest.storeSchemaSetFromModules('some dataspace', 'new schema set' , mapOfNewModules, listOfExistingModulesModuleReference) - then: 'no exception is thrown ' - noExceptionThrown() - and: 'the system will attempt to save the data 2 times with checksum integrity exception but then succeed' - 2 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - 1 * yangResourceRepository.saveAll(_) >> [] - } - -} diff --git a/cps-service/pom.xml b/cps-service/pom.xml index 7dda1327e0..1fe86cbb96 100644 --- a/cps-service/pom.xml +++ b/cps-service/pom.xml @@ -30,7 +30,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java index e71b44c40f..c6b8c60ab9 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2024 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. * ================================================================================ @@ -58,6 +58,15 @@ public interface CpsModuleService { Collection<ModuleReference> allModuleReferences); /** + * Check if a schema set exist in the given dataspace. + * + * @param dataspaceName Dataspace name + * @param schemaSetName Schema set name + * @return boolean, true if a schema set with the given name exist in the given dataspace + */ + boolean schemaSetExists(String dataspaceName, String schemaSetName); + + /** * Read schema set in the given dataspace. * * @param dataspaceName dataspace name @@ -150,47 +159,16 @@ public interface CpsModuleService { * The system will ignore the namespace of all module references. * * @param moduleReferencesToCheck the moduleReferencesToCheck - * @returns collection of module references (namespace will be always blank) + * @return collection of module references (namespace will be always blank) */ - Collection<ModuleReference> identifyNewModuleReferences( - Collection<ModuleReference> moduleReferencesToCheck); + Collection<ModuleReference> identifyNewModuleReferences(Collection<ModuleReference> moduleReferencesToCheck); /** - * Retrieves module references based on the provided dataspace name, anchor name and attribute filters - * for both parent and child fragments. - - * This method constructs and executes a SQL query to find module references from a database, using - * the specified `dataspaceName`, `anchorName` and two sets of attribute filters: one for parent fragments - * and one for child fragments. The method applies these filters to identify the appropriate fragments - * and schema sets, and then retrieves the corresponding module references. - - * The SQL query is dynamically built based on the provided attribute filters: - * - The `parentAttributes` map is used to filter the parent fragments. The entries in this map are - * converted into a WHERE clause for the parent fragments. - * - The `childAttributes` map is used to filter the child fragments. This is applied to the child fragments - * after filtering the parent fragments. + * Remove any Yang Resource Modules and Schema Sets from the given dataspace that are no longer referenced + * by any anchor. * - * @param dataspaceName the name of the dataspace to filter on. It is used to locate the relevant dataspace - * in the database. - * @param anchorName the name of the anchor to filter on. It is used to locate the relevant anchor within - * the dataspace. - * @param parentAttributes a map of attributes to filter parent fragments. Each entry in this map represents - * an attribute key-value pair used in the WHERE clause for parent fragments. - * @param childAttributes a map of attributes to filter child fragments. Each entry in this map represents - * an attribute key-value pair used in the WHERE clause for child fragments. - * @return a collection of {@link ModuleReference} objects that match the given criteria. - * Each {@code ModuleReference} contains information about a module's name and revision. - * @implNote The method assumes that both `parentAttributes` and `childAttributes` maps contain at least - * one entry. The first entry from `parentAttributes` is used to filter parent fragments, - * and the first entry from `childAttributes` is used to filter child fragments. - */ - Collection<ModuleReference> getModuleReferencesByAttribute(final String dataspaceName, final String anchorName, - final Map<String, String> parentAttributes, - final Map<String, String> childAttributes); - - /** - * Remove any Yang Resource Modules from the DB that are no longer referenced by any schema set. + * @param dataspaceName dataspace name */ - void deleteUnusedYangResourceModules(); + void deleteAllUnusedYangModuleData(String dataspaceName); } diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java b/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java index be80b636ad..6597aa3908 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java +++ b/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java @@ -26,20 +26,19 @@ import java.io.Serializable; import java.util.Collection; import java.util.Collections; import java.util.Map; -import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; -@Setter(AccessLevel.PROTECTED) +@Setter @Getter @EqualsAndHashCode +@NoArgsConstructor public class DataNode implements Serializable { private static final long serialVersionUID = 1482619410918597467L; - DataNode() {} - private String dataspace; private String schemaSetName; private String anchorName; diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java b/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java index df642628d0..761c6ad01d 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java +++ b/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java @@ -23,21 +23,20 @@ package org.onap.cps.api.model; import com.fasterxml.jackson.annotation.JsonInclude; import java.io.Serializable; import java.util.Map; -import lombok.AccessLevel; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; -@Setter(AccessLevel.PROTECTED) +@Setter @Getter @JsonInclude(JsonInclude.Include.NON_NULL) +@NoArgsConstructor public class DeltaReport { public static final String CREATE_ACTION = "create"; public static final String REMOVE_ACTION = "remove"; public static final String REPLACE_ACTION = "replace"; - DeltaReport() {} - private String action; private String xpath; private Map<String, Serializable> sourceData; diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsAnchorServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsAnchorServiceImpl.java index 1bd2b6af56..f18ae74c73 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsAnchorServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsAnchorServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,15 +18,15 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl; +package org.onap.cps.impl; import java.util.Collection; import lombok.RequiredArgsConstructor; import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.model.Anchor; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsAdminPersistenceService; import org.onap.cps.spi.CpsDataPersistenceService; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Service; @Service @@ -39,7 +39,7 @@ public class CpsAnchorServiceImpl implements CpsAnchorService { @Override public void createAnchor(final String dataspaceName, final String schemaSetName, final String anchorName) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName, anchorName); + cpsValidator.validateNameCharacters(dataspaceName, anchorName); cpsAdminPersistenceService.createAnchor(dataspaceName, schemaSetName, anchorName); } @@ -64,7 +64,7 @@ public class CpsAnchorServiceImpl implements CpsAnchorService { @Override public Collection<Anchor> getAnchorsBySchemaSetName(final String dataspaceName, final String schemaSetName) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); return cpsAdminPersistenceService.getAnchorsBySchemaSetName(dataspaceName, schemaSetName); } @@ -72,7 +72,6 @@ public class CpsAnchorServiceImpl implements CpsAnchorService { public Collection<Anchor> getAnchorsBySchemaSetNames(final String dataspaceName, final Collection<String> schemaSetNames) { cpsValidator.validateNameCharacters(dataspaceName); - cpsValidator.validateNameCharacters(schemaSetNames); return cpsAdminPersistenceService.getAnchorsBySchemaSetNames(dataspaceName, schemaSetNames); } diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java index 3b5baf5e68..71e6f79bb7 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java @@ -22,7 +22,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl; +package org.onap.cps.impl; import io.micrometer.core.annotation.Timed; import java.io.Serializable; @@ -42,15 +42,14 @@ import org.onap.cps.api.CpsDeltaService; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.model.Anchor; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; import org.onap.cps.api.model.DeltaReport; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.cpspath.parser.CpsPathUtil; import org.onap.cps.events.CpsDataUpdateEventsService; import org.onap.cps.events.model.Data.Operation; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsDataPersistenceService; import org.onap.cps.utils.ContentType; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.utils.DataMapUtils; import org.onap.cps.utils.JsonObjectMapper; import org.onap.cps.utils.PrefixResolver; diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataspaceServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java index af48f202bb..1a85147b64 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataspaceServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java @@ -21,14 +21,14 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl; +package org.onap.cps.impl; import java.util.Collection; import lombok.RequiredArgsConstructor; import org.onap.cps.api.CpsDataspaceService; import org.onap.cps.api.model.Dataspace; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsAdminPersistenceService; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Service; @Service diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDeltaServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java index a515b5d65a..d532001aec 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDeltaServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java @@ -18,7 +18,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl; +package org.onap.cps.impl; import java.io.Serializable; import java.util.ArrayList; @@ -32,7 +32,6 @@ import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.CpsDeltaService; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.model.DeltaReport; -import org.onap.cps.api.model.DeltaReportBuilder; import org.springframework.stereotype.Service; @Slf4j diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java index 9f5c0a3853..b16abcc31f 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2024 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2022 TechMahindra Ltd @@ -21,7 +21,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl; +package org.onap.cps.impl; import io.micrometer.core.annotation.Timed; import java.util.Collection; @@ -36,8 +36,8 @@ import org.onap.cps.api.model.ModuleDefinition; import org.onap.cps.api.model.ModuleReference; import org.onap.cps.api.model.SchemaSet; import org.onap.cps.api.parameters.CascadeDeleteAllowed; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsModulePersistenceService; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder; import org.onap.cps.yang.YangTextSchemaSourceSet; import org.springframework.stereotype.Service; @@ -58,7 +58,7 @@ public class CpsModuleServiceImpl implements CpsModuleService { description = "Time taken to create (and store) a schemaset") public void createSchemaSet(final String dataspaceName, final String schemaSetName, final Map<String, String> yangResourcesNameToContentMap) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); cpsModulePersistenceService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap); final YangTextSchemaSourceSet yangTextSchemaSourceSet = timedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap); @@ -69,14 +69,20 @@ public class CpsModuleServiceImpl implements CpsModuleService { public void createSchemaSetFromModules(final String dataspaceName, final String schemaSetName, final Map<String, String> newModuleNameToContentMap, final Collection<ModuleReference> allModuleReferences) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); cpsModulePersistenceService.storeSchemaSetFromModules(dataspaceName, schemaSetName, newModuleNameToContentMap, allModuleReferences); } @Override + public boolean schemaSetExists(final String dataspaceName, final String schemaSetName) { + cpsValidator.validateNameCharacters(dataspaceName); + return cpsModulePersistenceService.schemaSetExists(dataspaceName, schemaSetName); + } + + @Override public SchemaSet getSchemaSet(final String dataspaceName, final String schemaSetName) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); final var yangTextSchemaSourceSet = yangTextSchemaSourceSetCache .get(dataspaceName, schemaSetName); return SchemaSet.builder().name(schemaSetName).dataspaceName(dataspaceName) @@ -96,7 +102,7 @@ public class CpsModuleServiceImpl implements CpsModuleService { @Transactional public void deleteSchemaSet(final String dataspaceName, final String schemaSetName, final CascadeDeleteAllowed cascadeDeleteAllowed) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); final Collection<Anchor> anchors = cpsAnchorService.getAnchorsBySchemaSetName(dataspaceName, schemaSetName); if (!anchors.isEmpty() && isCascadeDeleteProhibited(cascadeDeleteAllowed)) { throw new SchemaSetInUseException(dataspaceName, schemaSetName); @@ -112,7 +118,6 @@ public class CpsModuleServiceImpl implements CpsModuleService { @Transactional public void deleteSchemaSetsWithCascade(final String dataspaceName, final Collection<String> schemaSetNames) { cpsValidator.validateNameCharacters(dataspaceName); - cpsValidator.validateNameCharacters(schemaSetNames); final Collection<String> anchorNames = cpsAnchorService.getAnchorsBySchemaSetNames(dataspaceName, schemaSetNames) .stream().map(Anchor::getName).collect(Collectors.toSet()); @@ -127,7 +132,7 @@ public class CpsModuleServiceImpl implements CpsModuleService { public void upgradeSchemaSetFromModules(final String dataspaceName, final String schemaSetName, final Map<String, String> newModuleNameToContentMap, final Collection<ModuleReference> allModuleReferences) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); cpsModulePersistenceService.updateSchemaSetFromModules(dataspaceName, schemaSetName, newModuleNameToContentMap, allModuleReferences); yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName); @@ -169,20 +174,10 @@ public class CpsModuleServiceImpl implements CpsModuleService { return cpsModulePersistenceService.identifyNewModuleReferences(moduleReferencesToCheck); } - @Timed(value = "cps.module.service.module.reference.query.by.attribute", - description = "Time taken to query list of module references by attribute (e.g moduleSetTag)") - @Override - public Collection<ModuleReference> getModuleReferencesByAttribute(final String dataspaceName, - final String anchorName, - final Map<String, String> parentAttributes, - final Map<String, String> childAttributes) { - return cpsModulePersistenceService.getModuleReferencesByAttribute(dataspaceName, anchorName, parentAttributes, - childAttributes); - } - @Override - public void deleteUnusedYangResourceModules() { - cpsModulePersistenceService.deleteUnusedYangResourceModules(); + public void deleteAllUnusedYangModuleData(final String dataspaceName) { + cpsValidator.validateNameCharacters(dataspaceName); + cpsModulePersistenceService.deleteAllUnusedYangModuleData(dataspaceName); } private boolean isCascadeDeleteProhibited(final CascadeDeleteAllowed cascadeDeleteAllowed) { diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsQueryServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsQueryServiceImpl.java index 1d6f0c3ab6..2687d8faee 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsQueryServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsQueryServiceImpl.java @@ -19,7 +19,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl; +package org.onap.cps.impl; import io.micrometer.core.annotation.Timed; import java.util.Collection; @@ -29,8 +29,8 @@ import org.onap.cps.api.CpsQueryService; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.api.parameters.PaginationOption; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsDataPersistenceService; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Service; @Service diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DataNodeBuilder.java b/cps-service/src/main/java/org/onap/cps/impl/DataNodeBuilder.java index d509f53525..a78f3d9826 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DataNodeBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/impl/DataNodeBuilder.java @@ -20,7 +20,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model; +package org.onap.cps.impl; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -33,6 +33,7 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.exceptions.DataValidationException; +import org.onap.cps.api.model.DataNode; import org.onap.cps.utils.YangUtils; import org.opendaylight.yangtools.yang.common.Ordering; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReportBuilder.java b/cps-service/src/main/java/org/onap/cps/impl/DeltaReportBuilder.java index a8e922f3df..fdc2e939d6 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReportBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/impl/DeltaReportBuilder.java @@ -18,11 +18,12 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model; +package org.onap.cps.impl; import java.io.Serializable; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.onap.cps.api.model.DeltaReport; @Slf4j public class DeltaReportBuilder { diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/YangTextSchemaSourceSetCache.java b/cps-service/src/main/java/org/onap/cps/impl/YangTextSchemaSourceSetCache.java index 8b85dfca32..e7e7b1c5ce 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/YangTextSchemaSourceSetCache.java +++ b/cps-service/src/main/java/org/onap/cps/impl/YangTextSchemaSourceSetCache.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 Bell Canada - * Modifications Copyright (C) 2022-2023 Nordix Foundation + * Modifications Copyright (C) 2022-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,15 +20,15 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl; +package org.onap.cps.impl; import com.google.errorprone.annotations.CanIgnoreReturnValue; import io.micrometer.core.instrument.Metrics; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import lombok.RequiredArgsConstructor; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsModulePersistenceService; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.yang.YangTextSchemaSourceSet; import org.onap.cps.yang.YangTextSchemaSourceSetBuilder; import org.springframework.cache.annotation.CacheConfig; @@ -60,7 +60,7 @@ public class YangTextSchemaSourceSetCache { */ @Cacheable(key = "#p0.concat('-').concat(#p1)") public YangTextSchemaSourceSet get(final String dataspaceName, final String schemaSetName) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); final Map<String, String> yangResourceNameToContent = cpsModulePersistenceService.getYangSchemaResources(dataspaceName, schemaSetName); return YangTextSchemaSourceSetBuilder.of(yangResourceNameToContent); @@ -78,7 +78,7 @@ public class YangTextSchemaSourceSetCache { @CanIgnoreReturnValue public YangTextSchemaSourceSet updateCache(final String dataspaceName, final String schemaSetName, final YangTextSchemaSourceSet yangTextSchemaSourceSet) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); yangSchemaCacheCounter.incrementAndGet(); return yangTextSchemaSourceSet; } @@ -91,9 +91,8 @@ public class YangTextSchemaSourceSetCache { */ @CacheEvict(key = "#p0.concat('-').concat(#p1)") public void removeFromCache(final String dataspaceName, final String schemaSetName) { - cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + cpsValidator.validateNameCharacters(dataspaceName); yangSchemaCacheCounter.decrementAndGet(); - // Spring provides implementation for removing object from cache } } diff --git a/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java b/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java deleted file mode 100644 index 6bd3e1f204..0000000000 --- a/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.init; - -import java.util.concurrent.TimeUnit; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.onap.cps.api.CpsModuleService; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -@Slf4j -@RequiredArgsConstructor -@Service -public class DbCleaner { - - private final CpsModuleService cpsModuleService; - - /** - * This method will clean up the db during application start up. - * It wil run once and currently only removes unused yang resource modules. - * - */ - @Scheduled(initialDelay = 1, timeUnit = TimeUnit.SECONDS) - public void cleanDbAtStartUp() { - log.info("CPS Application started, commencing DB clean up"); - cpsModuleService.deleteUnusedYangResourceModules(); - log.info("DB clean up completed"); - } -} diff --git a/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java b/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java index 4cfc287a5d..86ad50252d 100755 --- a/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java +++ b/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2024 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2022 TechMahindra Ltd. * ================================================================================ @@ -65,6 +65,14 @@ public interface CpsModulePersistenceService { final Map<String, String> newModuleNameToContentMap, final Collection<ModuleReference> allModuleReferences); + /** + * Checks whether a schema set exists in the specified dataspace. + * + * @param dataspaceName dataspace name + * @param schemaSetName schema set name + * @return {@code true} if the schema set exists in the given dataspace; {@code false} otherwise + */ + boolean schemaSetExists(String dataspaceName, String schemaSetName); /** * Get all schema sets for a given dataspace. @@ -138,35 +146,20 @@ public interface CpsModulePersistenceService { String moduleName, String moduleRevision); /** - * Remove unused Yang Resource Modules. + * Remove any unused Yang Resource Modules and Schema Sets from the given dataspace. + * + * @param dataspaceName dataspace name */ - void deleteUnusedYangResourceModules(); + void deleteAllUnusedYangModuleData(String dataspaceName); /** * Identify new module references from those returned by a node compared to what is in CPS already. * The system will ignore the namespace of all module references. * * @param moduleReferencesToCheck the module references ot check - * @returns Collection of {@link ModuleReference} (namespace will be always blank) - * - */ - Collection<ModuleReference> identifyNewModuleReferences( - Collection<ModuleReference> moduleReferencesToCheck); - - /** - * Retrieves module references based on the specified dataspace, anchor, and attribute filters. - - * Constructs and executes a SQL query to find module references by applying filters for parent and child fragments. - * Uses `parentAttributes` for filtering parent fragments and `childAttributes` for filtering child fragments. + * @return Collection of {@link ModuleReference} (namespace will be always blank) * - * @param dataspaceName the name of the dataspace to filter on. - * @param anchorName the name of the anchor to filter on. - * @param parentAttributes a map of attributes for filtering parent fragments. - * @param childAttributes a map of attributes for filtering child fragments. - * @return a collection of {@link ModuleReference} objects matching the criteria. */ - Collection<ModuleReference> getModuleReferencesByAttribute(final String dataspaceName, final String anchorName, - final Map<String, String> parentAttributes, - final Map<String, String> childAttributes); + Collection<ModuleReference> identifyNewModuleReferences(Collection<ModuleReference> moduleReferencesToCheck); } diff --git a/cps-service/src/main/java/org/onap/cps/impl/utils/CpsValidator.java b/cps-service/src/main/java/org/onap/cps/utils/CpsValidator.java index 75bcf126a4..93f51ee58d 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/utils/CpsValidator.java +++ b/cps-service/src/main/java/org/onap/cps/utils/CpsValidator.java @@ -18,7 +18,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.impl.utils; +package org.onap.cps.utils; import org.onap.cps.api.parameters.PaginationOption; diff --git a/cps-service/src/main/java/org/onap/cps/utils/PrefixResolver.java b/cps-service/src/main/java/org/onap/cps/utils/PrefixResolver.java index 646ed556f5..bd348a25d1 100644 --- a/cps-service/src/main/java/org/onap/cps/utils/PrefixResolver.java +++ b/cps-service/src/main/java/org/onap/cps/utils/PrefixResolver.java @@ -21,11 +21,11 @@ package org.onap.cps.utils; import lombok.RequiredArgsConstructor; -import org.onap.cps.api.impl.YangTextSchemaSourceSetCache; import org.onap.cps.api.model.Anchor; import org.onap.cps.cpspath.parser.CpsPathPrefixType; import org.onap.cps.cpspath.parser.CpsPathQuery; import org.onap.cps.cpspath.parser.CpsPathUtil; +import org.onap.cps.impl.YangTextSchemaSourceSetCache; import org.onap.cps.yang.YangTextSchemaSourceSet; import org.opendaylight.yangtools.yang.common.QName; import org.opendaylight.yangtools.yang.model.api.DataNodeContainer; diff --git a/cps-service/src/main/java/org/onap/cps/utils/YangParser.java b/cps-service/src/main/java/org/onap/cps/utils/YangParser.java index 7fa10a134a..08f450e2f1 100644 --- a/cps-service/src/main/java/org/onap/cps/utils/YangParser.java +++ b/cps-service/src/main/java/org/onap/cps/utils/YangParser.java @@ -29,8 +29,8 @@ import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.exceptions.DataValidationException; -import org.onap.cps.api.impl.YangTextSchemaSourceSetCache; import org.onap.cps.api.model.Anchor; +import org.onap.cps.impl.YangTextSchemaSourceSetCache; import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder; import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; import org.opendaylight.yangtools.yang.model.api.SchemaContext; diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsAnchorServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsAnchorServiceImplSpec.groovy index e8617d445d..d78c8bb47f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsAnchorServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsAnchorServiceImplSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,10 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl +package org.onap.cps.impl -import org.onap.cps.impl.utils.CpsValidator + +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsAdminPersistenceService import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.api.exceptions.ModuleNamesNotFoundException @@ -40,8 +41,8 @@ class CpsAnchorServiceImplSpec extends Specification { objectUnderTest.createAnchor('someDataspace', 'someSchemaSet', 'someAnchorName') then: 'the persistence service method is invoked with same parameters' 1 * mockCpsAdminPersistenceService.createAnchor('someDataspace', 'someSchemaSet', 'someAnchorName') - and: 'the CpsValidator is called on the dataspaceName, schemaSetName and anchorName' - 1 * mockCpsValidator.validateNameCharacters('someDataspace', 'someSchemaSet', 'someAnchorName') + and: 'the CpsValidator is called on the dataspaceName and anchorName' + 1 * mockCpsValidator.validateNameCharacters('someDataspace', 'someAnchorName') } def 'Retrieve all anchors for dataspace.'() { @@ -64,8 +65,8 @@ class CpsAnchorServiceImplSpec extends Specification { def result = objectUnderTest.getAnchorsBySchemaSetName('someDataspace', 'someSchemaSet') then: 'the collection provided by persistence service is returned as result' result == anchors - and: 'the CpsValidator is called on the dataspaceName, schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('someDataspace', 'someSchemaSet') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('someDataspace') } def 'Retrieve all anchors for multiple schema-sets.'() { @@ -76,9 +77,8 @@ class CpsAnchorServiceImplSpec extends Specification { def result = objectUnderTest.getAnchorsBySchemaSetNames('someDataspace', ['schemaSet1', 'schemaSet2']) then: 'the collection provided by persistence service is returned as result' result == anchors - and: 'the CpsValidator is called on the dataspace name and schema-set names' + and: 'the CpsValidator is called on the dataspace name' 1 * mockCpsValidator.validateNameCharacters('someDataspace') - 1 * mockCpsValidator.validateNameCharacters(_) } def 'Retrieve anchor for dataspace and provided anchor name.'() { diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy index ff44cb20a2..34d2b7564a 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy @@ -21,7 +21,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl +package org.onap.cps.impl import ch.qos.logback.classic.Level import ch.qos.logback.classic.Logger @@ -31,7 +31,7 @@ import org.onap.cps.TestUtils import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDeltaService import org.onap.cps.events.CpsDataUpdateEventsService -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.exceptions.ConcurrencyException @@ -40,7 +40,6 @@ import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.exceptions.SessionManagerException import org.onap.cps.api.exceptions.SessionTimeoutException import org.onap.cps.api.model.Anchor -import org.onap.cps.api.model.DataNodeBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import org.onap.cps.utils.PrefixResolver diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataspaceServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy index ae68929c42..97f6fba4d3 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataspaceServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy @@ -18,9 +18,10 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl +package org.onap.cps.impl -import org.onap.cps.impl.utils.CpsValidator + +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsAdminPersistenceService import org.onap.cps.api.model.Dataspace import spock.lang.Specification diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDeltaServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDeltaServiceImplSpec.groovy index 65dac62710..d3dfcf8826 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDeltaServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDeltaServiceImplSpec.groovy @@ -18,7 +18,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl +package org.onap.cps.impl import org.onap.cps.api.model.DataNode import spock.lang.Specification diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy index 97b9f7fffd..041ce605c4 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2024 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -21,11 +21,11 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl +package org.onap.cps.impl import org.onap.cps.TestUtils import org.onap.cps.api.CpsAnchorService -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsModulePersistenceService import org.onap.cps.api.exceptions.DuplicatedYangResourceException import org.onap.cps.api.exceptions.ModelValidationException @@ -54,23 +54,23 @@ class CpsModuleServiceImplSpec extends Specification { def 'Create schema set.'() { when: 'Create schema set method is invoked' - objectUnderTest.createSchemaSet('someDataspace', 'someSchemaSet', [:]) + objectUnderTest.createSchemaSet('someDataspace', 'schemaSetName@with Special!Characters', [:]) then: 'Parameters are validated and processing is delegated to persistence service' - 1 * mockCpsModulePersistenceService.storeSchemaSet('someDataspace', 'someSchemaSet', [:]) - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('someDataspace', 'someSchemaSet') + 1 * mockCpsModulePersistenceService.storeSchemaSet('someDataspace', 'schemaSetName@with Special!Characters', [:]) + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('someDataspace') } def 'Create schema set from new modules and existing modules.'() { given: 'a list of existing modules module reference' - def moduleReferenceForExistingModule = new ModuleReference('test', '2021-10-12','test.org') + def moduleReferenceForExistingModule = new ModuleReference('test', '2021-10-12', 'test.org') def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule] when: 'create schema set from modules method is invoked' objectUnderTest.createSchemaSetFromModules('someDataspaceName', 'someSchemaSetName', [newModule: 'newContent'], listOfExistingModulesModuleReference) then: 'processing is delegated to persistence service' 1 * mockCpsModulePersistenceService.storeSchemaSetFromModules('someDataspaceName', 'someSchemaSetName', [newModule: 'newContent'], listOfExistingModulesModuleReference) - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('someDataspaceName', 'someSchemaSetName') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('someDataspaceName') } def 'Create schema set from invalid resources'() { @@ -100,15 +100,15 @@ class CpsModuleServiceImplSpec extends Specification { given: 'an already present schema set' def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') and: 'yang resource cache returns the expected schema set' - mockYangTextSchemaSourceSetCache.get('someDataspace', 'someSchemaSet') >> YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + mockYangTextSchemaSourceSetCache.get('someDataspace', 'schemaSetName@with Special!Characters') >> YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) when: 'get schema set method is invoked' - def result = objectUnderTest.getSchemaSet('someDataspace', 'someSchemaSet') + def result = objectUnderTest.getSchemaSet('someDataspace', 'schemaSetName@with Special!Characters') then: 'the correct schema set is returned' - result.getName().contains('someSchemaSet') + result.getName().contains('schemaSetName@with Special!Characters') result.getDataspaceName().contains('someDataspace') result.getModuleReferences().contains(new ModuleReference('stores', '2020-09-15', 'org:onap:ccsdk:sample')) - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('someDataspace', 'someSchemaSet') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('someDataspace') } def 'Get schema sets by dataspace name.'() { @@ -142,8 +142,8 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset') and: 'schema set will be removed from the cache' 1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset') - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('my-dataspace', _) + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('my-dataspace') where: 'following parameters are used' numberOfAnchors << [0, 3] } @@ -159,8 +159,8 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset') and: 'schema set will be removed from the cache' 1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset') - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('my-dataspace') } def 'Delete schema-set when cascade is prohibited and schema-set has anchors.'() { @@ -185,17 +185,15 @@ class CpsModuleServiceImplSpec extends Specification { 2 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', _) and: 'the CpsValidator is called on the dataspaceName' 1 * mockCpsValidator.validateNameCharacters('my-dataspace') - and: 'the CpsValidator is called on the schemaSetNames' - 1 * mockCpsValidator.validateNameCharacters(_) where: 'following parameters are used' numberOfAnchors << [0, 3] } def 'Upgrade existing schema set'() { when: 'schema set update is requested' - objectUnderTest.upgradeSchemaSetFromModules('my-dataspace', 'my-schemaset', [:], moduleReferences) + objectUnderTest.upgradeSchemaSetFromModules('my-dataspace', 'my-schemaset', [:], moduleReferences) then: 'no exception is thrown ' - noExceptionThrown() + noExceptionThrown() } def 'Get all yang resources module references.'() { @@ -206,7 +204,7 @@ class CpsModuleServiceImplSpec extends Specification { def result = objectUnderTest.getYangResourceModuleReferences('someDataspaceName') then: 'the list provided by persistence service is returned as result' result == moduleReferences - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' + and: 'the CpsValidator is called on the dataspaceName' 1 * mockCpsValidator.validateNameCharacters('someDataspaceName') } @@ -218,7 +216,7 @@ class CpsModuleServiceImplSpec extends Specification { def result = objectUnderTest.getYangResourcesModuleReferences('someDataspaceName', 'someAnchorName') then: 'the list provided by persistence service is returned as result' result == moduleReferences - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' + and: 'the CpsValidator is called on the dataspaceName and anchorName' 1 * mockCpsValidator.validateNameCharacters('someDataspaceName', 'someAnchorName') } @@ -231,22 +229,6 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsModulePersistenceService.identifyNewModuleReferences(moduleReferencesToCheck) } - def 'Get module references when queried by attributes'() { - given: 'a valid dataspace name and anchor name' - def dataspaceName = 'someDataspace' - def anchorName = 'someAnchor' - and: 'a set of parent attributes and child attributes used for filtering' - def parentAttributes = ['some-property-key1': 'some-property-val1'] - def childAttributes = ['some-property-key2': 'some-property-val2'] - and: 'a list of expected module references returned by the persistence service' - def expectedModuleReferences = [new ModuleReference(moduleName: 'some-name', revision: 'some-revision')] - mockCpsModulePersistenceService.getModuleReferencesByAttribute(dataspaceName, anchorName, parentAttributes, childAttributes) >> expectedModuleReferences - when: 'the method is invoked to retrieve module references by attributes' - def actualModuleReferences = objectUnderTest.getModuleReferencesByAttribute(dataspaceName, anchorName, parentAttributes, childAttributes) - then: 'the retrieved module references should match the expected module references' - assert actualModuleReferences == expectedModuleReferences - } - def 'Getting module definitions with module name'() { given: 'module persistence service returns module definitions for module name' @@ -270,11 +252,18 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsValidator.validateNameCharacters('some-dataspace-name', 'some-anchor-name') } - def 'Delete unused yang resource modules.'() { - when: 'deleting unused yang resource modules' - objectUnderTest.deleteUnusedYangResourceModules() - then: 'it is delegated to the module persistence service' - 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() + def 'Delete unused yang module data for a dataspace.'() { + when: 'deleting unused yang module data' + objectUnderTest.deleteAllUnusedYangModuleData('some-dataspace-name') + then: 'it is delegated to the module persistence service with the correct parameters' + 1 * mockCpsModulePersistenceService.deleteAllUnusedYangModuleData('some-dataspace-name') + } + + def 'Schema set exists.'() { + when: 'checking if schema set exists' + objectUnderTest.schemaSetExists('some-dataspace-name', 'some-schema-set-name') + then: 'the call is delegated to the module persistence service' + 1 * mockCpsModulePersistenceService.schemaSetExists('some-dataspace-name', 'some-schema-set-name') } def getModuleReferences() { diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsQueryServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsQueryServiceImplSpec.groovy index 7d2932570e..237e4e4592 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsQueryServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsQueryServiceImplSpec.groovy @@ -19,9 +19,10 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl +package org.onap.cps.impl -import org.onap.cps.impl.utils.CpsValidator + +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.parameters.PaginationOption diff --git a/cps-service/src/test/groovy/org/onap/cps/api/model/DataNodeBuilderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/DataNodeBuilderSpec.groovy index 24c78864a5..1597d45761 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/model/DataNodeBuilderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/DataNodeBuilderSpec.groovy @@ -19,10 +19,11 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model +package org.onap.cps.impl import org.onap.cps.TestUtils import org.onap.cps.api.exceptions.DataValidationException +import org.onap.cps.api.model.DataNode import org.onap.cps.utils.ContentType import org.onap.cps.utils.DataMapUtils import org.onap.cps.utils.YangParserHelper diff --git a/cps-service/src/test/groovy/org/onap/cps/api/model/DeltaReportBuilderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/DeltaReportBuilderSpec.groovy index 94e3ed5c26..2decefff21 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/model/DeltaReportBuilderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/DeltaReportBuilderSpec.groovy @@ -18,7 +18,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model +package org.onap.cps.impl import spock.lang.Specification diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/E2ENetworkSliceSpec.groovy index f2fed7c899..8171209947 100755 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/E2ENetworkSliceSpec.groovy @@ -1,176 +1,176 @@ -/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2021-2024 Nordix Foundation.
- * Modifications Copyright (C) 2021-2022 Bell Canada.
- * Modifications Copyright (C) 2021 Pantheon.tech
- * Modifications Copyright (C) 2022-2024 TechMahindra Ltd.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-package org.onap.cps.api.impl
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.onap.cps.TestUtils
-import org.onap.cps.api.CpsAnchorService
-import org.onap.cps.api.CpsDeltaService
-import org.onap.cps.events.CpsDataUpdateEventsService
-import org.onap.cps.impl.utils.CpsValidator
-import org.onap.cps.spi.CpsDataPersistenceService
-import org.onap.cps.spi.CpsModulePersistenceService
-import org.onap.cps.api.model.Anchor
-import org.onap.cps.utils.ContentType
-import org.onap.cps.utils.JsonObjectMapper
-import org.onap.cps.utils.PrefixResolver
-import org.onap.cps.utils.YangParser
-import org.onap.cps.utils.YangParserHelper
-import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder
-import org.onap.cps.yang.YangTextSchemaSourceSetBuilder
-import spock.lang.Specification
-
-class E2ENetworkSliceSpec extends Specification {
- def mockModuleStoreService = Mock(CpsModulePersistenceService)
- def mockDataStoreService = Mock(CpsDataPersistenceService)
- def mockCpsAnchorService = Mock(CpsAnchorService)
- def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache)
- def mockCpsValidator = Mock(CpsValidator)
- def timedYangTextSchemaSourceSetBuilder = new TimedYangTextSchemaSourceSetBuilder()
- def yangParser = new YangParser(new YangParserHelper(), mockYangTextSchemaSourceSetCache, timedYangTextSchemaSourceSetBuilder)
- def mockCpsDeltaService = Mock(CpsDeltaService)
- def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper())
- def mockPrefixResolver = Mock(PrefixResolver)
-
- def cpsModuleServiceImpl = new CpsModuleServiceImpl(mockModuleStoreService,
- mockYangTextSchemaSourceSetCache, mockCpsAnchorService, mockCpsValidator,timedYangTextSchemaSourceSetBuilder)
-
- def mockDataUpdateEventsService = Mock(CpsDataUpdateEventsService)
- def cpsDataServiceImpl = new CpsDataServiceImpl(mockDataStoreService, mockDataUpdateEventsService, mockCpsAnchorService, mockCpsValidator,
- yangParser, mockCpsDeltaService, jsonObjectMapper, mockPrefixResolver)
- def dataspaceName = 'someDataspace'
- def anchorName = 'someAnchor'
- def schemaSetName = 'someSchemaSet'
- def noTimestamp = null
-
- def 'E2E model can be parsed by CPS.'() {
- given: 'Valid yang resource as name-to-content map'
- def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap(
- 'ietf/ietf-inet-types@2013-07-15.yang',
- 'ietf/ietf-yang-types@2013-07-15.yang',
- 'e2e/basic/ran-network2020-08-06.yang'
- )
- when: 'Create schema set method is invoked'
- cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap)
- then: 'Parameters are validated and processing is delegated to persistence service'
- 1 * mockModuleStoreService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap)
- }
-
- def 'E2E Coverage Area-Tracking Area & TA-Cell mapping model can be parsed by CPS.'() {
- given: 'Valid yang resource as name-to-content map'
- def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap(
- 'e2e/basic/cps-cavsta-onap-internal2021-01-28.yang')
- when: 'Create schema set method is invoked'
- cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap)
- then: 'Parameters are validated and processing is delegated to persistence service'
- 1 * mockModuleStoreService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap)
- }
-
- def 'E2E Coverage Area-Tracking Area & TA-Cell mapping data can be parsed by CPS.'() {
- given: 'Valid yang resource as name-to-content map'
- def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap(
- 'e2e/basic/cps-cavsta-onap-internal2021-01-28.yang')
- def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext()
- def dataNodeStored
- and : 'a valid json is provided for the model'
- def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-Cavsta-Data.txt')
- and : 'all the further dependencies are mocked '
- mockCpsAnchorService.getAnchor(dataspaceName, anchorName) >>
- new Anchor().builder().name(anchorName).schemaSetName(schemaSetName).dataspaceName(dataspaceName).build()
- mockYangTextSchemaSourceSetCache.get(dataspaceName, schemaSetName) >>
- YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap)
- mockModuleStoreService.getYangSchemaResources(dataspaceName, schemaSetName) >> schemaContext
- when: 'saveData method is invoked'
- cpsDataServiceImpl.saveData(dataspaceName, anchorName, jsonData, noTimestamp)
- then: 'Parameters are validated and processing is delegated to persistence service'
- 1 * mockDataStoreService.storeDataNodes('someDataspace', 'someAnchor', _) >>
- { args -> dataNodeStored = args[2]}
- def child = dataNodeStored[0].childDataNodes[0]
- assert child.childDataNodes.size() == 1
- and: 'list of Tracking Area for a Coverage Area are stored with correct xpath and child nodes '
- def listOfTAForCoverageArea = child.childDataNodes[0]
- listOfTAForCoverageArea.xpath == '/ran-coverage-area/pLMNIdList[@mcc=\'310\' and @mnc=\'410\']/' +
- 'coverage-area[@coverageArea=\'Washington\']'
- listOfTAForCoverageArea.childDataNodes[0].leaves.get('nRTAC') == 234
- and: 'list of cells in a tracking area are stored with correct xpath and child nodes '
- def listOfCellsInTrackingArea = listOfTAForCoverageArea.childDataNodes[0]
- listOfCellsInTrackingArea.xpath == '/ran-coverage-area/pLMNIdList[@mcc=\'310\' and @mnc=\'410\']/' +
- 'coverage-area[@coverageArea=\'Washington\']/coverageAreaTAList[@nRTAC=\'234\']'
- listOfCellsInTrackingArea.childDataNodes[0].leaves.get('cellLocalId') == 15709
- }
-
- def 'E2E Coverage Area-Tracking Area & TA-Cell mapping data can be parsed for RAN inventory.'() {
- def dataNodeStored
- given: 'valid yang resource as name-to-content map'
- def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap(
- 'e2e/basic/cps-ran-inventory@2021-01-28.yang')
- def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext()
- and : 'a valid json is provided for the model'
- def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-ran-inventory-data.json')
- and : 'all the further dependencies are mocked '
- mockCpsAnchorService.getAnchor('someDataspace', 'someAnchor') >>
- new Anchor().builder().name('someAnchor').schemaSetName('someSchemaSet').dataspaceName(dataspaceName).build()
- mockYangTextSchemaSourceSetCache.get('someDataspace', 'someSchemaSet') >> YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap)
- mockModuleStoreService.getYangSchemaResources('someDataspace', 'someSchemaSet') >> schemaContext
- when: 'saveData method is invoked'
- cpsDataServiceImpl.saveData('someDataspace', 'someAnchor', jsonData, noTimestamp)
- then: 'parameters are validated and processing is delegated to persistence service'
- 1 * mockDataStoreService.storeDataNodes('someDataspace', 'someAnchor', _) >>
- { args -> dataNodeStored = args[2]}
- and: 'the size of the tree is correct'
- def cpsRanInventory = TestUtils.getFlattenMapByXpath(dataNodeStored[0])
- assert cpsRanInventory.size() == 4
- and: 'ran-inventory contains the correct child node'
- def ranInventory = cpsRanInventory.get('/ran-inventory')
- def ranSlices = cpsRanInventory.get('/ran-inventory/ran-slices[@rannfnssiid=\'14559ead-f4fe-4c1c-a94c-8015fad3ea35\']')
- def sliceProfilesList = cpsRanInventory.get('/ran-inventory/ran-slices[@rannfnssiid=\'14559ead-f4fe-4c1c-a94c-8015fad3ea35\']/sliceProfilesList[@sliceProfileId=\'f33a9dd8-ae51-4acf-8073-c9390c25f6f1\']')
- def pLMNIdList = cpsRanInventory.get('/ran-inventory/ran-slices[@rannfnssiid=\'14559ead-f4fe-4c1c-a94c-8015fad3ea35\']/sliceProfilesList[@sliceProfileId=\'f33a9dd8-ae51-4acf-8073-c9390c25f6f1\']/pLMNIdList[@mcc=\'310\' and @mnc=\'410\']')
- ranInventory.getChildDataNodes().size() == 1
- ranInventory.getChildDataNodes().find( {it.xpath == ranSlices.xpath})
- and: 'ranSlices contains the correct child node'
- ranSlices.getChildDataNodes().size() == 1
- ranSlices.getChildDataNodes().find( {it.xpath == sliceProfilesList.xpath})
- and: 'sliceProfilesList contains the correct child node'
- sliceProfilesList.getChildDataNodes().size() == 1
- sliceProfilesList.getChildDataNodes().find( {it.xpath == pLMNIdList.xpath})
- and: 'pLMNIdList contains no children'
- pLMNIdList.getChildDataNodes().size() == 0
-
- }
-
- def 'E2E RAN Schema Model.'(){
- given: 'yang resources'
- def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap(
- 'ietf/ietf-inet-types@2013-07-15.yang',
- 'ietf/ietf-yang-types@2013-07-15.yang',
- 'e2e/basic/cps-ran-schema-model@2021-05-19.yang'
- )
- and : 'json data'
- def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-ran-schema-model-data-v4.json')
- expect: 'schema context is built with no exception indicating the schema set being valid '
- def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext()
- and: 'data is parsed with no exception indicating the model match'
- new YangParserHelper().parseData(ContentType.JSON, jsonData, schemaContext, '', false) != null
- }
-}
+/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021-2024 Nordix Foundation. + * Modifications Copyright (C) 2021-2022 Bell Canada. + * Modifications Copyright (C) 2021 Pantheon.tech + * Modifications Copyright (C) 2022-2024 TechMahindra Ltd. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.impl + +import com.fasterxml.jackson.databind.ObjectMapper +import org.onap.cps.TestUtils +import org.onap.cps.api.CpsAnchorService +import org.onap.cps.api.CpsDeltaService +import org.onap.cps.events.CpsDataUpdateEventsService +import org.onap.cps.utils.CpsValidator +import org.onap.cps.spi.CpsDataPersistenceService +import org.onap.cps.spi.CpsModulePersistenceService +import org.onap.cps.api.model.Anchor +import org.onap.cps.utils.ContentType +import org.onap.cps.utils.JsonObjectMapper +import org.onap.cps.utils.PrefixResolver +import org.onap.cps.utils.YangParser +import org.onap.cps.utils.YangParserHelper +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder +import org.onap.cps.yang.YangTextSchemaSourceSetBuilder +import spock.lang.Specification + +class E2ENetworkSliceSpec extends Specification { + def mockModuleStoreService = Mock(CpsModulePersistenceService) + def mockDataStoreService = Mock(CpsDataPersistenceService) + def mockCpsAnchorService = Mock(CpsAnchorService) + def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache) + def mockCpsValidator = Mock(CpsValidator) + def timedYangTextSchemaSourceSetBuilder = new TimedYangTextSchemaSourceSetBuilder() + def yangParser = new YangParser(new YangParserHelper(), mockYangTextSchemaSourceSetCache, timedYangTextSchemaSourceSetBuilder) + def mockCpsDeltaService = Mock(CpsDeltaService) + def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper()) + def mockPrefixResolver = Mock(PrefixResolver) + + def cpsModuleServiceImpl = new CpsModuleServiceImpl(mockModuleStoreService, + mockYangTextSchemaSourceSetCache, mockCpsAnchorService, mockCpsValidator,timedYangTextSchemaSourceSetBuilder) + + def mockDataUpdateEventsService = Mock(CpsDataUpdateEventsService) + def cpsDataServiceImpl = new CpsDataServiceImpl(mockDataStoreService, mockDataUpdateEventsService, mockCpsAnchorService, mockCpsValidator, + yangParser, mockCpsDeltaService, jsonObjectMapper, mockPrefixResolver) + def dataspaceName = 'someDataspace' + def anchorName = 'someAnchor' + def schemaSetName = 'someSchemaSet' + def noTimestamp = null + + def 'E2E model can be parsed by CPS.'() { + given: 'Valid yang resource as name-to-content map' + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + 'ietf/ietf-inet-types@2013-07-15.yang', + 'ietf/ietf-yang-types@2013-07-15.yang', + 'e2e/basic/ran-network2020-08-06.yang' + ) + when: 'Create schema set method is invoked' + cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + then: 'Parameters are validated and processing is delegated to persistence service' + 1 * mockModuleStoreService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + } + + def 'E2E Coverage Area-Tracking Area & TA-Cell mapping model can be parsed by CPS.'() { + given: 'Valid yang resource as name-to-content map' + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + 'e2e/basic/cps-cavsta-onap-internal2021-01-28.yang') + when: 'Create schema set method is invoked' + cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + then: 'Parameters are validated and processing is delegated to persistence service' + 1 * mockModuleStoreService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + } + + def 'E2E Coverage Area-Tracking Area & TA-Cell mapping data can be parsed by CPS.'() { + given: 'Valid yang resource as name-to-content map' + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + 'e2e/basic/cps-cavsta-onap-internal2021-01-28.yang') + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + def dataNodeStored + and : 'a valid json is provided for the model' + def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-Cavsta-Data.txt') + and : 'all the further dependencies are mocked ' + mockCpsAnchorService.getAnchor(dataspaceName, anchorName) >> + new Anchor().builder().name(anchorName).schemaSetName(schemaSetName).dataspaceName(dataspaceName).build() + mockYangTextSchemaSourceSetCache.get(dataspaceName, schemaSetName) >> + YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + mockModuleStoreService.getYangSchemaResources(dataspaceName, schemaSetName) >> schemaContext + when: 'saveData method is invoked' + cpsDataServiceImpl.saveData(dataspaceName, anchorName, jsonData, noTimestamp) + then: 'Parameters are validated and processing is delegated to persistence service' + 1 * mockDataStoreService.storeDataNodes('someDataspace', 'someAnchor', _) >> + { args -> dataNodeStored = args[2]} + def child = dataNodeStored[0].childDataNodes[0] + assert child.childDataNodes.size() == 1 + and: 'list of Tracking Area for a Coverage Area are stored with correct xpath and child nodes ' + def listOfTAForCoverageArea = child.childDataNodes[0] + listOfTAForCoverageArea.xpath == '/ran-coverage-area/pLMNIdList[@mcc=\'310\' and @mnc=\'410\']/' + + 'coverage-area[@coverageArea=\'Washington\']' + listOfTAForCoverageArea.childDataNodes[0].leaves.get('nRTAC') == 234 + and: 'list of cells in a tracking area are stored with correct xpath and child nodes ' + def listOfCellsInTrackingArea = listOfTAForCoverageArea.childDataNodes[0] + listOfCellsInTrackingArea.xpath == '/ran-coverage-area/pLMNIdList[@mcc=\'310\' and @mnc=\'410\']/' + + 'coverage-area[@coverageArea=\'Washington\']/coverageAreaTAList[@nRTAC=\'234\']' + listOfCellsInTrackingArea.childDataNodes[0].leaves.get('cellLocalId') == 15709 + } + + def 'E2E Coverage Area-Tracking Area & TA-Cell mapping data can be parsed for RAN inventory.'() { + def dataNodeStored + given: 'valid yang resource as name-to-content map' + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + 'e2e/basic/cps-ran-inventory@2021-01-28.yang') + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + and : 'a valid json is provided for the model' + def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-ran-inventory-data.json') + and : 'all the further dependencies are mocked ' + mockCpsAnchorService.getAnchor('someDataspace', 'someAnchor') >> + new Anchor().builder().name('someAnchor').schemaSetName('someSchemaSet').dataspaceName(dataspaceName).build() + mockYangTextSchemaSourceSetCache.get('someDataspace', 'someSchemaSet') >> YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + mockModuleStoreService.getYangSchemaResources('someDataspace', 'someSchemaSet') >> schemaContext + when: 'saveData method is invoked' + cpsDataServiceImpl.saveData('someDataspace', 'someAnchor', jsonData, noTimestamp) + then: 'parameters are validated and processing is delegated to persistence service' + 1 * mockDataStoreService.storeDataNodes('someDataspace', 'someAnchor', _) >> + { args -> dataNodeStored = args[2]} + and: 'the size of the tree is correct' + def cpsRanInventory = TestUtils.getFlattenMapByXpath(dataNodeStored[0]) + assert cpsRanInventory.size() == 4 + and: 'ran-inventory contains the correct child node' + def ranInventory = cpsRanInventory.get('/ran-inventory') + def ranSlices = cpsRanInventory.get('/ran-inventory/ran-slices[@rannfnssiid=\'14559ead-f4fe-4c1c-a94c-8015fad3ea35\']') + def sliceProfilesList = cpsRanInventory.get('/ran-inventory/ran-slices[@rannfnssiid=\'14559ead-f4fe-4c1c-a94c-8015fad3ea35\']/sliceProfilesList[@sliceProfileId=\'f33a9dd8-ae51-4acf-8073-c9390c25f6f1\']') + def pLMNIdList = cpsRanInventory.get('/ran-inventory/ran-slices[@rannfnssiid=\'14559ead-f4fe-4c1c-a94c-8015fad3ea35\']/sliceProfilesList[@sliceProfileId=\'f33a9dd8-ae51-4acf-8073-c9390c25f6f1\']/pLMNIdList[@mcc=\'310\' and @mnc=\'410\']') + ranInventory.getChildDataNodes().size() == 1 + ranInventory.getChildDataNodes().find( {it.xpath == ranSlices.xpath}) + and: 'ranSlices contains the correct child node' + ranSlices.getChildDataNodes().size() == 1 + ranSlices.getChildDataNodes().find( {it.xpath == sliceProfilesList.xpath}) + and: 'sliceProfilesList contains the correct child node' + sliceProfilesList.getChildDataNodes().size() == 1 + sliceProfilesList.getChildDataNodes().find( {it.xpath == pLMNIdList.xpath}) + and: 'pLMNIdList contains no children' + pLMNIdList.getChildDataNodes().size() == 0 + + } + + def 'E2E RAN Schema Model.'(){ + given: 'yang resources' + def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + 'ietf/ietf-inet-types@2013-07-15.yang', + 'ietf/ietf-yang-types@2013-07-15.yang', + 'e2e/basic/cps-ran-schema-model@2021-05-19.yang' + ) + and : 'json data' + def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-ran-schema-model-data-v4.json') + expect: 'schema context is built with no exception indicating the schema set being valid ' + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + and: 'data is parsed with no exception indicating the model match' + new YangParserHelper().parseData(ContentType.JSON, jsonData, schemaContext, '', false) != null + } +} diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/YangTextSchemaSourceSetCacheSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/YangTextSchemaSourceSetCacheSpec.groovy index 189e28521b..6694e7712f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/YangTextSchemaSourceSetCacheSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/YangTextSchemaSourceSetCacheSpec.groovy @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2022 Bell Canada - * Modifications Copyright (C) 2022 Nordix Foundation + * Modifications Copyright (C) 2022-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.impl +package org.onap.cps.impl import org.onap.cps.TestUtils -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsModulePersistenceService import org.onap.cps.yang.YangTextSchemaSourceSet import org.onap.cps.yang.YangTextSchemaSourceSetBuilder @@ -77,8 +77,8 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { assert cachedValue.getModuleReferences() == expectedYangTextSchemaSourceSet.getModuleReferences() and: 'the response is as expected' assert result.getModuleReferences() == expectedYangTextSchemaSourceSet.getModuleReferences() - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('my-dataspace') } def 'Cache Hit: Respond from cache'() { @@ -104,8 +104,8 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { then: 'cached value is same as expected' def cachedValue = getCachedValue('my-dataspace', 'my-schemaset') cachedValue.getModuleReferences() == yangTextSchemaSourceSet.getModuleReferences() - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('my-dataspace') } def 'Cache Evict:with invalid #scenario'() { @@ -119,8 +119,8 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { objectUnderTest.removeFromCache('my-dataspace', 'my-schemaset') then: 'cached does not have value' assert getCachedValue('my-dataspace', 'my-schemaset') == null - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('my-dataspace') } def 'Cache Evict: remove when does not exist'() { @@ -130,8 +130,8 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { objectUnderTest.removeFromCache('my-dataspace', 'my-schemaset') then: 'cached does not have value' assert getCachedValue('my-dataspace', 'my-schemaset') == null - and: 'the CpsValidator is called on the dataspaceName and schemaSetName' - 1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset') + and: 'the CpsValidator is called on the dataspaceName' + 1 * mockCpsValidator.validateNameCharacters('my-dataspace') } def getCachedValue(dataSpace, schemaSet) { @@ -142,5 +142,4 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { return new String("${dataSpace}-${schemaSet}") } - } diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy index bb0f5b0911..6ff41c128f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy @@ -22,7 +22,7 @@ package org.onap.cps.utils -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import spock.lang.Specification class DataMapUtilsSpec extends Specification { diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/PrefixResolverSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/PrefixResolverSpec.groovy index f80678f7d3..3198e54092 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/PrefixResolverSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/PrefixResolverSpec.groovy @@ -23,7 +23,7 @@ package org.onap.cps.utils import org.onap.cps.TestUtils -import org.onap.cps.api.impl.YangTextSchemaSourceSetCache +import org.onap.cps.impl.YangTextSchemaSourceSetCache import org.onap.cps.api.model.Anchor import org.onap.cps.yang.YangTextSchemaSourceSet import org.onap.cps.yang.YangTextSchemaSourceSetBuilder diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy index 62c681eb07..a2fadb7e9f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy @@ -29,7 +29,7 @@ import org.onap.cps.yang.YangTextSchemaSourceSet import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode import org.opendaylight.yangtools.yang.model.api.SchemaContext import spock.lang.Specification -import org.onap.cps.api.impl.YangTextSchemaSourceSetCache +import org.onap.cps.impl.YangTextSchemaSourceSetCache class YangParserSpec extends Specification { diff --git a/docker-compose/config/endurance.env b/docker-compose/config/endurance.env index be337219cf..e46bd5429d 100644 --- a/docker-compose/config/endurance.env +++ b/docker-compose/config/endurance.env @@ -1,6 +1,8 @@ DB_CONTAINER_NAME=endurance-dbpostgresql DB_PORT=5433 +POSTGRES_EXPORTER_PORT_RANGE=9187-9188 + NGINX_CONTAINER_NAME=endurance-nginx-loadbalancer CPS_CORE_PORT=8884 CPS_PORT_RANGE=8798-8799 diff --git a/docker-compose/config/grafana/data-dashboard.json b/docker-compose/config/grafana/data-dashboard.json new file mode 100644 index 0000000000..a75e7a59ea --- /dev/null +++ b/docker-compose/config/grafana/data-dashboard.json @@ -0,0 +1,276 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "NCMP Rest Interfaces", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 13, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "http_server_requests_seconds_count{instance=\"$Instance\", job=\"$Job\", status=~\"201|200|203|204\", uri=~\"/ncmp/v1/ch/id-searches|/ncmp/v1/ch/searches|/ncmp/v1/ch/{cm-handle}/data/ds/{datastore-name}|/ncmp/v1/data\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Inventory API calls by URI and STATUS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum({error=\"OutOfMemoryError\", instance=\"$Instance\", job=\"$Job\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Out of Memory Errors", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [ + "CPS", + "NCMP", + "k6" + ], + "templating": { + "list": [ + { + "current": { + "text": "129.192.80.24:9998", + "value": "129.192.80.24:9998" + }, + "definition": "label_values(instance)", + "label": "Instance", + "name": "Instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "cps-and-ncmp-kpi", + "value": "cps-and-ncmp-kpi" + }, + "definition": "label_values(job)", + "label": "job", + "name": "Job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "2025-01-24T10:09:19.223Z", + "to": "2025-01-24T10:32:39.091Z" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Data REST Interfaces", + "uid": "aeavdgvjyt2iob", + "version": 5, + "weekStart": "" +}
\ No newline at end of file diff --git a/docker-compose/config/grafana/inventory-dashboard.json b/docker-compose/config/grafana/inventory-dashboard.json new file mode 100644 index 0000000000..f6a7e8cbb2 --- /dev/null +++ b/docker-compose/config/grafana/inventory-dashboard.json @@ -0,0 +1,370 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "cmHandlesByState{instance=\"$Instance\", job=\"$Job\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "All LCM States", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "http_server_requests_seconds_count{instance=\"$Instance\", job=\"$Job\", status=~\"201|200\", uri=\"/ncmpInventory/v1/ch\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Inventory API calls by URI and STATUS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum({error=\"OutOfMemoryError\", instance=\"$Instance\", job=\"$Job\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Out of Memory Errors", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [ + "CPS", + "NCMP", + "k6" + ], + "templating": { + "list": [ + { + "current": { + "text": "129.192.80.24:9998", + "value": "129.192.80.24:9998" + }, + "definition": "label_values(instance)", + "label": "Instance", + "name": "Instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "cps-and-ncmp-kpi", + "value": "cps-and-ncmp-kpi" + }, + "definition": "label_values(job)", + "label": "job", + "name": "Job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Inventory REST Interfaces", + "uid": "beao8xrt6qjnkc", + "version": 7, + "weekStart": "" +}
\ No newline at end of file diff --git a/docker-compose/config/grafana/postgresql-statistics-dashboard.json b/docker-compose/config/grafana/postgresql-statistics-dashboard.json new file mode 100644 index 0000000000..642294167e --- /dev/null +++ b/docker-compose/config/grafana/postgresql-statistics-dashboard.json @@ -0,0 +1,2245 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for PostgreSQL Statistics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "title": "Settings", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_static{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 54, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_connections{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max Connections", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 1 + }, + "id": 56, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Shared Buffers", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 1 + }, + "id": 58, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Effective Cache", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 1 + }, + "id": 60, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Maintenance Work Mem", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 66, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Work Mem", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 32, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max WAL Size", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 62, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Random Page Cost", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 70, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_seq_page_cost{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Seq Page Cost", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 4 + }, + "id": 64, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max Worker Processes", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 4 + }, + "id": 68, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max Parallel Workers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 72, + "panels": [], + "title": "Database", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 72, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "editorMode": "code", + "expr": "pg_database_size_bytes{instance=\"$instance\", datname=\"$db\"}", + "legendFormat": "size", + "range": true, + "refId": "A" + } + ], + "title": "Size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 36, + "panels": [], + "title": "Connection / Transaction Statistics", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_stat_activity_count{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "commits", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "rollbacks", + "refId": "B" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rows", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_fetched{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SELECT (index scan)", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_returned{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SELECT (table scan)", + "refId": "B" + } + ], + "title": "Read Stats", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rows", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_inserted{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "INSERT", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_updated{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UPDATE", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_deleted{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DELETE", + "refId": "C" + } + ], + "title": "Change Stats", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_stat_activity_max_tx_duration{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max_tx_duration [{{state}}]", + "refId": "A" + } + ], + "title": "Longest Transaction", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=\"$db\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=\"$db\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=\"$db\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Cache Hit Rate", + "refId": "A" + } + ], + "title": "Cache Hit Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 50, + "panels": [], + "title": "misc", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 0, + "y": 44 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_backend_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_backend", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_alloc_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_alloc", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "backend_fsync", + "refId": "C" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_checkpoint", + "refId": "D" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_clean_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_clean", + "refId": "E" + } + ], + "title": "Buffers (bgwriter)", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 44 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "conflicts", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "deadlocks", + "refId": "A" + } + ], + "title": "Conflicts/Deadlocks", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 17, + "x": 0, + "y": 50 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_locks_count{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "title": "Lock Tables", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 50 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Temp Bytes", + "refId": "A" + } + ], + "title": "Temp File", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", + "refId": "A" + } + ], + "title": "Checkpoint Stats", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(pg_up, instance)", + "includeAll": false, + "label": "instance", + "name": "instance", + "options": [], + "query": { + "query": "label_values(pg_up, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "cpsdb", + "value": "cpsdb" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(pg_stat_database_tup_fetched{datname!~\"template.*|postgres\",instance=\"$instance\"},datname)", + "includeAll": false, + "label": "Database", + "name": "db", + "options": [], + "query": { + "query": "label_values(pg_stat_database_tup_fetched{datname!~\"template.*|postgres\",instance=\"$instance\"},datname)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PostgreSQL Statistics", + "uid": "OpKZVIAMz", + "version": 1, + "weekStart": "" +}
\ No newline at end of file diff --git a/docker-compose/config/prometheus.yml b/docker-compose/config/prometheus.yml index 8db07c8d05..1beea6ebce 100644 --- a/docker-compose/config/prometheus.yml +++ b/docker-compose/config/prometheus.yml @@ -20,4 +20,10 @@ scrape_configs: static_configs: - targets: - '172.17.0.1:8798' - - '172.17.0.1:8799'
\ No newline at end of file + - '172.17.0.1:8799' + +- job_name: 'postgres' + static_configs: + - targets: + - '172.17.0.1:9187' + - '172.17.0.1:9188'
\ No newline at end of file diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 1e3b9ab96f..e1c8c9793c 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # Copyright (c) 2020 Pantheon.tech. # Modifications Copyright (C) 2021 Bell Canada. -# Modifications Copyright (C) 2022-2024 Nordix Foundation. +# Modifications Copyright (C) 2022-2025 Nordix Foundation. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -51,6 +51,8 @@ services: image: ${DOCKER_REPO:-nexus3.onap.org:10003}/onap/cps-and-ncmp:${CPS_VERSION:-latest} ports: - ${CPS_PORT_RANGE:-8698-8699}:8080 + ### DEBUG: Uncomment next line to enable java debugging (ensure 'ports' aligns with 'deploy') + ### - ${CPS_CORE_DEBUG_PORT:-5005}:5005- environment: CPS_USERNAME: ${CPS_CORE_USERNAME:-cpsuser} CPS_PASSWORD: ${CPS_CORE_PASSWORD:-cpsr0cks!} @@ -66,9 +68,10 @@ services: ONAP_OTEL_EXPORTER_ENDPOINT: http://jaeger-service:4317 POLICY_SERVICE_ENABLED: 'false' POLICY_SERVICE_DEFAULT_DECISION: 'deny from env' - JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=75.0 -XX:MaxRAMPercentage=75.0" + CPS_MONITORING_MICROMETER_JVM_EXTRAS: 'true' + JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0" ### DEBUG: Uncomment next line to enable java debugging - ### DEBUG: JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + ### JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 restart: unless-stopped depends_on: - dbpostgresql @@ -78,10 +81,7 @@ services: resources: limits: cpus: '3' - memory: 2G - ### DEBUG: Uncomment next 2 lines to enable java debugging (ensure 'ports' aligns with 'deploy') - ### DEBUG ports: - ### DEBUG - ${CPS_CORE_DEBUG_PORT:-5005}:5005 + memory: 3G nginx: container_name: ${NGINX_CONTAINER_NAME:-nginx-loadbalancer} @@ -151,7 +151,7 @@ services: KAFKA_BOOTSTRAP_SERVER: kafka:29092 NCMP_CONSUMER_GROUP_ID: ncmp-group NCMP_ASYNC_M2M_TOPIC: ncmp-async-m2m - MODULE_INITIAL_PROCESSING_DELAY_MS: 120000 + MODULE_INITIAL_PROCESSING_DELAY_MS: 180000 MODULE_REFERENCES_DELAY_MS: 100 MODULE_RESOURCES_DELAY_MS: 1000 READ_DATA_FOR_CM_HANDLE_DELAY_MS: 300 @@ -200,6 +200,9 @@ services: volumes: - ./config/grafana/provisioning/:/etc/grafana/provisioning/ - ./config/grafana/jvm-micrometer-dashboard.json:/var/lib/grafana/dashboards/jvm-micrometer-dashboard.json + - ./config/grafana/inventory-dashboard.json:/var/lib/grafana/dashboards/inventory-dashboard.json + - ./config/grafana/data-dashboard.json:/var/lib/grafana/dashboards/data-dashboard.json + - ./config/grafana/postgresql-statistics-dashboard.json:/var/lib/grafana/dashboards/postgresql-statistics-dashboard.json - grafana:/var/lib/grafana environment: - GF_SECURITY_ADMIN_PASSWORD=admin @@ -228,6 +231,17 @@ services: profiles: - tracing + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter + environment: + - DATA_SOURCE_NAME=postgresql://${DB_USERNAME:-cps}:${DB_PASSWORD:-cps}@${DB_CONTAINER_NAME:-dbpostgresql}:5432/postgres?sslmode=disable + ports: + - ${POSTGRES_EXPORTER_PORT_RANGE:-9187-9188}:9187 + depends_on: + - dbpostgresql + profiles: + - monitoring + volumes: grafana: driver: local diff --git a/docs/admin-guide.rst b/docs/admin-guide.rst index 4a40f9b29c..9009de21c9 100644 --- a/docs/admin-guide.rst +++ b/docs/admin-guide.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2022 Nordix Foundation +.. Copyright (C) 2021-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _adminGuide: @@ -12,6 +12,40 @@ CPS Admin Guide .. toctree:: :maxdepth: 1 +Regular Maintenance +=================== +This section details tasks that an administrator of the CPS application should execute on regular basis +to ensure optimum working of CPS. + +Dataspace Clean Up +------------------ +Certain data in the CPS database might not be explicitly removed after it is no longer required ('orphaned data'). +For example, schema sets and their associated unique module resources no longer used by any anchor because of model upgrades. +This data would unnecessarily take up space and could eventually affect the performance of the DB if it is not deleted. +How often this needs to be done depends on how often schema sets are being deprecated. +Typically once per month should suffice. + +To remove orphaned data in a given dataspace use the following post request: + +.. code:: + + http://<cps-component-service-name>:<cps-port>/v2/admin/dataspaces/<dataspace-name>/actions/clean + +for example + +.. code-block:: bash + + curl --location --request POST 'http://cps:8080/admin/datsaspaces/bookstore/actions/clean' \ + --header 'Content-Type: application/json; charset=utf-8' + + Response : HTTP Status 204 + +For more details refer to the CPS-Core API: :doc:`design`. + +.. note:: + NCMP has no specific maintenance tasks but it will also build up orphaned data when CM Handles get updated and or deleted. + To delete this data execute the above procedure for the dataspace named 'NFP-Operational'. + Logging Configuration ===================== @@ -192,17 +226,15 @@ Prometheus Metrics can be checked at the following endpoint http://<cps-component-service-name>:8080/actuator/prometheus -Hazelcast ---------- +Heap Dump and Thread Dump +------------------------- -Hazelcast cluster state and health check can be seen using the below endpoints +On demand heap dump and thread dump generation using the below endpoints. .. code:: - http://<cps-component-service-name>:<member-port>/hazelcast/health - http://<cps-component-service-name>:<member-port>/hazelcast/rest/management/cluster/state - -See also : :ref:`cps_common_distributed_datastructures` + http://<cps-component-service-name>:8080/actuator/heapdump + http://<cps-component-service-name>:8080/actuator/threaddump Naming Validation ----------------- @@ -210,9 +242,8 @@ Naming Validation As part of the Kohn 3.1.0 release, CPS has added validation to the names of the following components: - Dataspace names - - Schema Set names - Anchor names - - Cm-Handle identifiers + - CM Handle identifiers The following characters along with spaces are no longer valid for naming of these components. diff --git a/docs/api/swagger/cps/openapi.yaml b/docs/api/swagger/cps/openapi.yaml index c84609b638..f6baadc8ea 100644 --- a/docs/api/swagger/cps/openapi.yaml +++ b/docs/api/swagger/cps/openapi.yaml @@ -330,6 +330,65 @@ paths: summary: Get a dataspace tags: - cps-admin + /{apiVersion}/admin/dataspaces/{dataspace-name}/actions/clean: + post: + description: Clean the dataspace (remove orphaned schema sets and modules) + operationId: cleanDataspace + parameters: + - description: apiVersion + in: path + name: apiVersion + required: true + schema: + default: v2 + enum: + - v1 + - v2 + type: string + - description: dataspace-name + in: path + name: dataspace-name + required: true + schema: + example: my-dataspace + type: string + responses: + "204": + content: {} + description: No Content + "400": + content: + application/json: + example: + status: 400 + message: Bad Request + details: The provided request is not valid + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Bad Request + "403": + content: + application/json: + example: + status: 403 + message: Request Forbidden + details: This request is forbidden + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Forbidden + "500": + content: + application/json: + example: + status: 500 + message: Internal Server Error + details: Internal Server Error occurred + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Internal Server Error + summary: Clean the dataspace + tags: + - cps-admin /v1/dataspaces/{dataspace-name}/anchors: post: deprecated: true diff --git a/docs/api/swagger/ncmp/openapi.yaml b/docs/api/swagger/ncmp/openapi.yaml index 4e369f5dce..692266183b 100644 --- a/docs/api/swagger/ncmp/openapi.yaml +++ b/docs/api/swagger/ncmp/openapi.yaml @@ -38,7 +38,7 @@ paths: examples: sample 1: value: - resourceIdentifier: "\GNBDUFunction" + resourceIdentifier: \GNBDUFunction sample 2: value: resourceIdentifier: "\\GNBDUFunction[@gNBId='1001']" @@ -155,7 +155,7 @@ paths: examples: sample 1: value: - resourceIdentifier: "\GNBDUFunction" + resourceIdentifier: \GNBDUFunction sample 2: value: resourceIdentifier: "\\GNBDUFunction[@gNBId='1001']" @@ -299,7 +299,7 @@ paths: examples: sample 1: value: - resourceIdentifier: "\GNBDUFunction" + resourceIdentifier: \GNBDUFunction sample 2: value: resourceIdentifier: "\\GNBDUFunction[@gNBId='1001']" @@ -419,7 +419,7 @@ paths: examples: sample 1: value: - resourceIdentifier: "\GNBDUFunction" + resourceIdentifier: \GNBDUFunction sample 2: value: resourceIdentifier: "\\GNBDUFunction[@gNBId='1001']" @@ -544,7 +544,7 @@ paths: examples: sample 1: value: - resourceIdentifier: "\GNBDUFunction" + resourceIdentifier: \GNBDUFunction sample 2: value: resourceIdentifier: "\\GNBDUFunction[@gNBId='1001']" @@ -1542,7 +1542,7 @@ components: examples: sample 1: value: - resourceIdentifier: "\GNBDUFunction" + resourceIdentifier: \GNBDUFunction sample 2: value: resourceIdentifier: "\\GNBDUFunction[@gNBId='1001']" @@ -1872,19 +1872,13 @@ components: RestModuleDefinition: example: moduleName: my-module-name - content: | - module _3gpp-nr-nrm-gnbdufunction { - yang-version 1.1; - namespace 'urn:3gpp:sa5:_3gpp-nr-nrm-gnbdufunction'; - prefix gnbdu3gpp; - revision '2020-09-15' { - description - 'Defines the YANG mapping of the GNBDUFunction Information - Object Class (IOC) that is part of the NR Network Resource Model (NRM). - Copyright 2024, 3GPP Organizational Partners (ARIB, ATIS, CCSA, ETSI, TSDSI, - TTA, TTC). All rights reserved.'; - } - } + content: "module _3gpp-nr-nrm-gnbdufunction {\n yang-version 1.1;\n namespace\ + \ 'urn:3gpp:sa5:_3gpp-nr-nrm-gnbdufunction';\n prefix gnbdu3gpp;\n revision\ + \ '2020-09-15' {\n description\n 'Defines the YANG mapping of the\ + \ GNBDUFunction Information \n Object Class (IOC) that is part of the\ + \ NR Network Resource Model (NRM). \n Copyright 2024, 3GPP Organizational\ + \ Partners (ARIB, ATIS, CCSA, ETSI, TSDSI,\n TTA, TTC). All rights reserved.';\n\ + \ }\n}\n" revision: 2020-09-15 properties: moduleName: @@ -1894,17 +1888,13 @@ components: example: 2020-09-15 type: string content: - example: | - module _3gpp-nr-nrm-gnbdufunction { - yang-version 1.1; - namespace 'urn:3gpp:sa5:_3gpp-nr-nrm-gnbdufunction'; - prefix gnbdu3gpp; - revision '2020-09-15' { - description - 'Defines the YANG mapping of the GNBDUFunction Information - Object Class (IOC) that is part of the NR Network Resource Model (NRM). - Copyright 2024, 3GPP Organizational Partners (ARIB, ATIS, CCSA, ETSI, TSDSI, - TTA, TTC). All rights reserved.'; + example: "module _3gpp-nr-nrm-gnbdufunction {\n yang-version 1.1;\n namespace\ + \ 'urn:3gpp:sa5:_3gpp-nr-nrm-gnbdufunction';\n prefix gnbdu3gpp;\n revision\ + \ '2020-09-15' {\n description\n 'Defines the YANG mapping of the\ + \ GNBDUFunction Information \n Object Class (IOC) that is part of\ + \ the NR Network Resource Model (NRM). \n Copyright 2024, 3GPP Organizational\ + \ Partners (ARIB, ATIS, CCSA, ETSI, TSDSI,\n TTA, TTC). All rights\ + \ reserved.';\n }\n}\n" type: string title: Module definitions type: object diff --git a/docs/cm-handle-lcm-events.rst b/docs/cm-handle-lcm-events.rst index 8446834c31..38339e206a 100644 --- a/docs/cm-handle-lcm-events.rst +++ b/docs/cm-handle-lcm-events.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmHandleLcmEvents: @@ -15,7 +15,7 @@ CM Handle Lifecycle Management (LCM) Events Introduction ============ -LCM events for CM Handles are published when a CM Handle is created, deleted or another change in the cm handle state occurs. +LCM events for CM Handles are published when a CM Handle is created, deleted or another change in the CM Handle state occurs. **3 possible event types:** @@ -55,7 +55,7 @@ Event payload varies based on the type of event. **CREATE** -Event payload for this event contains the properties of the new cm handle created. +Event payload for this event contains the properties of the new CM Handle created. *Create event payload prototype* @@ -77,7 +77,7 @@ Event payload for this event contains the properties of the new cm handle create **UPDATE** -Event payload for this event contains the difference in state and properties of the cm handle. +Event payload for this event contains the difference in state and properties of the CM Handle. *Update event payload prototype* @@ -106,7 +106,7 @@ Event payload for this event contains the difference in state and properties of **DELETE** -Event payload for this event contains the identifier of the deleted cm handle. +Event payload for this event contains the identifier of the deleted CM Handle. *Delete event payload prototype* @@ -114,4 +114,4 @@ Event payload for this event contains the identifier of the deleted cm handle. "event": { "cmHandleId" : "cmhandle-001", - }
\ No newline at end of file + } diff --git a/docs/cps-events.rst b/docs/cps-events.rst index 47aa73f12e..1097af9a12 100644 --- a/docs/cps-events.rst +++ b/docs/cps-events.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2022-2023 Nordix Foundation +.. Copyright (C) 2022-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cpsEvents: @@ -16,9 +16,9 @@ CPS Events cm-notification-subscriptions.rst .. note:: - Legacy async response on a client supplied topic for single cm handle data request are no longer supported. Click link below for the legacy specification. + Legacy async response on a client supplied topic for single CM Handle data request are no longer supported. Click link below for the legacy specification. .. toctree:: :maxdepth: 0 - ncmp-async-events.rst
\ No newline at end of file + ncmp-async-events.rst diff --git a/docs/cps-ncmp-message-status-codes.rst b/docs/cps-ncmp-message-status-codes.rst index e0a3f0308b..799838ae44 100644 --- a/docs/cps-ncmp-message-status-codes.rst +++ b/docs/cps-ncmp-message-status-codes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023-2024 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _dataOperationMessageStatusCodes: @@ -16,9 +16,9 @@ CPS-NCMP Message Status Codes +-----------------+------------------------------------------------------+-----------------------------------+ | 1 | ACCEPTED | CM Data Notification Subscription | +-----------------+------------------------------------------------------+-----------------------------------+ - | 100 | cm handle id(s) is(are) not found | All features | + | 100 | CM Handle id(s) is(are) not found | All features | +-----------------+------------------------------------------------------+-----------------------------------+ - | 101 | cm handle(s) not ready | Data Operation | + | 101 | CM Handle(s) not ready | Data Operation | +-----------------+------------------------------------------------------+-----------------------------------+ | 102 | dmi plugin service is not responding | Data Operation | +-----------------+------------------------------------------------------+-----------------------------------+ @@ -30,9 +30,9 @@ CPS-NCMP Message Status Codes +-----------------+------------------------------------------------------+-----------------------------------+ | 108 | Unknown error | All features | +-----------------+------------------------------------------------------+-----------------------------------+ - | 109 | cm-handle already exists | Inventory | + | 109 | CM Handle already exists | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ - | 110 | cm-handle has an invalid character(s) in id | Inventory | + | 110 | CM Handle has an invalid character(s) in id | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ | 111 | alternate id already associated | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ diff --git a/docs/cps-scheduled-processes.rst b/docs/cps-scheduled-processes.rst index c204e6ca0a..9af9a81c0c 100644 --- a/docs/cps-scheduled-processes.rst +++ b/docs/cps-scheduled-processes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2022 Nordix Foundation +.. Copyright (C) 2022-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING @@ -21,17 +21,17 @@ The following section is a list of the current scheduled processes running withi Module Sync ----------- The module sync is a user :ref:`configurable timed process<additional-cps-ncmp-customizations>`, -which is set to search for CM-Handles within CPS with an *'ADVISED'* state. -Once the CM-Handle is processed by the module sync, the CM-Handle state is then set to *'READY'*, if the process completes successfully. -If for any reason the module sync fails, the CM-Handle state will then be set to *'LOCKED'*, +which is set to search for CM Handles within CPS with an *'ADVISED'* state. +Once the CM Handle is processed by the module sync, the CM Handle state is then set to *'READY'*, if the process completes successfully. +If for any reason the module sync fails, the CM Handle state will then be set to *'LOCKED'*, and the reason for the lock will also be stored within CPS. -CM-Handles in the *'LOCKED'* state will be retried when the system has availability. CM-Handles in a *'LOCKED'* -state are processed by the retry mechanism, by setting CM-Handle state back to *'ADVISED'* so the next sync cycle will process those again. +CM Handles in the *'LOCKED'* state will be retried when the system has availability. CM Handles in a *'LOCKED'* +state are processed by the retry mechanism, by setting CM Handle state back to *'ADVISED'* so the next sync cycle will process those again. Data Sync --------- The data sync is a user :ref:`configurable timed process<additional-cps-ncmp-customizations>`, -which is set to search for CM-Handles with a sync state of *'UNSYNCHRONIZED'*. -Once the CM-Handle(s) with a sync state of *'UNSYNCHRONIZED'* is processed by the data sync, -the CM-Handle sync state is then set to *'SYNCHRONIZED'*, if the process completes successfully. -If the data sync fails, the CM-Handle sync state will remain as *'UNSYNCHRONIZED'*, and will be re-attempted. +which is set to search for CM Handles with a sync state of *'UNSYNCHRONIZED'*. +Once the CM Handle(s) with a sync state of *'UNSYNCHRONIZED'* is processed by the data sync, +the CM Handle sync state is then set to *'SYNCHRONIZED'*, if the process completes successfully. +If the data sync fails, the CM Handle sync state will remain as *'UNSYNCHRONIZED'*, and will be re-attempted. diff --git a/docs/deployment.rst b/docs/deployment.rst index 2a17e30a16..2af0dd0cd5 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2024 Nordix Foundation +.. Copyright (C) 2021-2025 Nordix Foundation .. Modifications Copyright (C) 2021 Bell Canada. .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING @@ -191,21 +191,16 @@ Any spring supported property can be configured by providing in ``config.additio | Property | Description | Default Value | +===========================================+=========================================================================================================+===============================+ | config.appUserName | User name used by cps-core service to configure the authentication for REST API it exposes. | ``cpsuser`` | -| | | | | | This is the user name to be used by cps-core REST clients to authenticate themselves. | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.appUserPassword | Password used by cps-core service to configure the authentication for REST API it exposes. | Not defined | -| | | | | | If not defined, the password is generated when deploying the application. | | -| | | | | | See also :ref:`cps_common_credentials_retrieval`. | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | postgres.config.pgUserName | Internal user name used by cps-core to connect to its own database. | ``cps`` | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | postgres.config.pgUserPassword | Internal password used by cps-core to connect to its own database. | Not defined | -| | | | | | If not defined, the password is generated when deploying the application. | | -| | | | | | See also :ref:`cps_common_credentials_retrieval`. | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | postgres.config.pgDatabase | Database name used by cps-core | ``cpsdb`` | @@ -225,28 +220,24 @@ Any spring supported property can be configured by providing in ``config.additio +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security protocol. | ``SASL_PLAINTEXT`` | | spring.kafka.security.protocol | Some possible values are: | | -| | | | | | * ``PLAINTEXT`` | | | | * ``SASL_PLAINTEXT``, for authentication | | | | * ``SASL_SSL``, for authentication and encryption | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL mechanism. Required for SASL_PLAINTEXT and SASL_SSL protocols. | Not defined | | spring.kafka.properties. | Some possible values are: | | -| sasl.mechanism | | | -| | * ``PLAIN``, for PLAINTEXT | | +| sasl.mechanism | * ``PLAIN``, for PLAINTEXT | | | | * ``SCRAM-SHA-512``, for SSL | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL JAAS configuration. Required for SASL_PLAINTEXT and SASL_SSL protocols. | Not defined | | spring.kafka.properties. | Some possible values are: | | -| sasl.jaas.config | | | -| | * ``org.apache.kafka.common.security.plain.PlainLoginModule required username="..." password="...";``, | | -| | for PLAINTEXT | | +| sasl.jaas.config | * ``org.apache.kafka.common.security.plain.PlainLoginModule required username="..." password="...";``, | | +| | for PLAINTEXT | | | | * ``org.apache.kafka.common.security.scram.ScramLoginModule required username="..." password="...";``, | | -| | for SSL | | +| | for SSL | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL SSL store type. Required for SASL_SSL protocol. | Not defined | | spring.kafka.ssl.trust-store-type | Some possible values are: | | -| | | | | | * ``JKS`` | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL SSL store file location. Required for SASL_SSL protocol. | Not defined | @@ -293,35 +284,84 @@ Any spring supported property can be configured by providing in ``config.additio Additional CPS-NCMP Customizations ================================== -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.dmiPluginUserName | User name used by cps-core to authenticate themselves for using ncmp-dmi-plugin service. | ``dmiuser`` | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.dmiPluginUserPassword | Internal password used by cps-core to connect to ncmp-dmi-plugin service. | Not defined | -| | | | -| | If not defined, the password is generated when deploying the application. | | -| | | | -| | See also :ref:`cps_common_credentials_retrieval`. | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.ncmp.timers | Specifies the delay in milliseconds in which the module sync watch dog will wake again after finishing. | ``5000`` | -| .advised-modules-sync.sleep-time-ms | | | -| | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.ncmp.timers | Specifies the delay in milliseconds in which the data sync watch dog will wake again after finishing. | ``30000`` | -| .cm-handle-data-sync.sleep-time-ms | | | -| | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum time in seconds, to wait for establishing a connection for the HTTP Client. | ``30`` | -| .connectionTimeoutInSeconds | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum number of connections allowed per route in the HTTP client. | ``50`` | -| .maximumConnectionsPerRoute | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum total number of connections that can be held by the HTTP client. | ``100`` | -| .maximumConnectionsTotal | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the duration in seconds for the threshold, after which idle connections will be evicted | ``5`` | -| .idleConnectionEvictionThresholdInSeconds | from the connection pool by the HTTP client. | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ + ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| Property | Description | Default Value | ++=================================================+=======================================================================================+=================================+ +| config.dmiPluginUserName | User name used by cps-core to authenticate themselves for using ncmp-dmi-plugin | ``dmiuser`` | +| | service. | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.dmiPluginUserPassword | Internal password used by cps-core to connect to ncmp-dmi-plugin service. | Not defined | +| | If not defined, the password is generated when deploying the application. | | +| | See also :ref:`cps_common_credentials_retrieval`. | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.ncmp.timers | Specifies the delay in milliseconds in which the module sync watch dog will wake again| ``5000`` | +| .advised-modules-sync.sleep-time-ms | after finishing. | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.ncmp.timers | Specifies the delay in milliseconds in which the data sync watch dog will wake again | ``30000`` | +| .cm-handle-data-sync.sleep-time-ms | after finishing. | | +| | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Maximum size (in MB) of the in-memory buffer for HTTP response data. | ``16`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .maximumInMemorySizeInMegabytes | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Maximum number of simultaneous connections allowed in the connection pool. | ``100`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .maximumConnectionsTotal | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Maximum number of pending requests when the connection pool is full. | ``50`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .pendingAcquireMaxCount | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Specifies the maximum time in seconds, to wait for establishing a connection for the | ``30`` | +| .[app] | HTTP Client. | | +| .httpclient | | | +| .[services] | | | +| .connectionTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Timeout (in seconds) for reading data from the server after the connection is | ``30`` | +| .[app] | established. | | +| .httpclient | | | +| .[services] | | | +| .readTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Timeout (in seconds) for writing data to the server. | ``30`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .writeTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Total timeout (in seconds) for receiving a complete response, including all processing| ``60`` | +| .[app] | stages. | | +| .httpclient | | | +| .[services] | | | +| .responseTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | Enables or disables the policy-executor feature. | ``false`` | +| .enabled | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | The default (fallback) decision in case a problem with the external service occurs. | ``allow`` | +| .defaultDecision | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | The server address for the external policy executor service. | ``http://policy-executor-stub`` | +| .server.address | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | The port used for the external policy executor service. | ``8093`` | +| .server.port | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ + +.. note:: + + - [app] : can be 'policy-executor' or 'dmi'. + - [services] 'all-services' for 'policy-executor'. + - [services] 'data-services' and 'model-services' for 'dmi'. CPS-Core Docker Installation ============================ @@ -342,22 +382,19 @@ Below are the list of distributed datastructures that we have. +--------------+------------------------------------+-----------------------------------------------------------+ | Component | Data Structure Name | Use | +==============+====================================+===========================================================+ -| cps-ncmp | moduleSyncStartedOnCmHandles | Watchdog process to register cm handles. | +| cps-ncmp | moduleSyncStartedOnCmHandles | Watchdog process to register CM Handles. | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | dataSyncSemaphores | Watchdog process to sync data from the nodes. | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | moduleSyncWorkQueue | Queue used internally for workers to pick the task. | +--------------+------------------------------------+-----------------------------------------------------------+ -| cps-ncmp | trustLevelPerCmHandle | Stores the trust level per cm handle id | +| cps-ncmp | trustLevelPerCmHandle | Stores the trust level per CM Handle id | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | trustLevelPerDmiPlugin | Stores the trust level for the dmi-plugins. | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | cmNotificationSubscriptionCache | Stores and tracks cm notification subscription requests. | +--------------+------------------------------------+-----------------------------------------------------------+ -| cps-ncmp | moduleSetTagsBeingProcessed | Track module set tags which are processed to prevent | -| | | multiple threads working with same tag. | -+--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | cpsAndNcmpLock | Cps and NCMP distributed lock for various use cases. | +--------------+------------------------------------+-----------------------------------------------------------+ -Total number of caches : 8 +Total number of caches : 7 diff --git a/docs/modeling.rst b/docs/modeling.rst index 7ebf6fecd5..65e4aa97ca 100644 --- a/docs/modeling.rst +++ b/docs/modeling.rst @@ -1,7 +1,7 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 .. Copyright (C) 2021 Pantheon.tech -.. Modifications Copyright (C) 2021-2023 Nordix Foundation +.. Modifications Copyright (C) 2021-2025 Nordix Foundation .. _modeling: .. toctree:: @@ -89,31 +89,31 @@ Note: Although additional-properties are present in the model of the dmi-registr Basic Concepts -------------- -- **CM-Handle** represents an instance a modeled Network Function(node) in ONAP. +- **CM Handle** represents an instance a modeled Network Function(node) in ONAP. These are stored as Anchors within CPS-Core. - - **CM-Handle States** are used to represent the potential states in which a CM-Handle can transition between. + - **CM Handle States** are used to represent the potential states in which a CM Handle can transition between. - The 5 possible CM-Handle states are: ADVISED, READY, LOCKED, DELETING, DELETED + The 5 possible CM Handle states are: ADVISED, READY, LOCKED, DELETING, DELETED - **ADVISED** indicates that a CM-Handle has been registered successfully, and is waiting for the module synchronization process to sync the CM-Handle. + **ADVISED** indicates that a CM Handle has been registered successfully, and is waiting for the module synchronization process to sync the CM Handle. - **READY** indicates that the CM-Handle has been synced successfully. + **READY** indicates that the CM Handle has been synced successfully. - **LOCKED** indicates that the CM-Handle has not synced successfully. A retry mechanism within CPS will set the state back to ADVISED after a set time. + **LOCKED** indicates that the CM Handle has not synced successfully. A retry mechanism within CPS will set the state back to ADVISED after a set time. - **DELETING** indicates that the CM-Handle is currently being deleted. + **DELETING** indicates that the CM Handle is currently being deleted. - **DELETED** indicates that the CM-Handle has been deleted successfully. + **DELETED** indicates that the CM Handle has been deleted successfully. - - **Data-sync state** is the state of the data synchronization process of the CM-Handle + - **Data-sync state** is the state of the data synchronization process of the CM Handle There are 3 possibles states: NONE_REQUESTED, UNSYNCHRONIZED, SYNCHRONIZED **NONE_REQUESTED** indicates that the data sync is not requested by the user - **UNSYNCHRONIZED** indicates the cm-handle is waiting for the data sync watchdog operation to carry out the sync process + **UNSYNCHRONIZED** indicates the CM Handle is waiting for the data sync watchdog operation to carry out the sync process **SYNCHRONIZED** indicates the watchdog process has finished the data synchronization successfully diff --git a/docs/ncmp-data-operation.rst b/docs/ncmp-data-operation.rst index 10c3bfaca5..e0b7bb88cd 100644 --- a/docs/ncmp-data-operation.rst +++ b/docs/ncmp-data-operation.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023-2024 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmHandleDataOperation: @@ -15,7 +15,7 @@ Data Operations Endpoint Introduction ============ -For all data operations on cm handle(s), we have a post endpoint: +For all data operations on CM Handle(s), we have a post endpoint: - /ncmp/v1/data?topic={client-topic-name} forward request to it's dmi plugin service. @@ -50,7 +50,7 @@ This endpoint executes data operation for given array of operations: | | | implementation. For ONAP DMI Plugin it will be RESTConf paths but it can| | | | really be anything. | +--------------------------+-------------+-------------------------------------------------------------------------+ - | targetIds | Yes | List of cm handle references | + | targetIds | Yes | List of CM Handle references | +--------------------------+-------------+-------------------------------------------------------------------------+ The status codes used in the events resulting from these operations are defined here: @@ -156,4 +156,4 @@ DMI Service 2 (POST) : `http://{dmi-host-name}:{dmi-port}/dmi/v1/data?topic=my-t Above examples are for illustration purposes only. Please refer to link below for latest schema. -:download:`Data operation event schema <schemas/data-operation-event-schema-1.0.0.json>`
\ No newline at end of file +:download:`Data operation event schema <schemas/data-operation-event-schema-1.0.0.json>` diff --git a/docs/overview.rst b/docs/overview.rst index 19ab8b4847..bc99214e2e 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -1,6 +1,8 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021 Pantheon.tech, Nordix Foundation +.. Copyright (C) 2021 Pantheon.tech +.. Modifications Copyright (C) 2021-2025 Nordix Foundation + .. _overview: CPS Overview @@ -45,7 +47,7 @@ even though CPS-Core could be deployed without the NCMP extension. NCMP-DMI-Plugin --------------- -The Data-Model-Inventory (DMI) Plugin is a rest interface used to synchronize CM-Handles data between CPS and DMI through the DMI-Plugin. +The Data-Model-Inventory (DMI) Plugin is a rest interface used to synchronize CM Handles data between CPS and DMI through the DMI-Plugin. This is built previously from the CPS-NF-Proxy component. CPS Project diff --git a/docs/policy-executor.rst b/docs/policy-executor.rst index b934a579b1..712b4fcf38 100644 --- a/docs/policy-executor.rst +++ b/docs/policy-executor.rst @@ -1,11 +1,10 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2024 Nordix Foundation +.. Copyright (C) 2024-2025 Nordix Foundation -.. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING +.. DO NOT CHANGE THIS LABEL - EVEN THOUGH IT GIVES A WARNING .. _policy_executor: - Policy Executor ############### @@ -15,7 +14,16 @@ Policy Executor Introduction ============ -Work In Progress: This feature is not yet completed and does not affect current NCMP functionality. +The Policy Executor feature can be used to connect an external system to make decisions on CM write operation. +When the feature is enabled, NCMP will first call the configured external system and depending on the response, return an error or continue. +The details of the interface can be found in the ':ref:`policy_executor_consumed_apis`' section. + +This feature is available on 'legacy data interface' for operation on a single cm handle: "/v1/ch/{cm-handle}/data/ds/{datastore-name}" and only applies to "ncmp-datastore:passthrough-running". + +By default, the feature is not enabled. This is controlled by 'config.additional.ncmp.policy-executor.enabled' and other deployment parameters in the same group to enable it. See :ref:`additional-cps-ncmp-customizations` + +.. DO NOT CHANGE THIS LABEL - EVEN THOUGH IT GIVES A WARNING +.. _policy_executor_consumed_apis: Consumed APIs ------------- diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 9c825e4d35..d2ed1b8044 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2024 Nordix Foundation +.. Copyright (C) 2021-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _release_notes: @@ -15,6 +15,29 @@ CPS Release Notes .. * * * PARIS * * * .. ==================== +Version: 3.6.1 +============== + +Release Data +------------ + ++--------------------------------------+--------------------------------------------------------+ +| **CPS Project** | | +| | | ++--------------------------------------+--------------------------------------------------------+ +| **Docker images** | onap/cps-and-ncmp:3.6.1 | +| | | ++--------------------------------------+--------------------------------------------------------+ +| **Release designation** | 3.6.1 Paris | +| | | ++--------------------------------------+--------------------------------------------------------+ +| **Release date** | Not yet released | +| | | ++--------------------------------------+--------------------------------------------------------+ + +Features +-------- + Version: 3.6.0 ============== @@ -31,15 +54,19 @@ Release Data | **Release designation** | 3.6.0 Paris | | | | +--------------------------------------+--------------------------------------------------------+ -| **Release date** | Not yet released | +| **Release date** | 2025 January 29 | | | | +--------------------------------------+--------------------------------------------------------+ Bug Fixes --------- + - `CPS-2563 <https://lf-onap.atlassian.net/browse/CPS-2563>`_ Fix for internal error code during duplicated registration. + - `CPS-2576 <https://lf-onap.atlassian.net/browse/CPS-2576>`_ Fix for cm handle stuck in LOCKED state during registration. Features -------- + - `CPS-2249 <https://lf-onap.atlassian.net/browse/CPS-2249>`_ NCMP to support Conflict Handling. + - `CPS-2540 <https://lf-onap.atlassian.net/browse/CPS-2540>`_ One schemaset per module set tag. .. ==================== @@ -68,21 +95,19 @@ Release Data Bug Fixes --------- -3.5.5 - `CPS-2509 <https://lf-onap.atlassian.net/browse/CPS-2509>`_ Fix module endpoints using alternate identifier. - `CPS-2517 <https://lf-onap.atlassian.net/browse/CPS-2517>`_ Make Content-Type header default to JSON for CPS APIs. - - `CPS-2530 <https://lf-onap.atlassian.net/browse/CPS-2530>`_ NCMP Modules API giving empty response on READY cm handles if two sub systems discovered in parallel. + - `CPS-2530 <https://lf-onap.atlassian.net/browse/CPS-2530>`_ NCMP Modules API giving empty response on READY CM Handles if two sub systems discovered in parallel. Features -------- -3.5.5 - `CPS-2009 <https://lf-onap.atlassian.net/browse/CPS-2009>`_ Update legacy NCMP APIs interfaces to support alternate id. - `CPS-2082 <https://lf-onap.atlassian.net/browse/CPS-2082>`_ Support XML content type to data node APIs in cps-core. - `CPS-2433 <https://lf-onap.atlassian.net/browse/CPS-2433>`_ Remove traces of unmaintained CPS-TBDMT repository. - `CPS-2436 <https://lf-onap.atlassian.net/browse/CPS-2436>`_ CM Avc Event to publish source key to target key while forwarding. - `CPS-2445 <https://lf-onap.atlassian.net/browse/CPS-2445>`_ Expose CPS and NCMP version information using git plugin. - `CPS-2451 <https://lf-onap.atlassian.net/browse/CPS-2451>`_ Removing oparent from CPS-NCMP and ONAP DMI Plugin repository. - - `CPS-2478 <https://lf-onap.atlassian.net/browse/CPS-2478>`_ Optimized Cm Handle Registration and De-Registration use case. + - `CPS-2478 <https://lf-onap.atlassian.net/browse/CPS-2478>`_ Optimized CM Handle Registration and De-Registration use case. - `CPS-2507 <https://lf-onap.atlassian.net/browse/CPS-2507>`_ Upgrade liquibase to 4.30.0 version. Performance @@ -111,12 +136,10 @@ Release Data Bug Fixes --------- -3.5.4 - `CPS-2403 <https://lf-onap.atlassian.net/browse/CPS-2403>`_ Improve lock handling and queue management during CM-handle Module Sync. Features -------- -3.5.4 - `CPS-2408 <https://lf-onap.atlassian.net/browse/CPS-2408>`_ One Hazelcast instance per JVM to manage the distributed data structures. Version: 3.5.3 @@ -141,7 +164,6 @@ Release Data Bug Fixes --------- -3.5.3 - `CPS-2353 <https://lf-onap.atlassian.net/browse/CPS-2353>`_ Slow cmHandle registration when we use moduleSetTag, alternateId and dataProducerIdentifier - `CPS-2395 <https://lf-onap.atlassian.net/browse/CPS-2395>`_ Retry mechanism (with back off algorithm) is removed with more frequent watchdog poll - `CPS-2409 <https://lf-onap.atlassian.net/browse/CPS-2409>`_ Return NONE for get effective trust level api if the trust level caches empty (restart case) @@ -150,9 +172,6 @@ Bug Fixes Features -------- -3.5.3 - - `CPS-2247 <https://lf-onap.atlassian.net/browse/CPS-2247>`_ Policy Executor: Invoke Policy Executor and handle 'deny' response - - `CPS-2412 <https://lf-onap.atlassian.net/browse/CPS-2412>`_ Policy Executor: handle errors - `CPS-2417 <https://lf-onap.atlassian.net/browse/CPS-2417>`_ Remove Hazelcast cache for prefix resolver @@ -178,14 +197,12 @@ Release Data Bug Fixes --------- -3.5.2 - `CPS-2306 <https://lf-onap.atlassian.net/browse/CPS-2306>`_ Update response message for data validation failure and make it consistent across APIs - `CPS-2319 <https://lf-onap.atlassian.net/browse/CPS-2319>`_ Fix "Create a node" and "Add List Elements" APIs response code - `CPS-2372 <https://lf-onap.atlassian.net/browse/CPS-2372>`_ Blank alternate ID overwrites existing one Features -------- -3.5.2 - `CPS-1812 <https://lf-onap.atlassian.net/browse/CPS-1812>`_ CM Data Subscriptions ( Create, Delete and Merging ) with positive scenarios - `CPS-2326 <https://lf-onap.atlassian.net/browse/CPS-2326>`_ Uplift liquibase-core dependency to 4.28.0 - `CPS-2353 <https://lf-onap.atlassian.net/browse/CPS-2353>`_ Improve registration performance with moduleSetTag @@ -213,14 +230,12 @@ Release Data Bug Fixes --------- -3.5.1 - `CPS-2302 <https://lf-onap.atlassian.net/browse/CPS-2302>`_ Fix handling of special characters in moduleSetTag. Features -------- -3.5.1 - `CPS-2121 <https://lf-onap.atlassian.net/browse/CPS-2121>`_ Enabled http client prometheus metrics and manage high cardinality using URL template. - - `CPS-2289 <https://lf-onap.atlassian.net/browse/CPS-2289>`_ Support for CPS Path Query in NCMP Inventory Cm Handle Search. + - `CPS-2289 <https://lf-onap.atlassian.net/browse/CPS-2289>`_ Support for CPS Path Query in NCMP Inventory CM Handle Search. Version: 3.5.0 ============== @@ -242,13 +257,8 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.5.0 - Features -------- -3.5.0 - `CPS-989 <https://lf-onap.atlassian.net/browse/CPS-989>`_ Replace RestTemplate with WebClient. - `CPS-2172 <https://lf-onap.atlassian.net/browse/CPS-2172>`_ Support for OpenTelemetry Tracing. @@ -278,12 +288,10 @@ Release Data Bug Fixes --------- -3.4.9 - `CPS-2211 <https://lf-onap.atlassian.net/browse/CPS-2211>`_ Toggle switch to disable CPS Core change events if not used by application. Set CPS_CHANGE_EVENT_NOTIFICATIONS_ENABLED environment variable for the same. Features -------- -3.4.9 - `CPS-1836 <https://lf-onap.atlassian.net/browse/CPS-1836>`_ Delta between anchor and JSON payload. Version: 3.4.8 @@ -308,15 +316,11 @@ Release Data Bug Fixes --------- -3.4.8 - `CPS-2186 <https://lf-onap.atlassian.net/browse/CPS-2186>`_ Report async task failures to client topic during data operations request - `CPS-2190 <https://lf-onap.atlassian.net/browse/CPS-2190>`_ Improve performance of NCMP module searches - `CPS-2194 <https://lf-onap.atlassian.net/browse/CPS-2194>`_ Added defaults for CPS and DMI username and password - `CPS-2204 <https://lf-onap.atlassian.net/browse/CPS-2204>`_ Added error handling for yang module upgrade operation -Features --------- - Version: 3.4.7 ============== @@ -339,12 +343,10 @@ Release Data Bug Fixes --------- -3.4.7 - `CPS-2150 <https://lf-onap.atlassian.net/browse/CPS-2150>`_ Fix for Async task execution failed by TimeoutException. Features -------- -3.4.7 - `CPS-2061 <https://lf-onap.atlassian.net/browse/CPS-2061>`_ Liquibase Steps Condensing and Cleanup. - `CPS-2101 <https://lf-onap.atlassian.net/browse/CPS-2101>`_ Uplift Spring Boot to 3.2.4 version. @@ -370,7 +372,6 @@ Release Data Bug Fixes --------- -3.4.6 - `CPS-2126 <https://lf-onap.atlassian.net/browse/CPS-2126>`_ Passing HTTP Authorization Bearer Token to DMI Plugins. @@ -406,10 +407,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.4.5 - Features -------- @@ -438,7 +435,6 @@ Release Data Bug Fixes --------- -3.4.4 - `CPS-2027 <https://lf-onap.atlassian.net/browse/CPS-2027>`_ Upgrade Yang modules using module set tag functionalities fix Features @@ -469,7 +465,6 @@ Release Data Bug Fixes --------- -3.4.3 - `CPS-2000 <https://lf-onap.atlassian.net/browse/CPS-2000>`_ Fix for Schema object cache not being distributed. - `CPS-2027 <https://lf-onap.atlassian.net/browse/CPS-2027>`_ Fixes for upgrade yang modules using module set tag. - `CPS-2070 <https://lf-onap.atlassian.net/browse/CPS-2070>`_ Add retry interval for Kafka consumer. @@ -506,11 +501,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.4.2 - - Features -------- - `CPS-1638 <https://lf-onap.atlassian.net/browse/CPS-1638>`_ Introduce trust level for CM handle. @@ -530,7 +520,7 @@ Known Limitations, Issues and Workarounds For upgrading, CPS uses Liquibase for database upgrades. In order to enable Hibernate write batching (`CPS-1795 <https://lf-onap.atlassian.net/browse/CPS-1795>`_), a change to the database entity ID generation is required. As such, *this release does not fully support In-Service Software Upgrade* - CPS will not store new DataNodes and -NCMP will not register new CM-handles during an upgrade with old and new versions of CPS running concurrently. +NCMP will not register new CM Handles during an upgrade with old and new versions of CPS running concurrently. Other operations (read, update, delete) are not impacted. @@ -556,7 +546,6 @@ Release Data Bug Fixes --------- -3.4.1 - `CPS-1979 <https://lf-onap.atlassian.net/browse/CPS-1979>`_ Bug fix for Invalid topic name suffix. Features @@ -594,7 +583,6 @@ Release Data Bug Fixes --------- -3.4.0 - `CPS-1956 <https://lf-onap.atlassian.net/browse/CPS-1956>`_ Bug fix for No yang resources stored during cmhandle discovery. .. ======================== @@ -623,13 +611,9 @@ Release Data Bug Fixes --------- -3.3.9 - `CPS-1923 <https://lf-onap.atlassian.net/browse/CPS-1923>`_ CPS and NCMP changed management endpoint and port from /manage to /actuator and port same as cps application port. - `CPS-1933 <https://lf-onap.atlassian.net/browse/CPS-1933>`_ Setting up the class loader explicitly in hazelcast config. -Features --------- - Version: 3.3.8 ============== @@ -650,10 +634,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.8 - Features -------- - `CPS-1888 <https://lf-onap.atlassian.net/browse/CPS-1888>`_ Uplift Spring Boot to 3.1.2. @@ -680,7 +660,6 @@ Release Data Bug Fixes --------- -3.3.7 - `CPS-1866 <https://lf-onap.atlassian.net/browse/CPS-1866>`_ Fix ClassDefNotFoundError in opendaylight Yang parser Features @@ -713,7 +692,6 @@ Release Data Bug Fixes --------- -3.3.6 - `CPS-1841 <https://lf-onap.atlassian.net/browse/CPS-1841>`_ Update of top-level data node fails with exception - `CPS-1842 <https://lf-onap.atlassian.net/browse/CPS-1842>`_ Replace event-id with correlation-id for data read operation cloud event @@ -743,10 +721,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.5 - Features -------- - `CPS-1760 <https://lf-onap.atlassian.net/browse/CPS-1760>`_ Improve handling of special characters in Cps Paths @@ -771,10 +745,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.4 - Features -------- - `CPS-1767 <https://lf-onap.atlassian.net/browse/CPS-1767>`_ Upgrade CPS to java 17 @@ -799,13 +769,9 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.3 - Features -------- - - `CPS-1515 <https://lf-onap.atlassian.net/browse/CPS-1515>`_ Support Multiple CM-Handles for NCMP Get Operation + - `CPS-1515 <https://lf-onap.atlassian.net/browse/CPS-1515>`_ Support Multiple CM Handles for NCMP Get Operation - `CPS-1675 <https://lf-onap.atlassian.net/browse/CPS-1675>`_ Persistence write performance improvement(s) - `CPS-1745 <https://lf-onap.atlassian.net/browse/CPS-1745>`_ Upgrade to Openapi 3.0.3 @@ -831,7 +797,6 @@ Release Data Bug Fixes --------- -3.3.2 - `CPS-1716 <https://lf-onap.atlassian.net/browse/CPS-1716>`_ NCMP: Java Heap OutOfMemory errors and slow registration in case of 20k cmhandles Features @@ -862,11 +827,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.1 - - None - Features -------- - `CPS-1272 <https://lf-onap.atlassian.net/browse/CPS-1272>`_ Add Contains operation to CPS Path @@ -895,11 +855,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.0 - - None - Features -------- - `CPS-1215 <https://lf-onap.atlassian.net/browse/CPS-1215>`_ Add OR operation for CPS Path @@ -931,7 +886,6 @@ Release Data Bug Fixes --------- -3.2.6 - `CPS-1526 <https://lf-onap.atlassian.net/browse/CPS-1526>`_ Fix response message for PATCH operation - `CPS-1563 <https://lf-onap.atlassian.net/browse/CPS-1563>`_ Fix 500 response error on id-searches with empty parameters @@ -961,13 +915,8 @@ Release Data Bug Fixes --------- -3.2.5 - `CPS-1537 <https://lf-onap.atlassian.net/browse/CPS-1537>`_ Introduce control switch for model loader functionality. -Features --------- - - None - Version: 3.2.4 ============== @@ -1265,7 +1214,7 @@ Features - `CPS-869 <https://lf-onap.atlassian.net/browse/CPS-869>`_ Apply Standardized logging fields to adhere to ONAP Best practice REQ-1072 - `CPS-870 <https://lf-onap.atlassian.net/browse/CPS-870>`_ Align CPS-Core output with SDN-C output (add module name) - `CPS-875 <https://lf-onap.atlassian.net/browse/CPS-875>`_ CM Handle State: Watchdog-process that syncs 'ADVISED' CM Handles - - `CPS-877 <https://lf-onap.atlassian.net/browse/CPS-877>`_ CM Handle State: Exclude any CM-Handles from queries/operations that are not in state 'READY' + - `CPS-877 <https://lf-onap.atlassian.net/browse/CPS-877>`_ CM Handle State: Exclude any CM Handles from queries/operations that are not in state 'READY' - `CPS-899 <https://lf-onap.atlassian.net/browse/CPS-899>`_ Start and stop sessions on Java API - `CPS-909 <https://lf-onap.atlassian.net/browse/CPS-909>`_ Separate NCMP endpoint for ch/{cm-handle}/properties and ch/{cm-handle}/state - `CPS-917 <https://lf-onap.atlassian.net/browse/CPS-917>`_ Structured Errors response for passthrough use-cases in NCMP @@ -1278,7 +1227,7 @@ Features - `CPS-1099 <https://lf-onap.atlassian.net/browse/CPS-1099>`_ Expose simplified 'external' lock reason enum state over REST interface - `CPS-1101 <https://lf-onap.atlassian.net/browse/CPS-1101>`_ Introducing the DELETING and DELETED Cmhandle State - `CPS-1102 <https://lf-onap.atlassian.net/browse/CPS-1102>`_ Register the Cmhandle Sends Advised State notification. - - `CPS-1133 <https://lf-onap.atlassian.net/browse/CPS-1133>`_ Enable/Disable Data Sync for Cm Handle + - `CPS-1133 <https://lf-onap.atlassian.net/browse/CPS-1133>`_ Enable/Disable Data Sync for CM Handle - `CPS-1136 <https://lf-onap.atlassian.net/browse/CPS-1136>`_ DMI Audit Support (get all CM Handles for a registered DMI) @@ -1379,27 +1328,27 @@ Features - `CPS-559 <https://lf-onap.atlassian.net/browse/CPS-559>`_ Define response objects (schemas) in cps-ncmp - `CPS-636 <https://lf-onap.atlassian.net/browse/CPS-636>`_ Update operation for datastore pass through running - `CPS-638 <https://lf-onap.atlassian.net/browse/CPS-638>`_ Delete operation for datastore pass through running - - `CPS-677 <https://lf-onap.atlassian.net/browse/CPS-677>`_ Support 'public' Cm Handle Properties - - `CPS-741 <https://lf-onap.atlassian.net/browse/CPS-741>`_ Re sync after removing cm handles + - `CPS-677 <https://lf-onap.atlassian.net/browse/CPS-677>`_ Support 'public' CM Handle Properties + - `CPS-741 <https://lf-onap.atlassian.net/browse/CPS-741>`_ Re sync after removing CM Handles - `CPS-777 <https://lf-onap.atlassian.net/browse/CPS-777>`_ Ensure all DMI operations use POST method - `CPS-780 <https://lf-onap.atlassian.net/browse/CPS-780>`_ Add examples for parameters, request and response in openapi yaml for cps-core - `CPS-789 <https://lf-onap.atlassian.net/browse/CPS-789>`_ CPS Data Updated Event Schema V2 to support delete operation - `CPS-791 <https://lf-onap.atlassian.net/browse/CPS-791>`_ CPS-Core sends delete notification event - - `CPS-817 <https://lf-onap.atlassian.net/browse/CPS-817>`_ Create Endpoint For Get Cm Handles (incl. public properties) By Name + - `CPS-817 <https://lf-onap.atlassian.net/browse/CPS-817>`_ Create Endpoint For Get CM Handles (incl. public properties) By Name - `CPS-837 <https://lf-onap.atlassian.net/browse/CPS-837>`_ Add Remove and Update properties (DMI and Public) as part of CM Handle Registration update Bug Fixes --------- - - `CPS-762 <https://lf-onap.atlassian.net/browse/CPS-762>`_ Query cm handles for module names returns incorrect cm handle identifiers + - `CPS-762 <https://lf-onap.atlassian.net/browse/CPS-762>`_ Query CM Handles for module names returns incorrect CM Handle identifiers - `CPS-788 <https://lf-onap.atlassian.net/browse/CPS-788>`_ Yang Resource formatting is incorrect - - `CPS-783 <https://lf-onap.atlassian.net/browse/CPS-783>`_ Remove cm handle does not completely remove all cm handle information + - `CPS-783 <https://lf-onap.atlassian.net/browse/CPS-783>`_ Remove CM Handle does not completely remove all CM Handle information - `CPS-841 <https://lf-onap.atlassian.net/browse/CPS-841>`_ Upgrade log4j to 2.17.1 as recommended by ONAP SECCOM - `CPS-856 <https://lf-onap.atlassian.net/browse/CPS-856>`_ Retry mechanism not working for concurrent CmHandle registration - `CPS-867 <https://lf-onap.atlassian.net/browse/CPS-867>`_ Database port made configurable through env variable DB_PORT - `CPS-886 <https://lf-onap.atlassian.net/browse/CPS-886>`_ Fragment handling decreasing performance for large number of cmHandles - `CPS-887 <https://lf-onap.atlassian.net/browse/CPS-887>`_ Increase performance of cmHandle registration for large number of schema sets in DB - - `CPS-892 <https://lf-onap.atlassian.net/browse/CPS-892>`_ Fixed the response code during CM-Handle Registration from 201 CREATED to 204 NO_CONTENT + - `CPS-892 <https://lf-onap.atlassian.net/browse/CPS-892>`_ Fixed the response code during CM Handle Registration from 201 CREATED to 204 NO_CONTENT - `CPS-893 <https://lf-onap.atlassian.net/browse/CPS-893>`_ NCMP Java API depends on NCMP-Rest-API (cyclic) through json properties on Java API Known Limitations, Issues and Workarounds @@ -1407,9 +1356,9 @@ Known Limitations, Issues and Workarounds *System Limitations* -Null can no longer be passed within the dmi plugin service names when registering a cm handle, as part of +Null can no longer be passed within the dmi plugin service names when registering a CM Handle, as part of `CPS-837 <https://lf-onap.atlassian.net/browse/CPS-837>`_ null is now used to indicate if a property should be removed as part -of cm handle registration. +of CM Handle registration. The Absolute path to list with integer key will not work. Please refer `CPS-961 <https://lf-onap.atlassian.net/browse/CPS-961>`_ for more information. diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 531d353b0c..5a00e8dd6e 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -23,7 +23,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index fa65d9d873..3f7b25d2fa 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * Modifications Copyright (C) 2024-2025 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -21,7 +21,6 @@ package org.onap.cps.integration.base -import com.hazelcast.collection.ISet import com.hazelcast.map.IMap import okhttp3.mockwebserver.MockWebServer import org.onap.cps.api.CpsAnchorService @@ -29,24 +28,23 @@ import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsDataspaceService import org.onap.cps.api.CpsModuleService import org.onap.cps.api.CpsQueryService +import org.onap.cps.api.exceptions.DataspaceNotFoundException +import org.onap.cps.api.model.DataNode import org.onap.cps.integration.DatabaseTestContainer import org.onap.cps.integration.KafkaTestContainer -import org.onap.cps.ncmp.impl.NetworkCmProxyInventoryFacadeImpl +import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.api.inventory.models.DmiPluginRegistration import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle +import org.onap.cps.ncmp.impl.NetworkCmProxyInventoryFacadeImpl import org.onap.cps.ncmp.impl.data.NetworkCmProxyFacade import org.onap.cps.ncmp.impl.data.NetworkCmProxyQueryService import org.onap.cps.ncmp.impl.inventory.InventoryPersistence import org.onap.cps.ncmp.impl.inventory.ParameterizedCmHandleQueryService -import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncService import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.onap.cps.ncmp.impl.utils.AlternateIdMatcher import org.onap.cps.ri.repository.DataspaceRepository import org.onap.cps.ri.utils.SessionManager -import org.onap.cps.api.exceptions.DataspaceNotFoundException -import org.onap.cps.api.model.DataNode -import static org.onap.cps.utils.ContentType.* import org.onap.cps.utils.JsonObjectMapper import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value @@ -136,9 +134,6 @@ abstract class CpsIntegrationSpecBase extends Specification { @Autowired AlternateIdMatcher alternateIdMatcher - @Autowired - ISet<String> moduleSetTagsBeingProcessed - @Value('${ncmp.policy-executor.server.port:8080}') private String policyServerPort; @@ -158,11 +153,13 @@ abstract class CpsIntegrationSpecBase extends Specification { static NO_ALTERNATE_ID = '' static GENERAL_TEST_DATASPACE = 'generalTestDataspace' static BOOKSTORE_SCHEMA_SET = 'bookstoreSchemaSet' - static MODULE_SYNC_WAIT_TIME_IN_SECONDS = 10 + static MODULE_SYNC_WAIT_TIME_IN_SECONDS = 2 static initialized = false def now = OffsetDateTime.now() + enum ModuleNameStrategy { UNIQUE, OVERLAPPING } + def setup() { if (!initialized) { cpsDataspaceService.createDataspace(GENERAL_TEST_DATASPACE) @@ -186,7 +183,7 @@ abstract class CpsIntegrationSpecBase extends Specification { mockDmiServer1.shutdown() mockDmiServer2.shutdown() mockPolicyServer.shutdown() - moduleSetTagsBeingProcessed.clear() + cpsModuleService.deleteAllUnusedYangModuleData('NFP-Operational') } def static readResourceDataFile(filename) { @@ -264,14 +261,19 @@ abstract class CpsIntegrationSpecBase extends Specification { } def registerCmHandleWithoutWaitForReady(dmiPlugin, cmHandleId, moduleSetTag, alternateId) { - def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: cmHandleId, moduleSetTag: moduleSetTag, alternateId: alternateId) + def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: cmHandleId, moduleSetTag: moduleSetTag, alternateId: alternateId, dataProducerIdentifier: 'some data producer id') networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, createdCmHandles: [cmHandleToCreate])) } def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset) { + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy.UNIQUE) + } + + def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy moduleNameStrategy ) { def cmHandles = [] def id = offset - def moduleReferences = (1..200).collect { "${moduleSetTag}Module${it}" } + def modulePrefix = moduleNameStrategy.OVERLAPPING.equals(moduleNameStrategy) ? 'same' : moduleSetTag + def moduleReferences = (1..200).collect { "${modulePrefix}Module${it}" } (1..numberOfCmHandles).each { def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: "ch-${id}", moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID) cmHandles.add(ncmpServiceCmHandle) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy index 35a7b6a7c2..c790521627 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -113,7 +113,7 @@ class DmiDispatcher extends Dispatcher { def destination = Matcher.lastMatcher[0][1] def subJobWriteRequest = jsonSlurper.parseText(request.getBody().readUtf8()) this.receivedSubJobs.put(destination, subJobWriteRequest) - def response = '{"subJobId":"some sub job id", "dmiServiceName":"some dmi service name", "dataProducerId":"some data producer id"}' + def response = '{"subJobId":"some sub job id"}' return mockResponseWithBody(HttpStatus.OK, response) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy index 257f10b58b..ca321119ea 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -22,7 +22,6 @@ package org.onap.cps.integration.functional.cps import java.time.OffsetDateTime - import org.onap.cps.api.CpsAnchorService import org.onap.cps.integration.base.FunctionalSpecBase import org.onap.cps.api.parameters.FetchDescendantsOption @@ -69,7 +68,7 @@ class AnchorServiceIntegrationSpec extends FunctionalSpecBase { } def 'Querying anchor(name)s (depends on previous test!).'() { - expect: 'there are now 3 anchors using the "stores" module (both schema sets use the same modules) ' + expect: 'there are now 3 anchors using the "stores" module (both schema sets use the same modules)' assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'bookstore-types']).size() == 3 and: 'there are no anchors using both "stores" and a "unused-model"' assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'unused-model']).size() == 0 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy index 6cd7f21df3..47a332adc9 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -32,8 +32,6 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsDataspaceService } - def cleanup() { cpsModuleService.deleteUnusedYangResourceModules() } - def 'Dataspace CRUD operations.'() { when: 'a dataspace is created' objectUnderTest.createDataspace('newDataspace') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy index efdd71d5f4..c787b4209e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -41,17 +41,17 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { private static def bookStoreTypesModuleReference = new ModuleReference('bookstore-types','2024-01-30') private static def bookStoreTypesModuleReferenceWithNamespace = new ModuleReference('bookstore-types','2024-01-30', 'org:onap:cps:types:sample') static def NEW_RESOURCE_REVISION = '2023-05-10' - static def NEW_RESOURCE_CONTENT = 'module test_module {\n' + - ' yang-version 1.1;\n' + - ' namespace "org:onap:ccsdk:sample";\n' + - '\n' + - ' prefix book-store;\n' + - '\n' + - ' revision "2023-05-10" {\n' + - ' description\n' + - ' "Sample Model";\n' + - ' }' + - '}' + static def NEW_RESOURCE_CONTENT = """ + module test_module { + yang-version 1.1; + namespace "org:onap:ccsdk:sample"; + prefix book-store; + revision "2023-05-10" { + description + "Sample Model"; + } + } + """ def newYangResourcesNameToContentMap = [:] def moduleReferences = [] @@ -61,8 +61,6 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsModuleService } - def cleanup() { objectUnderTest.deleteUnusedYangResourceModules() } - /* C R E A T E S C H E M A S E T U S E - C A S E S */ diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy index 6f063fb222..c4946b48b9 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -178,9 +178,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { assert dmiPluginRegistrationResponse.createdCmHandles.sort { it.cmHandle } == [ CmHandleRegistrationResponse.createSuccessResponse('ch-3'), CmHandleRegistrationResponse.createSuccessResponse('ch-4'), - CmHandleRegistrationResponse.createFailureResponse('ch-5', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED), + CmHandleRegistrationResponse.createFailureResponse('ch-5', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST), CmHandleRegistrationResponse.createSuccessResponse('ch-6'), - CmHandleRegistrationResponse.createFailureResponse('ch-7', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED), + CmHandleRegistrationResponse.createFailureResponse('ch-7', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST), ] cleanup: 'deregister CM handles' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy index f2593ce587..22bbaa81df 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -76,8 +76,8 @@ class CmHandleUpdateSpec extends CpsIntegrationSpecBase { def dmiPluginRegistrationResponse = objectUnderTest.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: DMI1_URL, updatedCmHandles: [cmHandleToUpdate])) - then: 'registration gives failure response, due to alternate ID being already associated' - assert dmiPluginRegistrationResponse.updatedCmHandles == [CmHandleRegistrationResponse.createFailureResponse('ch-1', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED)] + then: 'registration gives failure response, due to cm-handle already existing' + assert dmiPluginRegistrationResponse.updatedCmHandles == [CmHandleRegistrationResponse.createFailureResponse('ch-1', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST)] and: 'the CM-handle still has the old alternate ID' assert objectUnderTest.getNcmpServiceCmHandle('ch-1').alternateId == 'original' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy index 11a4f2c6a7..097a043556 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -21,12 +21,12 @@ package org.onap.cps.integration.functional.ncmp import org.onap.cps.integration.base.CpsIntegrationSpecBase -import org.onap.cps.ncmp.impl.NetworkCmProxyInventoryFacadeImpl import org.onap.cps.ncmp.api.inventory.models.CmHandleRegistrationResponse -import org.onap.cps.ncmp.api.inventory.models.DmiPluginRegistration -import org.onap.cps.ncmp.api.inventory.models.UpgradedCmHandles import org.onap.cps.ncmp.api.inventory.models.CmHandleState +import org.onap.cps.ncmp.api.inventory.models.DmiPluginRegistration import org.onap.cps.ncmp.api.inventory.models.LockReasonCategory +import org.onap.cps.ncmp.api.inventory.models.UpgradedCmHandles +import org.onap.cps.ncmp.impl.NetworkCmProxyInventoryFacadeImpl import spock.util.concurrent.PollingConditions class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { @@ -38,10 +38,9 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { def setup() { objectUnderTest = networkCmProxyInventoryFacade - moduleSyncService.clearPrivateModuleSetCache() } - def 'Upgrade CM-handle with new moduleSetTag or no moduleSetTag.'() { + def 'Upgrade CM-handle with and without (new) module set tags.'() { given: 'a CM-handle is created with expected initial modules: M1 and M2' dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2'] registerCmHandle(DMI1_URL, cmHandleId, initialModuleSetTag) @@ -78,15 +77,15 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { and: 'CM-handle has expected updated modules: M1 and M3' assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort() - cleanup: 'deregister CM-handle' + cleanup: 'deregister CM-handle and remove all associated module resources' deregisterCmHandle(DMI1_URL, cmHandleId) where: 'following module set tags are used' initialModuleSetTag | updatedModuleSetTag NO_MODULE_SET_TAG | NO_MODULE_SET_TAG - NO_MODULE_SET_TAG | 'new' - 'initial' | NO_MODULE_SET_TAG - 'initial' | 'new' + NO_MODULE_SET_TAG | 'new@set' + 'initial set' | NO_MODULE_SET_TAG + 'initial set' | 'new@set' } def 'Upgrade CM-handle with existing moduleSetTag.'() { @@ -127,8 +126,8 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { where: initialModuleSetTag | updatedModuleSetTag - NO_MODULE_SET_TAG | 'moduleSet2' - 'moduleSet1' | 'moduleSet2' + NO_MODULE_SET_TAG | 'module@Set2' + 'module@Set1' | 'module@Set2' } def 'Skip upgrade of CM-handle with same moduleSetTag as before.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy index a6e56ab22d..81fbc3c4f7 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -20,11 +20,13 @@ package org.onap.cps.integration.functional.ncmp +import com.hazelcast.map.IMap import io.micrometer.core.instrument.MeterRegistry import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.springframework.beans.factory.annotation.Autowired import org.springframework.util.StopWatch +import spock.lang.Ignore import spock.util.concurrent.PollingConditions import java.util.concurrent.Executors @@ -37,16 +39,19 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { @Autowired MeterRegistry meterRegistry + @Autowired + IMap<String, Integer> cmHandlesByState + def executorService = Executors.newFixedThreadPool(2) def PARALLEL_SYNC_SAMPLE_SIZE = 100 def setup() { objectUnderTest = moduleSyncWatchdog + clearCmHandleStateGauge() } def cleanup() { try { - deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) moduleSyncWorkQueue.clear() } finally { executorService.shutdownNow() @@ -54,21 +59,26 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { } def 'Watchdog is disabled for test.'() { - given: + given: 'some cm handles are registered' registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, PARALLEL_SYNC_SAMPLE_SIZE, 1) when: 'wait a while but less then the initial delay of 10 minutes' Thread.sleep(3000) then: 'the work queue remains empty' assert moduleSyncWorkQueue.isEmpty() + cleanup: 'remove advised cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } + /** this test has intermittent failures, due to race conditions + * Ignored but left here as it might be valuable to further optimization investigations. + **/ + @Ignore def 'CPS-2478 Highlight (and improve) module sync inefficiencies.'() { given: 'register 250 cm handles with module set tag cps-2478-A' def numberOfTags = 2 def cmHandlesPerTag = 250 def totalCmHandles = numberOfTags * cmHandlesPerTag def offset = 1 - def minimumBatches = totalCmHandles / 100 registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset) and: 'register anther 250 cm handles with module set tag cps-2478-B' offset += cmHandlesPerTag @@ -78,30 +88,25 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { when: 'sync all advised cm handles' objectUnderTest.moduleSyncAdvisedCmHandles() Thread.sleep(100) - then: 'retry until all schema sets are stored in db (1 schema set for each cm handle)' - def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer() - new PollingConditions().within(10, () -> { - objectUnderTest.moduleSyncAdvisedCmHandles() - Thread.sleep(100) - assert dbSchemaSetStorageTimer.count() >= 500 - }) - then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)' - def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() + then: 'Keep processing until there are no more LOCKED or ADVISED cm handles' new PollingConditions().within(10, () -> { - assert dbStateUpdateTimer.count() >= minimumBatches + def advised = cmHandlesByState.get('advisedCmHandlesCount') + def locked = cmHandlesByState.get('lockedCmHandlesCount') + if ( locked > 0 | advised > 0 ) { + println "CPS-2576 Need to retry ${locked} LOCKED / ${advised} ADVISED cm Handles" + objectUnderTest.moduleSyncAdvisedCmHandles() + Thread.sleep(100) + } + assert cmHandlesByState.get('lockedCmHandlesCount') + cmHandlesByState.get('advisedCmHandlesCount') == 0 }) - and: 'the db has been queried for tags exactly 2 times.' - def dbModuleQueriesTimer = meterRegistry.get('cps.module.service.module.reference.query.by.attribute').timer() - assert dbModuleQueriesTimer.count() == 2 - and: 'exactly 2 calls to DMI to get module references' - def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer() - assert dmiModuleRetrievalTimer.count() == 2 and: 'log the relevant instrumentation' - logInstrumentation(dbModuleQueriesTimer, 'query module references') + def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer() + def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer() + def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() logInstrumentation(dmiModuleRetrievalTimer, 'get modules from DMI ') logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets ') logInstrumentation(dbStateUpdateTimer, 'batch state updates ') - cleanup: 'remove all cm handles' + cleanup: 'remove all test cm handles' // To properly measure performance the sample-size should be increased to 20,000 cm handles or higher (10,000 per tag) def stopWatch = new StopWatch() stopWatch.start() @@ -122,6 +127,32 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { Thread.sleep(50) then: 'the queue size is exactly the sample size' assert moduleSyncWorkQueue.size() == PARALLEL_SYNC_SAMPLE_SIZE + cleanup: 'remove all test cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) + } + + /** this test has intermittent failures, due to race conditions + * Ignored but left here as it might be valuable to further optimization investigations. + **/ + @Ignore + def 'Schema sets with overlapping modules processed at the same time (DB constraint violation).'() { + given: 'register one batch (100) cm handles of tag A (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagA', 100, 1, ModuleNameStrategy.OVERLAPPING) + and: 'register another batch cm handles of tag B (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagB', 100, 101, ModuleNameStrategy.OVERLAPPING) + and: 'populate the work queue with both batches' + objectUnderTest.populateWorkQueueIfNeeded() + when: 'advised cm handles are processed on 2 threads (exactly one batch for each)' + objectUnderTest.moduleSyncAdvisedCmHandles() + executorService.execute(moduleSyncAdvisedCmHandles) + then: 'wait till all cm handles have been processed' + new PollingConditions().within(10, () -> { + assert getNumberOfProcessedCmHandles() == 200 + }) + then: 'at least 1 cm handle is in state LOCKED' + assert cmHandlesByState.get('lockedCmHandlesCount') >= 1 + cleanup: 'remove all test cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, 200, 1) } def 'Populate module sync work queue on two parallel threads with a slight difference in start time.'() { @@ -136,6 +167,8 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { Thread.sleep(50) then: 'the queue size is exactly the sample size' assert moduleSyncWorkQueue.size() == PARALLEL_SYNC_SAMPLE_SIZE + cleanup: 'remove all test cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } def logInstrumentation(timer, description) { @@ -151,22 +184,30 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { } } - def populateQueueWithoutDelayCallable = () -> { + def populateQueueWithDelay = () -> { try { + Thread.sleep(10) objectUnderTest.populateWorkQueueIfNeeded() - return 'task acquired the lock first' } catch (InterruptedException e) { e.printStackTrace() } } - def populateQueueWithDelay = () -> { + def moduleSyncAdvisedCmHandles = () -> { try { - Thread.sleep(10) - objectUnderTest.populateWorkQueueIfNeeded() + objectUnderTest.moduleSyncAdvisedCmHandles() } catch (InterruptedException e) { e.printStackTrace() } } + def clearCmHandleStateGauge() { + cmHandlesByState.keySet().each { cmHandlesByState.put(it, 0)} + } + + def getNumberOfProcessedCmHandles() { + return cmHandlesByState.get('readyCmHandlesCount') + cmHandlesByState.get('lockedCmHandlesCount') + } + + } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/WriteSubJobSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/WriteSubJobSpec.groovy index 834e1399e3..5d3ea1919a 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/WriteSubJobSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/WriteSubJobSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -59,9 +59,9 @@ class WriteSubJobSpec extends CpsIntegrationSpecBase { then: 'each DMI received the expected sub-jobs and the response has the expected values' assert response.size() == 2 assert response[0].class == SubJobWriteResponse.class - assert response[0].subJobId == "some sub job id" - assert response[0].dmiServiceName == "some dmi service name" - assert response[0].dataProducerId == "some data producer id" + assert response[0].subJobId == 'some sub job id' + assert response[0].dmiServiceName.startsWith('http://localhost:') + assert response[0].dataProducerId == 'some data producer id' and: 'dmi 1 received the correct job details' def receivedSubJobsForDispatcher1 = dmiDispatcher1.receivedSubJobs['?destination=d1']['data'].collect() assert receivedSubJobsForDispatcher1.size() == 2 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/YangModulesSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/YangModulesSpec.groovy index 4492e3d183..e01af55c35 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/YangModulesSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/YangModulesSpec.groovy @@ -38,14 +38,16 @@ class YangModulesSpec extends CpsIntegrationSpecBase { def setup() { dmiDispatcher1.moduleNamesPerCmHandleId['ch-1'] = ['M1', 'M2'] dmiDispatcher1.moduleNamesPerCmHandleId['ch-2'] = ['M1', 'M3'] + dmiDispatcher1.moduleNamesPerCmHandleId['ch-3'] = ['M4'] registerCmHandle(DMI1_URL, 'ch-1', NO_MODULE_SET_TAG, 'alt-1') registerCmHandle(DMI1_URL, 'ch-2', NO_MODULE_SET_TAG, 'alt-2') + registerCmHandle(DMI1_URL, 'ch-3', 'my-module-set-tag', 'alt-3') // Note DMI dispatcher is not configured to return modules for this handle, so module sync will fail registerCmHandleWithoutWaitForReady(DMI1_URL, 'not-ready-id', NO_MODULE_SET_TAG, NO_ALTERNATE_ID) } def cleanup() { - deregisterCmHandles(DMI1_URL, ['ch-1', 'ch-2', 'not-ready-id']) + deregisterCmHandles(DMI1_URL, ['ch-1', 'ch-2', 'ch-3', 'not-ready-id']) } def 'Get yang module references returns expected modules with #scenario.'() { @@ -56,11 +58,12 @@ class YangModulesSpec extends CpsIntegrationSpecBase { .andExpect(jsonPath('$[*].moduleName', containsInAnyOrder(expectedModuleNames.toArray()))) .andExpect(jsonPath('$[*].revision', everyItem(equalTo('2024-01-01')))) where: 'following scenarios are applied' - scenario | cmHandleReference || expectedModuleNames - 'cm-handle id' | 'ch-1' || ['M1', 'M2'] - 'alternate id' | 'alt-2' || ['M1', 'M3'] - 'not ready CM handle' | 'not-ready-id' || [] - 'non-existing CM handle' | 'non-existing' || [] + scenario | cmHandleReference || expectedModuleNames + 'cm-handle id' | 'ch-1' || ['M1', 'M2'] + 'alternate id' | 'alt-2' || ['M1', 'M3'] + 'CM handle with module set tag' | 'ch-3' || ['M4'] + 'not ready CM handle' | 'not-ready-id' || [] + 'non-existing CM handle' | 'non-existing' || [] } def 'Get yang module definitions returns expected modules with #scenario.'() { @@ -72,11 +75,12 @@ class YangModulesSpec extends CpsIntegrationSpecBase { .andExpect(jsonPath('$[*].revision', everyItem(equalTo('2024-01-01')))) .andExpect(jsonPath('$[*].content', not(is(emptyString())))) where: 'following scenarios are applied' - scenario | cmHandleReference || expectedModuleNames - 'cm-handle id' | 'ch-1' || ['M1', 'M2'] - 'alternate id' | 'alt-2' || ['M1', 'M3'] - 'not ready CM handle' | 'not-ready-id' || [] - 'non-existing CM handle' | 'non-existing' || [] + scenario | cmHandleReference || expectedModuleNames + 'cm-handle id' | 'ch-1' || ['M1', 'M2'] + 'alternate id' | 'alt-2' || ['M1', 'M3'] + 'CM handle with module set tag' | 'ch-3' || ['M4'] + 'not ready CM handle' | 'not-ready-id' || [] + 'non-existing CM handle' | 'non-existing' || [] } def 'Get yang module definition for specific module with #scenario.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy index b239a78d71..e52d3f819c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -100,7 +100,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, (i..i+100).collect {SCHEMA_SET_PREFIX + it}) } cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, [SCHEMA_SET_PREFIX + '0']) - cpsModuleService.deleteUnusedYangResourceModules() + cpsModuleService.deleteAllUnusedYangModuleData(CPS_PERFORMANCE_TEST_DATASPACE) } // This makes a Yang module of approximately target length in bytes by padding the description field with many '*' diff --git a/jacoco-report/pom.xml b/jacoco-report/pom.xml index be1d42788d..3cd2e5d62e 100644 --- a/jacoco-report/pom.xml +++ b/jacoco-report/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/k6-tests/ncmp/common/cmhandle-crud.js b/k6-tests/ncmp/common/cmhandle-crud.js index 0fea1a44f2..285028f13c 100644 --- a/k6-tests/ncmp/common/cmhandle-crud.js +++ b/k6-tests/ncmp/common/cmhandle-crud.js @@ -53,9 +53,11 @@ function createCmHandlePayload(cmHandleIds) { "dmiPlugin": DMI_PLUGIN_URL, "createdCmHandles": cmHandleIds.map((cmHandleId, index) => ({ "cmHandle": cmHandleId, - "alternateId": cmHandleId.replace('ch-', 'alt-'), + "alternateId": cmHandleId.replace('ch-', 'Subnetwork=Europe,ManagedElement='), "moduleSetTag": MODULE_SET_TAGS[index % MODULE_SET_TAGS.length], - "cmHandleProperties": {"neType": "RadioNode"}, + "cmHandleProperties": { + "id": "123" + }, "publicCmHandleProperties": { "Color": "yellow", "Size": "small", diff --git a/k6-tests/ncmp/common/passthrough-crud.js b/k6-tests/ncmp/common/passthrough-crud.js index 86fcef6242..251ddf8890 100644 --- a/k6-tests/ncmp/common/passthrough-crud.js +++ b/k6-tests/ncmp/common/passthrough-crud.js @@ -29,7 +29,7 @@ import { export function passthroughRead(useAlternateId) { const cmHandleReference = getRandomCmHandleReference(useAlternateId); - const resourceIdentifier = 'my-resource-identifier'; + const resourceIdentifier = 'NRCellDU/attributes/cellLocalId'; const datastoreName = 'ncmp-datastore:passthrough-operational'; const includeDescendants = true; const url = generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants); @@ -38,11 +38,14 @@ export function passthroughRead(useAlternateId) { export function passthroughWrite(useAlternateId) { const cmHandleReference = getRandomCmHandleReference(useAlternateId); - const resourceIdentifier = 'my-resource-identifier'; + const resourceIdentifier = 'NRCellDU/attributes/cellLocalId'; const datastoreName = 'ncmp-datastore:passthrough-running'; const includeDescendants = false; const url = generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants); - const payload = JSON.stringify({"neType": "BaseStation"}); + const payload = JSON.stringify({ + "id": "123", + "attributes": {"userLabel": "test"} + }); return performPostRequest(url, payload, 'passthroughWrite'); } @@ -51,10 +54,10 @@ export function legacyBatchRead(cmHandleIds) { const payload = JSON.stringify({ "operations": [ { - "resourceIdentifier": "parent/child", + "resourceIdentifier": "NRCellDU/attributes/cellLocalId", "targetIds": cmHandleIds, "datastore": "ncmp-datastore:passthrough-operational", - "options": "(fields=schemas/schema)", + "options": "(fields=NRCellDU/attributes/cellLocalId)", "operationId": "12", "operation": "read" } @@ -64,8 +67,8 @@ export function legacyBatchRead(cmHandleIds) { } function getRandomCmHandleReference(useAlternateId) { - const prefix = useAlternateId ? 'alt' : 'ch'; - return `${prefix}-${randomIntBetween(1, TOTAL_CM_HANDLES)}`; + const prefix = useAlternateId ? 'Subnetwork=Europe,ManagedElement=' : 'ch-'; + return `${prefix}${randomIntBetween(1, TOTAL_CM_HANDLES)}`; } function generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants) { diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index 6a1faa7ccb..6bedb1f949 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl; export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl; export const CONTAINER_UP_TIME_IN_SECONDS = testConfig.hosts.containerUpTimeInSeconds; export const LEGACY_BATCH_TOPIC_NAME = 'legacy_batch_topic'; -export const TOTAL_CM_HANDLES = 20000; +export const TOTAL_CM_HANDLES = 50000; export const REGISTRATION_BATCH_SIZE = 100; export const READ_DATA_FOR_CM_HANDLE_DELAY_MS = 300; // must have same value as in docker-compose.yml export const WRITE_DATA_FOR_CM_HANDLE_DELAY_MS = 670; // must have same value as in docker-compose.yml @@ -88,10 +88,10 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) { const summaryCsvLines = [ '#,Test Name,Unit,Fs Requirement,Current Expectation,Actual', makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig), - makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 50, testResults, scenarioConfig), - makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 120, testResults, scenarioConfig), + makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 90, testResults, scenarioConfig), + makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 140, testResults, scenarioConfig), makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 300, testResults, scenarioConfig), - makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 300, testResults, scenarioConfig), + makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 120, testResults, scenarioConfig), makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 750, testResults, scenarioConfig), makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 750, testResults, scenarioConfig), makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 3000, testResults, scenarioConfig), diff --git a/policy-executor-stub/pom.xml b/policy-executor-stub/pom.xml index bdd21e191f..20de885af8 100644 --- a/policy-executor-stub/pom.xml +++ b/policy-executor-stub/pom.xml @@ -6,7 +6,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> @@ -27,7 +27,7 @@ <modelVersion>4.0.0</modelVersion>
<groupId>org.onap.cps</groupId>
<artifactId>cps-aggregator</artifactId>
- <version>3.6.0-SNAPSHOT</version>
+ <version>3.6.1-SNAPSHOT</version>
<packaging>pom</packaging>
<name>cps</name>
diff --git a/releases/3.6.0-container.yaml b/releases/3.6.0-container.yaml new file mode 100644 index 0000000000..f97a5184fc --- /dev/null +++ b/releases/3.6.0-container.yaml @@ -0,0 +1,8 @@ +distribution_type: container +container_release_tag: 3.6.0 +project: cps +log_dir: cps-maven-docker-stage-master/949/ +ref: 851ad00169168fab91d81f1fd23d7f84f7a1005d +containers: + - name: 'cps-and-ncmp' + version: '3.6.0-20250129T162415Z' diff --git a/releases/3.6.0.yaml b/releases/3.6.0.yaml new file mode 100644 index 0000000000..7476bd47a4 --- /dev/null +++ b/releases/3.6.0.yaml @@ -0,0 +1,4 @@ +distribution_type: maven +log_dir: cps-maven-stage-master/957/ +project: cps +version: 3.6.0 diff --git a/spotbugs/pom.xml b/spotbugs/pom.xml index 8c685e95bc..a6e47860c9 100644 --- a/spotbugs/pom.xml +++ b/spotbugs/pom.xml @@ -25,7 +25,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>spotbugs</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <properties> <onap.nexus.url>https://nexus.onap.org</onap.nexus.url> diff --git a/version.properties b/version.properties index 81372a7e58..6cf6639717 100644 --- a/version.properties +++ b/version.properties @@ -22,7 +22,7 @@ major=3 minor=6 -patch=0 +patch=1 base_version=${major}.${minor}.${patch} |