diff options
159 files changed, 4847 insertions, 2339 deletions
diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index d6fbcd98d5..ea79b718fc 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -26,7 +26,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>checkstyle</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <profiles> <profile> diff --git a/cps-application/pom.xml b/cps-application/pom.xml index 5ac2202e85..44d0677ead 100644 --- a/cps-application/pom.xml +++ b/cps-application/pom.xml @@ -28,7 +28,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> @@ -78,6 +78,10 @@ <artifactId>micrometer-tracing-bridge-otel</artifactId> </dependency> <dependency> + <groupId>io.github.mweirauch</groupId> + <artifactId>micrometer-jvm-extras</artifactId> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-xml</artifactId> </dependency> diff --git a/cps-application/src/main/java/org/onap/cps/Application.java b/cps-application/src/main/java/org/onap/cps/Application.java index 053139fcc8..62103bf368 100644 --- a/cps-application/src/main/java/org/onap/cps/Application.java +++ b/cps-application/src/main/java/org/onap/cps/Application.java @@ -22,9 +22,7 @@ package org.onap.cps; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.retry.annotation.EnableRetry;
-@EnableRetry
@SpringBootApplication
public class Application {
public static void main(final String[] args) {
diff --git a/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java index de981164f5..b85f391b8e 100644 --- a/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java +++ b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java @@ -21,10 +21,14 @@ package org.onap.cps.config; import com.hazelcast.map.IMap; +import io.github.mweirauch.micrometer.jvm.extras.ProcessMemoryMetrics; +import io.github.mweirauch.micrometer.jvm.extras.ProcessThreadMetrics; import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -41,6 +45,18 @@ public class MicroMeterConfig { return new TimedAspect(meterRegistry); } + @Bean + @ConditionalOnProperty("cps.monitoring.micrometer-jvm-extras") + public MeterBinder processMemoryMetrics() { + return new ProcessMemoryMetrics(); + } + + @Bean + @ConditionalOnProperty("cps.monitoring.micrometer-jvm-extras") + public MeterBinder processThreadMetrics() { + return new ProcessThreadMetrics(); + } + /** * Register gauge metric for cm handles with state 'advised'. * @@ -52,7 +68,7 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("advisedCmHandlesCount")) .tag(STATE_TAG, "ADVISED") - .description("Current number of cmhandles in advised state") + .description("Current number of cm handles in advised state") .register(meterRegistry); } @@ -67,7 +83,7 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("readyCmHandlesCount")) .tag(STATE_TAG, "READY") - .description("Current number of cmhandles in ready state") + .description("Current number of cm handles in ready state") .register(meterRegistry); } @@ -82,7 +98,7 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("lockedCmHandlesCount")) .tag(STATE_TAG, "LOCKED") - .description("Current number of cmhandles in locked state") + .description("Current number of cm handles in locked state") .register(meterRegistry); } @@ -97,7 +113,22 @@ public class MicroMeterConfig { return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, value -> cmHandlesByState.get("deletingCmHandlesCount")) .tag(STATE_TAG, "DELETING") - .description("Current number of cmhandles in deleting state") + .description("Current number of cm handles in deleting state") + .register(meterRegistry); + } + + /** + * Register gauge metric for cm handles with state 'deleted'. + * + * @param meterRegistry meter registry + * @return cm handle state gauge + */ + @Bean + public Gauge deletedCmHandles(final MeterRegistry meterRegistry) { + return Gauge.builder(CM_HANDLE_STATE_GAUGE, cmHandlesByState, + value -> cmHandlesByState.get("deletedCmHandlesCount")) + .tag(STATE_TAG, "DELETED") + .description("Number of cm handles that have been deleted since the application started") .register(meterRegistry); } diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml index 0b5d59ecc9..6eb9e108be 100644 --- a/cps-application/src/main/resources/application.yml +++ b/cps-application/src/main/resources/application.yml @@ -106,6 +106,7 @@ app: cm-subscription-dmi-out: ${CM_SUBSCRIPTION_DMI_OUT_TOPIC:dmi-ncmp-cm-avc-subscription} cm-subscription-ncmp-out: ${CM_SUBSCRIPTION_NCMP_OUT_TOPIC:subscription-response} cm-events-topic: ${NCMP_CM_EVENTS_TOPIC:cm-events} + inventory-events-topic: ncmp-inventory-events lcm: events: topic: ${LCM_EVENTS_TOPIC:ncmp-events} @@ -152,6 +153,8 @@ security: password: ${CPS_PASSWORD:cpsr0cks!} cps: + monitoring: + micrometer-jvm-extras: false tracing: sampler: jaeger_remote: @@ -245,10 +248,6 @@ ncmp: trust-level: dmi-availability-watchdog-ms: 30000 - modules-sync-watchdog: - async-executor: - parallelism-level: 10 - model-loader: maximum-attempt-count: 20 diff --git a/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy index da3afc6f2c..b9302ccd72 100644 --- a/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy +++ b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy @@ -31,10 +31,17 @@ class MicroMeterConfigSpec extends Specification { def simpleMeterRegistry = new SimpleMeterRegistry() def 'Creating a timed aspect.'() { - expect: ' a timed aspect can be created' + expect: 'a timed aspect can be created' assert objectUnderTest.timedAspect(simpleMeterRegistry) != null } + def 'Creating JVM process metrics.'() { + expect: 'process memory metrics can be created' + assert objectUnderTest.processMemoryMetrics() != null + and: 'process thread metrics can be created' + assert objectUnderTest.processThreadMetrics() != null + } + def 'Creating gauges for cm handle states.'() { given: 'cache returns value for each state' cmHandlesByState.get(_) >> 1 @@ -43,10 +50,10 @@ class MicroMeterConfigSpec extends Specification { objectUnderTest.readyCmHandles(simpleMeterRegistry) objectUnderTest.lockedCmHandles(simpleMeterRegistry) objectUnderTest.deletingCmHandles(simpleMeterRegistry) + objectUnderTest.deletedCmHandles(simpleMeterRegistry) then: 'each state has the correct value when queried' - def states = ["ADVISED", "READY", "LOCKED", "DELETING"] - states.each { state -> - def gaugeValue = simpleMeterRegistry.get("cmHandlesByState").tag("state",state).gauge().value() + ['ADVISED', 'READY', 'LOCKED', 'DELETING', 'DELETED'].each { state -> + def gaugeValue = simpleMeterRegistry.get('cmHandlesByState').tag('state',state).gauge().value() assert gaugeValue == 1 } } diff --git a/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java b/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java index 1d39060024..c1d65758c7 100644 --- a/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java +++ b/cps-application/src/test/java/org/onap/cps/architecture/ArchitectureTestBase.java @@ -36,6 +36,7 @@ public class ArchitectureTestBase { "lombok..", "org.apache..", "org.mapstruct..", + "org.opendaylight..", "org.slf4j..", "org.springframework..", "reactor.." diff --git a/cps-bom/pom.xml b/cps-bom/pom.xml index 1bb8308e07..f01709f5d1 100644 --- a/cps-bom/pom.xml +++ b/cps-bom/pom.xml @@ -25,7 +25,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>cps-bom</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <packaging>pom</packaging> <description>This artifact contains dependencyManagement declarations of all published CPS components.</description> diff --git a/cps-dependencies/pom.xml b/cps-dependencies/pom.xml index 5783ef7fcd..545476c628 100644 --- a/cps-dependencies/pom.xml +++ b/cps-dependencies/pom.xml @@ -27,7 +27,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>cps-dependencies</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <packaging>pom</packaging> <name>${project.groupId}:${project.artifactId}</name> @@ -146,7 +146,7 @@ <dependency> <groupId>com.hazelcast</groupId> <artifactId>hazelcast-spring</artifactId> - <version>5.3.7</version> + <version>5.5.0</version> </dependency> <dependency> <groupId>com.squareup.okhttp3</groupId> @@ -173,6 +173,11 @@ <scope>import</scope> </dependency> <dependency> + <groupId>io.github.mweirauch</groupId> + <artifactId>micrometer-jvm-extras</artifactId> + <version>0.2.2</version> + </dependency> + <dependency> <groupId>io.gsonfire</groupId> <artifactId>gson-fire</artifactId> <version>1.9.0</version> diff --git a/cps-events/pom.xml b/cps-events/pom.xml index 9d49f181fb..50df9c8aa1 100644 --- a/cps-events/pom.xml +++ b/cps-events/pom.xml @@ -24,7 +24,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-events/src/main/resources/schemas/updatenode/cps-data-updated-event-schema-1.0.0.json b/cps-events/src/main/resources/schemas/cps.dataupdated/cps-data-updated-event-schema-1.0.0.json index a3eaf63fa4..a3eaf63fa4 100644 --- a/cps-events/src/main/resources/schemas/updatenode/cps-data-updated-event-schema-1.0.0.json +++ b/cps-events/src/main/resources/schemas/cps.dataupdated/cps-data-updated-event-schema-1.0.0.json diff --git a/cps-ncmp-events/pom.xml b/cps-ncmp-events/pom.xml index bb45493554..2667d0924b 100644 --- a/cps-ncmp-events/pom.xml +++ b/cps-ncmp-events/pom.xml @@ -23,7 +23,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-events/src/main/resources/schemas/dmidataavc/avc-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/dmi/cm-events/avc-event-schema-1.0.0.json index 474520d142..474520d142 100644 --- a/cps-ncmp-events/src/main/resources/schemas/dmidataavc/avc-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/dmi/cm-events/avc-event-schema-1.0.0.json diff --git a/cps-ncmp-events/src/main/resources/schemas/trustlevel/device-trust-level-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/dmi/device-heartbeat/device-trust-level-event-schema-1.0.0.json index e1796fbc73..e1796fbc73 100644 --- a/cps-ncmp-events/src/main/resources/schemas/trustlevel/device-trust-level-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/dmi/device-heartbeat/device-trust-level-event-schema-1.0.0.json diff --git a/cps-ncmp-events/src/main/resources/schemas/async/data-operation-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/async-m2m/data-operation-event-schema-1.0.0.json index c2915187c7..c2915187c7 100644 --- a/cps-ncmp-events/src/main/resources/schemas/async/data-operation-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/async-m2m/data-operation-event-schema-1.0.0.json diff --git a/cps-ncmp-events/src/main/resources/schemas/dmi-async-request-response-event-schema-v1.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/async-m2m/dmi-async-request-response-event-schema-v1.json index 2340a4bc4c..4db07bec95 100644 --- a/cps-ncmp-events/src/main/resources/schemas/dmi-async-request-response-event-schema-v1.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/async-m2m/dmi-async-request-response-event-schema-v1.json @@ -6,6 +6,7 @@ "DmiAsyncRequestResponseEvent": { "description": "The payload for NCMP async request response event.", "type": "object", + "javaType" : "org.onap.cps.ncmp.event.model.DmiAsyncRequestResponseEvent", "properties": { "eventId": { "description": "The unique id identifying the event generated by DMI.", diff --git a/cps-ncmp-events/src/main/resources/schemas/ncmp-async-request-response-event-schema-v1.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/async-m2m/ncmp-async-request-response-event-schema-v1.json index 51c2cf4d40..32b7becd05 100644 --- a/cps-ncmp-events/src/main/resources/schemas/ncmp-async-request-response-event-schema-v1.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/async-m2m/ncmp-async-request-response-event-schema-v1.json @@ -6,6 +6,7 @@ "NcmpAsyncRequestResponseEvent": { "description": "The payload for CPS async request response event.", "type": "object", + "javaType" : "org.onap.cps.ncmp.event.model.NcmpAsyncRequestResponseEvent", "properties": { "eventId": { "description": "The unique id identifying the event generated by DMI.", diff --git a/cps-ncmp-events/src/main/resources/schemas/ncmpdataavc/avc-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/avc-event-schema-1.0.0.json index d24ec2c737..d24ec2c737 100644 --- a/cps-ncmp-events/src/main/resources/schemas/ncmpdataavc/avc-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/avc-event-schema-1.0.0.json diff --git a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/dmi-in-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/dmi-in-event-schema-1.0.0.json index 93ec216e3d..93ec216e3d 100644 --- a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/dmi-in-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/dmi-in-event-schema-1.0.0.json diff --git a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/dmi-out-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/dmi-out-event-schema-1.0.0.json index 0910de1529..0910de1529 100644 --- a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/dmi-out-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/dmi-out-event-schema-1.0.0.json diff --git a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/ncmp-in-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/ncmp-in-event-schema-1.0.0.json index f8b6c2e680..f8b6c2e680 100644 --- a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/ncmp-in-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/ncmp-in-event-schema-1.0.0.json diff --git a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/ncmp-out-event-schema-1.0.0.json b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/ncmp-out-event-schema-1.0.0.json index 11dc4e1114..11dc4e1114 100644 --- a/cps-ncmp-events/src/main/resources/schemas/cmnotificationsubscription/ncmp-out-event-schema-1.0.0.json +++ b/cps-ncmp-events/src/main/resources/schemas/ncmp/avc/ncmp-out-event-schema-1.0.0.json diff --git a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml index 690ec01b8f..30a704f509 100644 --- a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml +++ b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-app/pom.xml @@ -22,7 +22,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-ncmp-rest-stub</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> </parent> <artifactId>cps-ncmp-rest-stub-app</artifactId> diff --git a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml index bba80cdcf1..18d468e4f4 100644 --- a/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml +++ b/cps-ncmp-rest-stub/cps-ncmp-rest-stub-service/pom.xml @@ -21,7 +21,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-ncmp-rest-stub</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> </parent> <artifactId>cps-ncmp-rest-stub-service</artifactId> diff --git a/cps-ncmp-rest-stub/pom.xml b/cps-ncmp-rest-stub/pom.xml index 056e52a4f0..b5654262b4 100644 --- a/cps-ncmp-rest-stub/pom.xml +++ b/cps-ncmp-rest-stub/pom.xml @@ -22,7 +22,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-rest/pom.xml b/cps-ncmp-rest/pom.xml index 4e2c48dd6f..2d8cb4dfd2 100644 --- a/cps-ncmp-rest/pom.xml +++ b/cps-ncmp-rest/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-service/pom.xml b/cps-ncmp-service/pom.xml index eb0aed1e7c..70e0b4f8f4 100644 --- a/cps-ncmp-service/pom.xml +++ b/cps-ncmp-service/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java index 75007e2e35..1a7ef758d8 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================== - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import com.hazelcast.config.MapConfig; import com.hazelcast.config.NamedConfig; import com.hazelcast.config.NearCacheConfig; import com.hazelcast.config.QueueConfig; -import com.hazelcast.config.RestEndpointGroup; import com.hazelcast.config.SetConfig; import com.hazelcast.core.Hazelcast; import com.hazelcast.core.HazelcastInstance; @@ -61,7 +60,6 @@ public class HazelcastCacheConfig { config.setClusterName(clusterName); config.setClassLoader(Dataspace.class.getClassLoader()); configureDataStructures(namedConfig, config); - exposeClusterInformation(config); updateDiscoveryMode(config); return config; } @@ -130,9 +128,4 @@ public class HazelcastCacheConfig { } } - protected void exposeClusterInformation(final Config config) { - config.getNetworkConfig().getRestApiConfig().setEnabled(true) - .enableGroups(RestEndpointGroup.HEALTH_CHECK, RestEndpointGroup.CLUSTER_READ); - } - } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java index 59d0f9704e..ae913ddfe7 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImpl.java @@ -38,7 +38,6 @@ import org.onap.cps.api.CpsQueryService; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.cpspath.parser.CpsPathUtil; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.ncmp.api.inventory.DataStoreSyncState; import org.onap.cps.ncmp.api.inventory.models.CmHandleState; import org.onap.cps.ncmp.api.inventory.models.TrustLevel; @@ -46,6 +45,7 @@ import org.onap.cps.ncmp.impl.inventory.models.ModelledDmiServiceLeaves; import org.onap.cps.ncmp.impl.inventory.models.PropertyType; import org.onap.cps.ncmp.impl.inventory.trustlevel.TrustLevelCacheConfig; import org.onap.cps.ncmp.impl.utils.YangDataConverter; +import org.onap.cps.utils.CpsValidator; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java index e7fd247a08..75c52f3c60 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationService.java @@ -23,7 +23,6 @@ package org.onap.cps.ncmp.impl.inventory; -import static org.onap.cps.ncmp.api.NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_FOUND; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_READY; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST; @@ -347,7 +346,7 @@ public class CmHandleRegistrationService { final Collection<String> rejectedCmHandleIds = alternateIdChecker .getIdsOfCmHandlesWithRejectedAlternateId(cmHandlesToBeCreated, AlternateIdChecker.Operation.CREATE); cmHandleRegistrationResponses.addAll(CmHandleRegistrationResponse.createFailureResponses( - rejectedCmHandleIds, ALTERNATE_ID_ALREADY_ASSOCIATED)); + rejectedCmHandleIds, CM_HANDLE_ALREADY_EXIST)); return rejectedCmHandleIds; } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java index b7a13d9989..3415793478 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandler.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ @@ -22,8 +22,8 @@ package org.onap.cps.ncmp.impl.inventory; -import static org.onap.cps.ncmp.api.NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_FOUND; +import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST; import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_INVALID_ID; import static org.onap.cps.ncmp.impl.inventory.CmHandleRegistrationServicePropertyHandler.PropertyType.DMI_PROPERTY; import static org.onap.cps.ncmp.impl.inventory.CmHandleRegistrationServicePropertyHandler.PropertyType.PUBLIC_PROPERTY; @@ -49,7 +49,7 @@ import org.onap.cps.api.CpsDataService; import org.onap.cps.api.exceptions.DataNodeNotFoundException; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; +import org.onap.cps.impl.DataNodeBuilder; import org.onap.cps.ncmp.api.inventory.models.CmHandleRegistrationResponse; import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; @@ -81,7 +81,7 @@ public class CmHandleRegistrationServicePropertyHandler { final Collection<String> rejectedCmHandleIds = alternateIdChecker .getIdsOfCmHandlesWithRejectedAlternateId(updatedNcmpServiceCmHandles, AlternateIdChecker.Operation.UPDATE); final List<CmHandleRegistrationResponse> failureResponses = - CmHandleRegistrationResponse.createFailureResponses(rejectedCmHandleIds, ALTERNATE_ID_ALREADY_ASSOCIATED); + CmHandleRegistrationResponse.createFailureResponses(rejectedCmHandleIds, CM_HANDLE_ALREADY_EXIST); final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>(failureResponses); for (final NcmpServiceCmHandle updatedNcmpServiceCmHandle : updatedNcmpServiceCmHandles) { final String cmHandleId = updatedNcmpServiceCmHandle.getCmHandleId(); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java index e7ec9cd13c..e145c62921 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java @@ -45,13 +45,13 @@ import org.onap.cps.api.model.DataNode; import org.onap.cps.api.model.ModuleDefinition; import org.onap.cps.api.model.ModuleReference; import org.onap.cps.api.parameters.FetchDescendantsOption; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException; import org.onap.cps.ncmp.api.inventory.models.CompositeState; import org.onap.cps.ncmp.api.inventory.models.CompositeStateBuilder; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; import org.onap.cps.ncmp.impl.utils.YangDataConverter; import org.onap.cps.utils.ContentType; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.utils.JsonObjectMapper; import org.springframework.stereotype.Component; diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/AsyncTaskExecutor.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/AsyncTaskExecutor.java deleted file mode 100644 index 80bc4ab69f..0000000000 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/AsyncTaskExecutor.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ncmp.impl.inventory.sync; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -import jakarta.annotation.PostConstruct; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeoutException; -import java.util.function.Supplier; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class AsyncTaskExecutor { - - @Value("${ncmp.modules-sync-watchdog.async-executor.parallelism-level:10}") - @Getter - private int asyncTaskParallelismLevel; - private ExecutorService executorService; - private static final int DEFAULT_PARALLELISM_LEVEL = 10; - - /** - * Set up executor service with thread-pool size as per configuration parameter. - * If modules-sync-watchdog.async-executor.parallelism-level not set a default of 10 threads will be applied. - */ - @PostConstruct - public void setupThreadPool() { - executorService = Executors.newWorkStealingPool( - asyncTaskParallelismLevel == 0 ? DEFAULT_PARALLELISM_LEVEL : asyncTaskParallelismLevel); - } - - /** - * Execute supplied task asynchronously. - * - * @param taskSupplier functional method is get() task need to executed asynchronously - * @param timeOutInMillis the task timeout value in milliseconds - */ - public void executeTask(final Supplier<Object> taskSupplier, final long timeOutInMillis) { - CompletableFuture.supplyAsync(taskSupplier::get, executorService) - .orTimeout(timeOutInMillis, MILLISECONDS) - .whenCompleteAsync(this::handleTaskCompletion); - } - - private void handleTaskCompletion(final Object response, final Throwable throwable) { - if (throwable != null) { - if (throwable instanceof TimeoutException) { - log.error("Async task didn't complete within the required time.", throwable); - } else { - log.error("Watchdog async batch failed.", throwable); - } - } - } -} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java index 1e24671f8d..2cc4375447 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,13 +61,14 @@ public class DmiModelOperations { * Retrieves module references. * * @param yangModelCmHandle the yang model cm handle + * @param targetModuleSetTag module set tag to send to dmi * @return module references */ @Timed(value = "cps.ncmp.inventory.module.references.from.dmi", description = "Time taken to get all module references for a cm handle from dmi") - public List<ModuleReference> getModuleReferences(final YangModelCmHandle yangModelCmHandle) { - final DmiRequestBody dmiRequestBody = DmiRequestBody.builder() - .moduleSetTag(yangModelCmHandle.getModuleSetTag()).build(); + public List<ModuleReference> getModuleReferences(final YangModelCmHandle yangModelCmHandle, + final String targetModuleSetTag) { + final DmiRequestBody dmiRequestBody = DmiRequestBody.builder().moduleSetTag(targetModuleSetTag).build(); dmiRequestBody.asDmiProperties(yangModelCmHandle.getDmiProperties()); final ResponseEntity<Object> dmiFetchModulesResponseEntity = getResourceFromDmiWithJsonData( yangModelCmHandle.resolveDmiServiceName(MODEL), @@ -79,18 +80,20 @@ public class DmiModelOperations { * Retrieve yang resources from dmi for any modules that CPS-NCMP hasn't cached before. * * @param yangModelCmHandle the yangModelCmHandle + * @param targetModuleSetTag module set tag to send to dmi * @param newModuleReferences the unknown module references * @return yang resources as map of module name to yang(re)source */ @Timed(value = "cps.ncmp.inventory.yang.resources.from.dmi", description = "Time taken to get list of yang resources from dmi") public Map<String, String> getNewYangResourcesFromDmi(final YangModelCmHandle yangModelCmHandle, + final String targetModuleSetTag, final Collection<ModuleReference> newModuleReferences) { if (newModuleReferences.isEmpty()) { return Collections.emptyMap(); } final String jsonWithDataAndDmiProperties = getRequestBodyToFetchYangResources(newModuleReferences, - yangModelCmHandle.getDmiProperties(), yangModelCmHandle.getModuleSetTag()); + yangModelCmHandle.getDmiProperties(), targetModuleSetTag); final ResponseEntity<Object> responseEntity = getResourceFromDmiWithJsonData( yangModelCmHandle.resolveDmiServiceName(MODEL), jsonWithDataAndDmiProperties, @@ -123,13 +126,13 @@ public class DmiModelOperations { private static String getRequestBodyToFetchYangResources(final Collection<ModuleReference> newModuleReferences, final List<YangModelCmHandle.Property> dmiProperties, - final String moduleSetTag) { + final String targetModuleSetTag) { final JsonArray moduleReferencesAsJson = getModuleReferencesAsJson(newModuleReferences); final JsonObject data = new JsonObject(); data.add("modules", moduleReferencesAsJson); final JsonObject jsonRequestObject = new JsonObject(); - if (!moduleSetTag.isEmpty()) { - jsonRequestObject.addProperty("moduleSetTag", moduleSetTag); + if (!targetModuleSetTag.isEmpty()) { + jsonRequestObject.addProperty("moduleSetTag", targetModuleSetTag); } jsonRequestObject.add("data", data); jsonRequestObject.add("cmHandleProperties", toJsonObject(dmiProperties)); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java index e9f3d9b475..80e41652ee 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleOperationsUtils.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -189,7 +189,12 @@ public class ModuleOperationsUtils { .getLockReasonCategory())); } - public static String getTargetModuleSetTagFromLockReason(final CompositeState.LockReason lockReason) { + public static String getTargetModuleSetTagForUpgrade(final YangModelCmHandle yangModelCmHandle) { + final CompositeState.LockReason lockReason = yangModelCmHandle.getCompositeState().getLockReason(); + return getTargetModuleSetTagFromLockReason(lockReason); + } + + private static String getTargetModuleSetTagFromLockReason(final CompositeState.LockReason lockReason) { return getLockedCompositeStateDetails(lockReason).getOrDefault(MODULE_SET_TAG_KEY, ""); } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java index 9534cf35b1..79f5496eb7 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java @@ -36,7 +36,6 @@ import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; import org.onap.cps.api.exceptions.AlreadyDefinedException; -import org.onap.cps.api.exceptions.DuplicatedYangResourceException; import org.onap.cps.api.model.ModuleReference; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; import org.onap.cps.utils.ContentType; @@ -57,7 +56,7 @@ public class ModuleSyncService { @AllArgsConstructor private static final class ModuleDelta { Collection<ModuleReference> allModuleReferences; - Map<String, String> newModuleNameToContentMap; + Map<String, String> newYangResourceContentPerName; } /** @@ -67,10 +66,15 @@ public class ModuleSyncService { */ public void syncAndCreateSchemaSetAndAnchor(final YangModelCmHandle yangModelCmHandle) { final String cmHandleId = yangModelCmHandle.getId(); - final String moduleSetTag = yangModelCmHandle.getModuleSetTag(); - final String schemaSetName = getSchemaSetName(cmHandleId, moduleSetTag); - syncAndCreateSchemaSet(yangModelCmHandle, schemaSetName); - cpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetName, cmHandleId); + final String targetModuleSetTag = yangModelCmHandle.getModuleSetTag(); + final String schemaSetName = getSchemaSetNameForModuleSetTag(cmHandleId, targetModuleSetTag); + syncAndCreateSchemaSet(yangModelCmHandle, schemaSetName, targetModuleSetTag); + try { + cpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetName, cmHandleId); + } catch (final AlreadyDefinedException alreadyDefinedException) { + log.warn("Ignoring (Anchor) already exists exception for {}. Exception details: {}", cmHandleId, + alreadyDefinedException.getDetails()); + } } /** @@ -81,39 +85,40 @@ public class ModuleSyncService { public void syncAndUpgradeSchemaSet(final YangModelCmHandle yangModelCmHandle) { final String cmHandleId = yangModelCmHandle.getId(); final String sourceModuleSetTag = yangModelCmHandle.getModuleSetTag(); - final String targetModuleSetTag = ModuleOperationsUtils.getTargetModuleSetTagFromLockReason( - yangModelCmHandle.getCompositeState().getLockReason()); + final String targetModuleSetTag = ModuleOperationsUtils.getTargetModuleSetTagForUpgrade(yangModelCmHandle); + final String schemaSetName = getSchemaSetNameForModuleSetTag(cmHandleId, targetModuleSetTag); if (sourceModuleSetTag.isEmpty() && targetModuleSetTag.isEmpty()) { - final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle); + final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle, targetModuleSetTag); cpsModuleService.upgradeSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, - cmHandleId, moduleDelta.newModuleNameToContentMap, moduleDelta.allModuleReferences); + schemaSetName, moduleDelta.newYangResourceContentPerName, moduleDelta.allModuleReferences); } else { - final String targetSchemaSetName = getSchemaSetName(cmHandleId, targetModuleSetTag); - syncAndCreateSchemaSet(yangModelCmHandle, targetSchemaSetName); - cpsAnchorService.updateAnchorSchemaSet(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, - targetSchemaSetName); + syncAndCreateSchemaSet(yangModelCmHandle, schemaSetName, targetModuleSetTag); + cpsAnchorService.updateAnchorSchemaSet(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, schemaSetName); setCmHandleModuleSetTag(yangModelCmHandle, targetModuleSetTag); log.info("Upgrading schema set for CM handle ID: {}, Source Tag: {}, Target Tag: {}", cmHandleId, sourceModuleSetTag, targetModuleSetTag); } } - private void syncAndCreateSchemaSet(final YangModelCmHandle yangModelCmHandle, final String schemaSetName) { + private void syncAndCreateSchemaSet(final YangModelCmHandle yangModelCmHandle, + final String schemaSetName, + final String targetModuleSetTag) { if (isNewSchemaSet(schemaSetName)) { - final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle); + final String cmHandleId = yangModelCmHandle.getId(); + final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle, targetModuleSetTag); try { - log.info("Creating Schema Set {} for CM Handle {}", schemaSetName, yangModelCmHandle.getId()); + log.info("Creating Schema Set {} for CM Handle {}", schemaSetName, cmHandleId); cpsModuleService.createSchemaSetFromModules( - NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, - schemaSetName, - moduleDelta.newModuleNameToContentMap, - moduleDelta.allModuleReferences + NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, + schemaSetName, + moduleDelta.newYangResourceContentPerName, + moduleDelta.allModuleReferences ); - log.info("Successfully created Schema Set {} for CM Handle {}", - schemaSetName, yangModelCmHandle.getId()); - } catch (final AlreadyDefinedException | DuplicatedYangResourceException exception) { - log.warn("Schema Set {} already exists, no need to (re)create it for {}", - schemaSetName, yangModelCmHandle.getId()); + log.info("Successfully created Schema Set {} for CM Handle {}", schemaSetName, + yangModelCmHandle.getId()); + } catch (final AlreadyDefinedException alreadyDefinedException) { + log.warn("Ignoring (Schema Set) already exists exception for {}. Exception details: {}", cmHandleId, + alreadyDefinedException.getDetails()); } } } @@ -122,16 +127,17 @@ public class ModuleSyncService { return !cpsModuleService.schemaSetExists(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, schemaSetName); } - private ModuleDelta getModuleDelta(final YangModelCmHandle yangModelCmHandle) { + private ModuleDelta getModuleDelta(final YangModelCmHandle yangModelCmHandle, + final String targetModuleSetTag) { final Collection<ModuleReference> allModuleReferences = - dmiModelOperations.getModuleReferences(yangModelCmHandle); + dmiModelOperations.getModuleReferences(yangModelCmHandle, targetModuleSetTag); final Collection<ModuleReference> newModuleReferences = cpsModuleService.identifyNewModuleReferences(allModuleReferences); - final Map<String, String> newYangResources = dmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, - newModuleReferences); + final Map<String, String> newYangResourceContentPerName = + dmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, targetModuleSetTag, newModuleReferences); log.debug("Module delta calculated for CM handle ID: {}. All references: {}. New modules: {}", - yangModelCmHandle.getId(), allModuleReferences, newYangResources.keySet()); - return new ModuleDelta(allModuleReferences, newYangResources); + yangModelCmHandle.getId(), allModuleReferences, newYangResourceContentPerName.keySet()); + return new ModuleDelta(allModuleReferences, newYangResourceContentPerName); } private void setCmHandleModuleSetTag(final YangModelCmHandle yangModelCmHandle, final String newModuleSetTag) { @@ -141,7 +147,7 @@ public class ModuleSyncService { jsonForUpdate, OffsetDateTime.now(), ContentType.JSON); } - private static String getSchemaSetName(final String cmHandleId, final String moduleSetTag) { + private static String getSchemaSetNameForModuleSetTag(final String cmHandleId, final String moduleSetTag) { return moduleSetTag.isEmpty() ? cmHandleId : moduleSetTag; } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java index 40404b719a..f8f023e0f8 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java @@ -24,8 +24,6 @@ import com.hazelcast.map.IMap; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicInteger; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.exceptions.DataNodeNotFoundException; @@ -51,12 +49,8 @@ public class ModuleSyncTasks { * Perform module sync on a batch of cm handles. * * @param cmHandleIds a batch of cm handle ids to perform module sync on - * @param batchCounter the number of batches currently being processed, will be decreased when - * task is finished or fails - * @return completed future to handle post-processing */ - public CompletableFuture<Void> performModuleSync(final Collection<String> cmHandleIds, - final AtomicInteger batchCounter) { + public void performModuleSync(final Collection<String> cmHandleIds) { final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(cmHandleIds.size()); try { for (final String cmHandleId : cmHandleIds) { @@ -74,11 +68,8 @@ public class ModuleSyncTasks { } } } finally { - batchCounter.getAndDecrement(); lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle); - log.info("Processing module sync batch finished. {} batch(es) active.", batchCounter.get()); } - return CompletableFuture.completedFuture(null); } /** @@ -114,7 +105,7 @@ public class ModuleSyncTasks { compositeState.setLockReason(null); return CmHandleState.READY; } catch (final Exception e) { - log.warn("Processing of {} failed,reason : {}.", yangModelCmHandle.getId(), e.getMessage()); + log.warn("Processing of {} failed, reason: {}.", yangModelCmHandle.getId(), e.getMessage()); final LockReasonCategory lockReasonCategory = inUpgrade ? LockReasonCategory.MODULE_UPGRADE_FAILED : LockReasonCategory.MODULE_SYNC_FAILED; @@ -124,8 +115,8 @@ public class ModuleSyncTasks { } private void removeResetCmHandleFromModuleSyncMap(final String resetCmHandleId) { - moduleSyncStartedOnCmHandles.removeAsync(resetCmHandleId); - log.info("{} will be removed asynchronously from in progress map", resetCmHandleId); + moduleSyncStartedOnCmHandles.delete(resetCmHandleId); + log.info("{} removed from in progress map", resetCmHandleId); } private static boolean isCmHandleInAdvisedState(final YangModelCmHandle yangModelCmHandle) { diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java index 32e1c49f17..6eefedb633 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,13 +27,9 @@ import com.hazelcast.map.IMap; import java.util.Collection; import java.util.HashSet; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; -import org.onap.cps.ncmp.impl.utils.Sleeper; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -46,16 +42,10 @@ public class ModuleSyncWatchdog { private final BlockingQueue<String> moduleSyncWorkQueue; private final IMap<String, Object> moduleSyncStartedOnCmHandles; private final ModuleSyncTasks moduleSyncTasks; - private final AsyncTaskExecutor asyncTaskExecutor; private final IMap<String, String> cpsAndNcmpLock; - private final Sleeper sleeper; private static final int MODULE_SYNC_BATCH_SIZE = 100; - private static final long PREVENT_CPU_BURN_WAIT_TIME_MILLIS = 10; private static final String VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP = "Started"; - private static final long ASYNC_TASK_TIMEOUT_IN_MILLISECONDS = TimeUnit.MINUTES.toMillis(5); - @Getter - private AtomicInteger batchCounter = new AtomicInteger(1); /** * Check DB for any cm handles in 'ADVISED' state. @@ -69,18 +59,11 @@ public class ModuleSyncWatchdog { log.debug("Processing module sync watchdog waking up."); populateWorkQueueIfNeeded(); while (!moduleSyncWorkQueue.isEmpty()) { - if (batchCounter.get() <= asyncTaskExecutor.getAsyncTaskParallelismLevel()) { - final Collection<String> nextBatch = prepareNextBatch(); - log.info("Processing module sync batch of {}. {} batch(es) active.", - nextBatch.size(), batchCounter.get()); - if (!nextBatch.isEmpty()) { - asyncTaskExecutor.executeTask(() -> - moduleSyncTasks.performModuleSync(nextBatch, batchCounter), - ASYNC_TASK_TIMEOUT_IN_MILLISECONDS); - batchCounter.getAndIncrement(); - } - } else { - preventBusyWait(); + final Collection<String> nextBatch = prepareNextBatch(); + if (!nextBatch.isEmpty()) { + log.info("Processing module sync batch of {}. 1 batch(es) active.", nextBatch.size()); + moduleSyncTasks.performModuleSync(nextBatch); + log.info("Processing module sync batch finished. 0 batch(es) active."); } } } @@ -153,13 +136,4 @@ public class ModuleSyncWatchdog { log.info("nextBatch size : {}", nextBatch.size()); return nextBatch; } - - private void preventBusyWait() { - try { - log.debug("Busy waiting now"); - sleeper.haveALittleRest(PREVENT_CPU_BURN_WAIT_TIME_MILLIS); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - } - } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java index f68bb3b543..692bf5caee 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/trustlevel/TrustLevelManager.java @@ -193,14 +193,14 @@ public class TrustLevelManager { final TrustLevel newEffectiveTrustLevel) { if (oldEffectiveTrustLevel.equals(newEffectiveTrustLevel)) { log.debug("The Cm Handle: {} has already the same trust level: {}", notificationCandidateCmHandleId, - newEffectiveTrustLevel); + newEffectiveTrustLevel); } else { log.info("The trust level for Cm Handle: {} is now: {} ", notificationCandidateCmHandleId, - newEffectiveTrustLevel); + newEffectiveTrustLevel); cmAvcEventPublisher.publishAvcEvent(notificationCandidateCmHandleId, - AVC_CHANGED_ATTRIBUTE_NAME, - oldEffectiveTrustLevel.name(), - newEffectiveTrustLevel.name()); + AVC_CHANGED_ATTRIBUTE_NAME, + oldEffectiveTrustLevel.name(), + newEffectiveTrustLevel.name()); } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/Sleeper.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/Sleeper.java deleted file mode 100644 index 7a02fa06e0..0000000000 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/Sleeper.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ncmp.impl.utils; - -import java.util.concurrent.TimeUnit; -import org.springframework.stereotype.Service; - -/** - * This class is to extract out sleep functionality so the interrupted exception handling can - * be covered with a test (e.g. using spy on Sleeper) and help to get to 100% code coverage. - */ -@Service -public class Sleeper { - public void haveALittleRest(final long timeInMillis) throws InterruptedException { - TimeUnit.MILLISECONDS.sleep(timeInMillis); - } -} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/utils/events/CmAvcEventPublisher.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/utils/events/CmAvcEventPublisher.java index 2a9717cc1a..bdc7899724 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/utils/events/CmAvcEventPublisher.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/utils/events/CmAvcEventPublisher.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,8 +38,8 @@ public class CmAvcEventPublisher { private final EventsPublisher<CloudEvent> eventsPublisher; - @Value("${app.ncmp.avc.cm-events-topic}") - private String avcTopic; + @Value("${app.ncmp.avc.inventory-events-topic}") + private String ncmpInventoryEventsTopicName; /** * Publish attribute value change event. @@ -52,10 +52,10 @@ public class CmAvcEventPublisher { final Map<String, String> extensions = createAvcEventExtensions(eventKey); final CloudEvent avcCloudEvent = - NcmpEvent.builder().type(AvcEvent.class.getTypeName()) - .data(avcEvent).extensions(extensions).build().asCloudEvent(); + NcmpEvent.builder().type(AvcEvent.class.getTypeName()) + .data(avcEvent).extensions(extensions).build().asCloudEvent(); - eventsPublisher.publishCloudEvent(avcTopic, eventKey, avcCloudEvent); + eventsPublisher.publishCloudEvent(ncmpInventoryEventsTopicName, eventKey, avcCloudEvent); } private AvcEvent buildAvcEvent(final String attributeName, @@ -78,4 +78,4 @@ public class CmAvcEventPublisher { extensions.put("correlationid", eventKey); return extensions; } -} +}
\ No newline at end of file diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy index 055a6e7448..c49af0f01b 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CmHandleRegistrationResponseSpec.groovy @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2022 Bell Canada - * Modifications Copyright (C) 2023-2024 Nordix Foundation + * Modifications Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import spock.lang.Specification import java.util.stream.Collectors -import static org.onap.cps.ncmp.api.NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST import static org.onap.cps.ncmp.api.NcmpResponseStatus.UNKNOWN_ERROR @@ -89,14 +88,14 @@ class CmHandleRegistrationResponseSpec extends Specification { } def 'Failed cm-handle registration based on cm handle id and registration error'() { - when: 'the failure response is created with "alternate id already associated" error code for 1 cm handle' + when: 'the failure response is created with "cm-handle already exists" error code for 1 cm handle' def cmHandleRegistrationResponses = - CmHandleRegistrationResponse.createFailureResponses(['ch 1'], ALTERNATE_ID_ALREADY_ASSOCIATED) + CmHandleRegistrationResponse.createFailureResponses(['ch 1'], CM_HANDLE_ALREADY_EXIST) then: 'the response with expected values' assert cmHandleRegistrationResponses[0].cmHandle == 'ch 1' assert cmHandleRegistrationResponses[0].status == Status.FAILURE - assert cmHandleRegistrationResponses[0].ncmpResponseStatus == ALTERNATE_ID_ALREADY_ASSOCIATED - assert cmHandleRegistrationResponses[0].errorText == 'alternate id already associated' + assert cmHandleRegistrationResponses[0].ncmpResponseStatus == CM_HANDLE_ALREADY_EXIST + assert cmHandleRegistrationResponses[0].errorText == 'cm-handle already exists' } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy index 4d42e62025..8b04568239 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/models/CompositeStateBuilderSpec.groovy @@ -24,7 +24,7 @@ package org.onap.cps.ncmp.api.inventory.models import org.onap.cps.ncmp.api.inventory.DataStoreSyncState import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import spock.lang.Specification import java.time.OffsetDateTime diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy index c08ff75a44..0026d7c4e6 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,7 @@ package org.onap.cps.ncmp.impl.cache -import com.hazelcast.config.Config -import com.hazelcast.config.RestEndpointGroup + import com.hazelcast.core.Hazelcast import spock.lang.Specification @@ -60,17 +59,4 @@ class HazelcastCacheConfigSpec extends Specification { 'Set Config' | HazelcastCacheConfig.createSetConfig('my set config') || false | false | true } - def 'Verify Hazelcast Cluster Information'() { - given: 'a test configuration' - def testConfig = new Config() - when: 'cluster information is exposed' - objectUnderTest.exposeClusterInformation(testConfig) - then: 'REST api configs are enabled' - assert testConfig.networkConfig.restApiConfig.enabled - and: 'only health check and cluster read are enabled' - def enabledGroups = testConfig.networkConfig.restApiConfig.enabledGroups - assert enabledGroups.size() == 2 - assert enabledGroups.containsAll([RestEndpointGroup.CLUSTER_READ, RestEndpointGroup.HEALTH_CHECK]) - } - } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy index 811e4ea526..1cbdc7beca 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleQueryServiceImplSpec.groovy @@ -26,7 +26,7 @@ import com.hazelcast.core.Hazelcast import com.hazelcast.instance.impl.HazelcastInstanceFactory import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsQueryService -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.ncmp.api.inventory.DataStoreSyncState import org.onap.cps.ncmp.api.inventory.models.TrustLevel import org.onap.cps.ncmp.api.inventory.models.CmHandleState diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy index b600d02be5..70bd418026 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy @@ -32,7 +32,7 @@ import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle import org.onap.cps.api.exceptions.DataNodeNotFoundException import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import org.slf4j.LoggerFactory diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy index d8d92e99f5..0ed9dd8aae 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy @@ -35,7 +35,7 @@ import org.onap.cps.api.model.DataNode import org.onap.cps.api.model.ModuleDefinition import org.onap.cps.api.model.ModuleReference import org.onap.cps.api.parameters.FetchDescendantsOption -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException import org.onap.cps.ncmp.api.inventory.models.CompositeState import org.onap.cps.ncmp.api.inventory.models.CmHandleState diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/AsyncTaskExecutorSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/AsyncTaskExecutorSpec.groovy deleted file mode 100644 index 751c97a4d0..0000000000 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/AsyncTaskExecutorSpec.groovy +++ /dev/null @@ -1,63 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ncmp.impl.inventory.sync - - -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest -import spock.lang.Specification - -import java.util.concurrent.TimeoutException -import java.util.function.Supplier - -@SpringBootTest(classes = AsyncTaskExecutor) -class AsyncTaskExecutorSpec extends Specification { - - @Autowired - AsyncTaskExecutor objectUnderTest - def mockTaskSupplier = Mock(Supplier<Object>) - - def 'Parallelism level configuration.'() { - expect: 'Parallelism level is configured with the correct value' - assert objectUnderTest.getAsyncTaskParallelismLevel() == 3 - } - - def 'Task completion with #caseDescriptor.'() { - when: 'task completion is handled' - def irrelevantResponse = null - objectUnderTest.handleTaskCompletion(irrelevantResponse, exception); - then: 'any exception is swallowed by the task completion (logged)' - noExceptionThrown() - where: 'following cases are tested' - caseDescriptor | exception - 'no exception' | null - 'time out exception' | new TimeoutException("time-out") - 'unexpected exception' | new Exception("some exception") - } - - def 'Task execution.'() { - when: 'a task is submitted for execution' - objectUnderTest.executeTask(() -> mockTaskSupplier, 0) - then: 'the task submission is successful' - noExceptionThrown() - } - -} diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy index 714555958a..302e43f170 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperationsSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,14 +21,11 @@ package org.onap.cps.ncmp.impl.inventory.sync -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.ObjectMapper import org.onap.cps.ncmp.impl.dmi.DmiOperationsBaseSpec import org.onap.cps.ncmp.impl.dmi.DmiProperties import org.onap.cps.ncmp.impl.utils.http.UrlTemplateParameters import org.onap.cps.api.model.ModuleReference import org.onap.cps.utils.JsonObjectMapper -import org.spockframework.spring.SpringBean import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus @@ -40,9 +37,12 @@ import static org.onap.cps.ncmp.api.data.models.OperationType.READ import static org.onap.cps.ncmp.impl.models.RequiredDmiService.MODEL @SpringBootTest -@ContextConfiguration(classes = [DmiProperties, DmiModelOperations]) +@ContextConfiguration(classes = [DmiProperties, DmiModelOperations, JsonObjectMapper]) class DmiModelOperationsSpec extends DmiOperationsBaseSpec { + def NO_AUTH_HEADER = null + def NO_MODULE_SET_TAG = '' + def expectedModulesUrlTemplateWithVariables = new UrlTemplateParameters('myServiceName/dmi/v1/ch/{cmHandleId}/modules', ['cmHandleId': cmHandleId]) def expectedModuleResourcesUrlTemplateWithVariables = new UrlTemplateParameters('myServiceName/dmi/v1/ch/{cmHandleId}/moduleResources', ['cmHandleId': cmHandleId]) @@ -52,11 +52,6 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { @Autowired DmiModelOperations objectUnderTest - @SpringBean - JsonObjectMapper spiedJsonObjectMapper = Spy(new JsonObjectMapper(new ObjectMapper())) - - def NO_AUTH_HEADER = null - def 'Retrieving module references.'() { given: 'a cm handle' mockYangModelCmHandleRetrieval([]) @@ -65,7 +60,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity([schemas: moduleReferencesAsLisOfMaps], HttpStatus.OK) mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModulesUrlTemplateWithVariables, '{"cmHandleProperties":{},"moduleSetTag":""}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get module references is called' - def result = objectUnderTest.getModuleReferences(yangModelCmHandle) + def result = objectUnderTest.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) then: 'the result consists of expected module references' assert result == [new ModuleReference(moduleName: 'mod1', revision: 'A'), new ModuleReference(moduleName: 'mod2', revision: 'X')] } @@ -78,7 +73,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity(bodyAsMap, HttpStatus.NO_CONTENT) mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> responseFromDmi when: 'get module references is called' - def result = objectUnderTest.getModuleReferences(yangModelCmHandle) + def result = objectUnderTest.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) then: 'the result is empty' assert result == [] where: 'the DMI response body has the following content' @@ -97,7 +92,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModulesUrlTemplateWithVariables, '{"cmHandleProperties":' + expectedAdditionalPropertiesInRequest + ',"moduleSetTag":""}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'a get module references is called' - def result = objectUnderTest.getModuleReferences(yangModelCmHandle) + def result = objectUnderTest.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) then: 'the result is the response from DMI service' assert result == [] where: 'the following DMI properties are used' @@ -116,7 +111,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModuleResourcesUrlTemplateWithVariables, '{"data":{"modules":[' + expectedModuleReferencesInRequest + ']},"cmHandleProperties":{}}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, newModuleReferences) then: 'the result has the 2 expected yang (re)sources (order is not guaranteed)' assert result.size() == 2 assert result.get('mod1') == 'some yang source' @@ -131,7 +126,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { def responseFromDmi = new ResponseEntity(responseFromDmiBody, HttpStatus.NO_CONTENT) mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, newModuleReferences) then: 'the result is empty' assert result == [:] where: 'the DMI response body has the following content' @@ -149,7 +144,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { '{"data":{"modules":[{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}]},"cmHandleProperties":' + expectedAdditionalPropertiesInRequest + '}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, newModuleReferences) then: 'the result is the response from DMI service' assert result == [mod1:'some yang source'] where: 'the following DMI properties are used' @@ -166,7 +161,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { mockDmiRestClient.synchronousPostOperationWithJsonData(MODEL, expectedModuleResourcesUrlTemplateWithVariables, '{' + expectedModuleSetTagInRequest + '"data":{"modules":[{"name":"mod1","revision":"A"},{"name":"mod2","revision":"X"}]},"cmHandleProperties":{}}', READ, NO_AUTH_HEADER) >> responseFromDmi when: 'get new yang resources from DMI service' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, newModuleReferences) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, moduleSetTag, newModuleReferences) then: 'the result is the response from DMI service' assert result == [mod1:'some yang source'] where: 'the following Module Set Tags are used' @@ -180,7 +175,7 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { given: 'a cm handle' mockYangModelCmHandleRetrieval([]) when: 'a get new yang resources from DMI is called with no module references' - def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, []) + def result = objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, []) then: 'no resources are returned' assert result == [:] and: 'no request is sent to DMI' @@ -191,21 +186,35 @@ class DmiModelOperationsSpec extends DmiOperationsBaseSpec { given: 'a cm handle' mockYangModelCmHandleRetrieval(null) when: 'a get new yang resources from DMI is called' - objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, [new ModuleReference('mod1', 'A')]) + objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, NO_MODULE_SET_TAG, [new ModuleReference('mod1', 'A')]) then: 'a null pointer is thrown (we might need to address this later)' thrown(NullPointerException) } - def 'Retrieving module references with Json processing exception.'() { - given: 'a cm handle' - mockYangModelCmHandleRetrieval([]) - and: 'a Json processing exception occurs' - spiedJsonObjectMapper.asJsonString(_) >> {throw (new JsonProcessingException('parsing error'))} - when: 'a DMI operation is executed' - objectUnderTest.getModuleReferences(yangModelCmHandle) - then: 'an ncmp exception is thrown' - def exceptionThrown = thrown(JsonProcessingException) - and: 'the message indicates a parsing error' - exceptionThrown.message.toLowerCase().contains('parsing error') + def 'Retrieving module references forwards the new module set tag to DMI during CM-handle upgrade.'() { + given: 'a cm handle with an existing module set tag' + mockYangModelCmHandleRetrieval([], 'OLD-TAG') + when: 'get module references is called' + objectUnderTest.getModuleReferences(yangModelCmHandle, 'NEW-TAG') + then: 'a request was sent to DMI with the NEW module set tag in the body' + 1 * mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> { args -> + def requestBodyAsJson = args[2] as String + assert requestBodyAsJson.contains('"moduleSetTag":"NEW-TAG"') + return new ResponseEntity([schemas: [[moduleName: 'mod1', revision: 'A'], [moduleName: 'mod2', revision: 'X']]], HttpStatus.OK) + } + } + + def 'Retrieving yang resources forwards the new module set tag to DMI during CM-handle upgrade.'() { + given: 'a cm handle with an existing module set tag' + mockYangModelCmHandleRetrieval([], 'OLD-TAG') + when: 'get new yang resources from DMI service' + objectUnderTest.getNewYangResourcesFromDmi(yangModelCmHandle, 'NEW-TAG', newModuleReferences) + then: 'a request was sent to DMI with the NEW module set tag in the body' + 1 * mockDmiRestClient.synchronousPostOperationWithJsonData(*_) >> { args -> + def requestBodyAsJson = args[2] as String + assert requestBodyAsJson.contains('"moduleSetTag":"NEW-TAG"') + return new ResponseEntity([[moduleName: 'mod1', revision: 'A', yangSource: 'some yang source'], + [moduleName: 'mod2', revision: 'X', yangSource: 'other yang source']], HttpStatus.OK) + } } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy index f8adfe5578..b4837f7bab 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy @@ -39,6 +39,8 @@ import static org.onap.cps.ncmp.api.inventory.models.LockReasonCategory.MODULE_U class ModuleSyncServiceSpec extends Specification { + def NO_MODULE_SET_TAG = '' + def mockCpsModuleService = Mock(CpsModuleService) def mockDmiModelOperations = Mock(DmiModelOperations) def mockCpsAnchorService = Mock(CpsAnchorService) @@ -53,9 +55,9 @@ class ModuleSyncServiceSpec extends Specification { def yangModelCmHandle = createAdvisedCmHandle(moduleSetTag) and: 'DMI operations returns some module references' def moduleReferences = [ new ModuleReference('module1','1'), new ModuleReference('module2','2') ] - mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences + mockDmiModelOperations.getModuleReferences(yangModelCmHandle, moduleSetTag) >> moduleReferences and: 'DMI-Plugin returns resource(s) for "new" module(s)' - mockDmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, identifiedNewModuleReferences) >> newModuleNameContentToMap + mockDmiModelOperations.getNewYangResourcesFromDmi(yangModelCmHandle, moduleSetTag, identifiedNewModuleReferences) >> newModuleNameContentToMap and: 'the module service identifies #identifiedNewModuleReferences.size() new modules' mockCpsModuleService.identifyNewModuleReferences(moduleReferences) >> identifiedNewModuleReferences when: 'module sync is triggered' @@ -90,20 +92,45 @@ class ModuleSyncServiceSpec extends Specification { 'without' | '' } - def 'Attempt Sync models for a cm handle with existing schema set (#exception).'() { + def 'Sync models for a cm handle with already defined exception upon schema set creation.'() { given: 'a cm handle to be synced' def yangModelCmHandle = createAdvisedCmHandle('existing tag') and: 'dmi returns no new yang resources' mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] and: 'already defined exception occurs when creating schema (existing)' - mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw exception } + mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw AlreadyDefinedException.forSchemaSet('', '', null) } when: 'module sync is triggered' objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) - then: 'no exception is thrown up' + then: 'the exception is ignored' noExceptionThrown() - where: 'following exceptions occur' - exception << [ AlreadyDefinedException.forSchemaSet('', '', null), - new DuplicatedYangResourceException('', '', null) ] + } + + def 'Sync models for a cm handle with already defined exception upon anchor set creation.'() { + given: 'a cm handle to be synced' + def yangModelCmHandle = createAdvisedCmHandle('existing tag') + and: 'dmi returns no new yang resources' + mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] + and: 'already defined exception occurs when creating schema (existing)' + mockCpsAnchorService.createAnchor(*_) >> { throw AlreadyDefinedException.forAnchor('', '', null) } + when: 'module sync is triggered' + objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) + then: 'the exception is ignored' + noExceptionThrown() + } + + def 'Attempt Sync models for a cm handle with duplicate yang resources exception).'() { + given: 'a cm handle to be synced' + def yangModelCmHandle = createAdvisedCmHandle('existing tag') + and: 'dmi returns no new yang resources' + mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] + and: 'duplicate yang resource exception occurs when creating schema' + def originalException = new DuplicatedYangResourceException('', '', null) + mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw originalException } + when: 'module sync is triggered' + objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) + then: 'same exception is thrown up' + def thrownException = thrown(Exception) + assert thrownException == originalException } def 'Model upgrade without using Module Set Tags (legacy) where the modules are in database.'() { @@ -115,8 +142,8 @@ class ModuleSyncServiceSpec extends Specification { def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(dmiServiceName, '', '', ncmpServiceCmHandle,'', '', '') and: 'DMI operations returns some module references for upgraded cm handle' def moduleReferences = [ new ModuleReference('module1','1') ] - mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences - mockDmiModelOperations.getNewYangResourcesFromDmi(_, []) >> [:] + mockDmiModelOperations.getModuleReferences(yangModelCmHandle, NO_MODULE_SET_TAG) >> moduleReferences + mockDmiModelOperations.getNewYangResourcesFromDmi(_, NO_MODULE_SET_TAG, []) >> [:] and: 'none of these module references are new (all already known to the system)' mockCpsModuleService.identifyNewModuleReferences(_) >> [] when: 'module sync is triggered' @@ -138,7 +165,7 @@ class ModuleSyncServiceSpec extends Specification { mockCpsModuleService.schemaSetExists(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, tagTo) >> schemaExists and: 'DMI operations returns some module references for upgraded cm handle' def moduleReferences = [ new ModuleReference('module1','1') ] - expectedCallsToDmi * mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences + expectedCallsToDmi * mockDmiModelOperations.getModuleReferences(yangModelCmHandle, tagTo) >> moduleReferences and: 'dmi returns no new yang resources' mockDmiModelOperations.getNewYangResourcesFromDmi(*_) >> [:] and: 'none of these module references are new (all already known to the system)' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy index 92f4b38f31..a2f38c89eb 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy @@ -39,8 +39,6 @@ import org.onap.cps.ncmp.impl.inventory.sync.lcm.LcmEventsCmHandleStateHandler import org.slf4j.LoggerFactory import spock.lang.Specification -import java.util.concurrent.atomic.AtomicInteger - import static org.onap.cps.ncmp.api.inventory.models.LockReasonCategory.MODULE_SYNC_FAILED import static org.onap.cps.ncmp.api.inventory.models.LockReasonCategory.MODULE_UPGRADE import static org.onap.cps.ncmp.api.inventory.models.LockReasonCategory.MODULE_UPGRADE_FAILED @@ -70,8 +68,6 @@ class ModuleSyncTasksSpec extends Specification { .getOrCreateHazelcastInstance(new Config('hazelcastInstanceName')) .getMap('mapInstanceName') - def batchCount = new AtomicInteger(5) - def objectUnderTest = new ModuleSyncTasks(mockInventoryPersistence, mockSyncUtils, mockModuleSyncService, mockLcmEventsCmHandleStateHandler, moduleSyncStartedOnCmHandles) @@ -87,7 +83,7 @@ class ModuleSyncTasksSpec extends Specification { mockInventoryPersistence.getYangModelCmHandle('cm-handle-1') >> cmHandle1 mockInventoryPersistence.getYangModelCmHandle('cm-handle-2') >> cmHandle2 when: 'module sync poll is executed' - objectUnderTest.performModuleSync(['cm-handle-1', 'cm-handle-2'], batchCount) + objectUnderTest.performModuleSync(['cm-handle-1', 'cm-handle-2']) then: 'module sync service is invoked for each cm handle' 1 * mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(_) >> { args -> assert args[0].id == 'cm-handle-1' } 1 * mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(_) >> { args -> assert args[0].id == 'cm-handle-2' } @@ -95,8 +91,6 @@ class ModuleSyncTasksSpec extends Specification { 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args -> assertBatch(args, ['cm-handle-1', 'cm-handle-2'], CmHandleState.READY) } - and: 'batch count is decremented by one' - assert batchCount.get() == 4 } def 'Handle CM handle failure during #scenario and log MODULE_UPGRADE lock reason'() { @@ -108,15 +102,13 @@ class ModuleSyncTasksSpec extends Specification { mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(_) >> { throw new Exception('some exception') } mockModuleSyncService.syncAndUpgradeSchemaSet(_) >> { throw new Exception('some exception') } when: 'module sync is executed' - objectUnderTest.performModuleSync(['cm-handle'], batchCount) + objectUnderTest.performModuleSync(['cm-handle']) then: 'lock reason is updated with number of attempts' 1 * mockSyncUtils.updateLockReasonWithAttempts(_, expectedLockReasonCategory, 'some exception') and: 'the state handler is called to update the state to LOCKED' 1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args -> assertBatch(args, ['cm-handle'], CmHandleState.LOCKED) } - and: 'batch count is decremented by one' - assert batchCount.get() == 4 where: scenario | lockReasonCategory | lockReasonDetails || expectedLockReasonCategory 'module sync' | MODULE_SYNC_FAILED | 'some lock details' || MODULE_SYNC_FAILED @@ -132,7 +124,7 @@ class ModuleSyncTasksSpec extends Specification { and: 'a cm handle in advised state' mockInventoryPersistence.getYangModelCmHandle('cm-handle-3') >> cmHandleByIdAndState('cm-handle-3', CmHandleState.ADVISED) when: 'module sync poll is executed' - objectUnderTest.performModuleSync(['cm-handle-1', 'cm-handle-2', 'cm-handle-3'], batchCount) + objectUnderTest.performModuleSync(['cm-handle-1', 'cm-handle-2', 'cm-handle-3']) then: 'no exception is thrown' noExceptionThrown() and: 'the deleted cm-handle did not sync' @@ -176,7 +168,7 @@ class ModuleSyncTasksSpec extends Specification { and: 'entry in progress map for other cm handle' moduleSyncStartedOnCmHandles.put('other-cm-handle', 'started') when: 'module sync poll is executed' - objectUnderTest.performModuleSync(['cm-handle-1'], batchCount) + objectUnderTest.performModuleSync(['cm-handle-1']) then: 'module sync service is invoked for cm handle' 1 * mockModuleSyncService.syncAndCreateSchemaSetAndAnchor(_) >> { args -> assert args[0].id == 'cm-handle-1' } and: 'the entry for other cm handle is still in the progress map' @@ -192,7 +184,7 @@ class ModuleSyncTasksSpec extends Specification { def loggingEvent = getLoggingEvent() assert loggingEvent.level == Level.INFO and: 'the log indicates the cm handle entry is removed successfully' - assert loggingEvent.formattedMessage == 'ch-1 will be removed asynchronously from in progress map' + assert loggingEvent.formattedMessage == 'ch-1 removed from in progress map' } def 'Sync and upgrade CM handle if in upgrade state for #scenario'() { @@ -201,7 +193,7 @@ class ModuleSyncTasksSpec extends Specification { cmHandle.compositeState.setLockReason(CompositeState.LockReason.builder().lockReasonCategory(lockReasonCategory).build()) mockInventoryPersistence.getYangModelCmHandle('cm-handle') >> cmHandle when: 'module sync is executed' - objectUnderTest.performModuleSync(['cm-handle'], batchCount) + objectUnderTest.performModuleSync(['cm-handle']) then: 'the module sync service should attempt to sync and upgrade the CM handle' 1 * mockModuleSyncService.syncAndUpgradeSchemaSet(_) >> { args -> assert args[0].id == 'cm-handle' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy index a9b88c2d3b..68aa6a1b6a 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2024 Nordix Foundation + * Copyright (C) 2022-2025 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,14 +22,10 @@ package org.onap.cps.ncmp.impl.inventory.sync import com.hazelcast.map.IMap +import java.util.concurrent.ArrayBlockingQueue import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle -import org.onap.cps.ncmp.impl.utils.Sleeper -import org.onap.cps.api.model.DataNode import spock.lang.Specification -import java.util.concurrent.ArrayBlockingQueue -import java.util.concurrent.locks.Lock - class ModuleSyncWatchdogSpec extends Specification { def mockModuleOperationsUtils = Mock(ModuleOperationsUtils) @@ -42,17 +38,9 @@ class ModuleSyncWatchdogSpec extends Specification { def mockModuleSyncTasks = Mock(ModuleSyncTasks) - def spiedAsyncTaskExecutor = Spy(AsyncTaskExecutor) - def mockCpsAndNcmpLock = Mock(IMap<String,String>) - def spiedSleeper = Spy(Sleeper) - - def objectUnderTest = new ModuleSyncWatchdog(mockModuleOperationsUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, spiedAsyncTaskExecutor, mockCpsAndNcmpLock, spiedSleeper) - - void setup() { - spiedAsyncTaskExecutor.setupThreadPool() - } + def objectUnderTest = new ModuleSyncWatchdog(mockModuleOperationsUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, mockCpsAndNcmpLock) def 'Module sync advised cm handles with #scenario.'() { given: 'module sync utilities returns #numberOfAdvisedCmHandles advised cm handles' @@ -61,12 +49,10 @@ class ModuleSyncWatchdogSpec extends Specification { mockModuleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> [] and: 'the work queue can be locked' mockCpsAndNcmpLock.tryLock('workQueueLock') >> true - and: 'the executor has enough available threads' - spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 3 when: ' module sync is started' objectUnderTest.moduleSyncAdvisedCmHandles() then: 'it performs #expectedNumberOfTaskExecutions tasks' - expectedNumberOfTaskExecutions * spiedAsyncTaskExecutor.executeTask(*_) + expectedNumberOfTaskExecutions * mockModuleSyncTasks.performModuleSync(*_) and: 'the executing thread is unlocked' 1 * mockCpsAndNcmpLock.unlock('workQueueLock') where: 'the following parameter are used' @@ -84,12 +70,10 @@ class ModuleSyncWatchdogSpec extends Specification { mockModuleOperationsUtils.getAdvisedCmHandleIds() >> createCmHandleIds(1) and: 'the work queue can be locked' mockCpsAndNcmpLock.tryLock('workQueueLock') >> true - and: 'the executor first has no threads but has one thread on the second attempt' - spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >>> [ 0, 1 ] when: ' module sync is started' objectUnderTest.moduleSyncAdvisedCmHandles() then: 'it performs one task' - 1 * spiedAsyncTaskExecutor.executeTask(*_) + 1 * mockModuleSyncTasks.performModuleSync(*_) } def 'Module sync advised cm handle already handled by other thread.'() { @@ -97,27 +81,21 @@ class ModuleSyncWatchdogSpec extends Specification { mockModuleOperationsUtils.getAdvisedCmHandleIds() >> createCmHandleIds(1) and: 'the work queue can be locked' mockCpsAndNcmpLock.tryLock('workQueueLock') >> true - and: 'the executor has a thread available' - spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 1 and: 'the semaphore cache indicates the cm handle is already being processed' mockModuleSyncStartedOnCmHandles.putIfAbsent(*_) >> 'Started' - when: ' module sync is started' + when: 'module sync is started' objectUnderTest.moduleSyncAdvisedCmHandles() then: 'it does NOT execute a task to process the (empty) batch' - 0 * spiedAsyncTaskExecutor.executeTask(*_) + 0 * mockModuleSyncTasks.performModuleSync(*_) } def 'Module sync with previous cm handle(s) left in work queue.'() { given: 'there is still a cm handle in the queue' moduleSyncWorkQueue.offer('ch-1') - and: 'sync utilities returns many advise cm handles' - mockModuleOperationsUtils.getAdvisedCmHandleIds() >> createCmHandleIds(500) - and: 'the executor has plenty threads available' - spiedAsyncTaskExecutor.getAsyncTaskParallelismLevel() >> 10 - when: ' module sync is started' + when: 'module sync is started' objectUnderTest.moduleSyncAdvisedCmHandles() then: 'it does executes only one task to process the remaining handle in the queue' - 1 * spiedAsyncTaskExecutor.executeTask(*_) + 1 * mockModuleSyncTasks.performModuleSync(*_) } def 'Reset failed cm handles.'() { @@ -147,15 +125,6 @@ class ModuleSyncWatchdogSpec extends Specification { true || false || 1 } - def 'Sleeper gets interrupted.'() { - given: 'sleeper gets interrupted' - spiedSleeper.haveALittleRest(_) >> { throw new InterruptedException() } - when: 'the watchdog attempts to sleep to save cpu cycles' - objectUnderTest.preventBusyWait() - then: 'no exception is thrown' - noExceptionThrown() - } - def createCmHandleIds(numberOfCmHandles) { return (numberOfCmHandles > 0) ? (1..numberOfCmHandles).collect { 'ch-'+it } : [] } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/utils/events/MessagingBaseSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/utils/events/MessagingBaseSpec.groovy index 377a1a6637..d38d5442f2 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/utils/events/MessagingBaseSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/utils/events/MessagingBaseSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (c) 2023 Nordix Foundation. + * Copyright (c) 2023-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,8 +29,7 @@ import org.springframework.kafka.core.KafkaTemplate import org.springframework.kafka.support.serializer.JsonSerializer import org.springframework.test.context.DynamicPropertyRegistry import org.springframework.test.context.DynamicPropertySource -import org.testcontainers.containers.KafkaContainer -import org.testcontainers.utility.DockerImageName +import org.testcontainers.kafka.ConfluentKafkaContainer import spock.lang.Specification class MessagingBaseSpec extends Specification { @@ -43,7 +42,7 @@ class MessagingBaseSpec extends Specification { kafkaTestContainer.stop() } - static kafkaTestContainer = new KafkaContainer(DockerImageName.parse('registry.nordix.org/onaptest/confluentinc/cp-kafka:6.2.1').asCompatibleSubstituteFor('confluentinc/cp-kafka')) + static kafkaTestContainer = new ConfluentKafkaContainer("confluentinc/cp-kafka:7.8.0") def legacyEventKafkaTemplate = new KafkaTemplate<>(new DefaultKafkaProducerFactory<String, String>(eventProducerConfigProperties(JsonSerializer))) diff --git a/cps-ncmp-service/src/test/resources/application.yml b/cps-ncmp-service/src/test/resources/application.yml index 12db639633..3276ceb534 100644 --- a/cps-ncmp-service/src/test/resources/application.yml +++ b/cps-ncmp-service/src/test/resources/application.yml @@ -77,10 +77,6 @@ ncmp: trust-level: dmi-availability-watchdog-ms: 30000 - modules-sync-watchdog: - async-executor: - parallelism-level: 3 - policy-executor: enabled: true defaultDecision: "some default decision" diff --git a/cps-parent/pom.xml b/cps-parent/pom.xml index 554cc63956..88097f219f 100644 --- a/cps-parent/pom.xml +++ b/cps-parent/pom.xml @@ -27,7 +27,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <packaging>pom</packaging> <properties> diff --git a/cps-path-parser/pom.xml b/cps-path-parser/pom.xml index 469357462e..e30c728053 100644 --- a/cps-path-parser/pom.xml +++ b/cps-path-parser/pom.xml @@ -23,7 +23,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-rest/docs/openapi/cpsAdmin.yml b/cps-rest/docs/openapi/cpsAdmin.yml index f394270dd5..6cfffa48f3 100644 --- a/cps-rest/docs/openapi/cpsAdmin.yml +++ b/cps-rest/docs/openapi/cpsAdmin.yml @@ -1,6 +1,6 @@ # ============LICENSE_START======================================================= # Copyright (c) 2021 Bell Canada. -# Modifications Copyright (C) 2021-2022 Nordix Foundation +# Modifications Copyright (C) 2021-2025 Nordix Foundation # Modifications Copyright (C) 2022 TechMahindra Ltd. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); @@ -230,3 +230,23 @@ adminDataspace: $ref: 'components.yml#/components/responses/Forbidden' '500': $ref: 'components.yml#/components/responses/InternalServerError' + +adminCleanDataspace: + post: + description: Clean the dataspace (remove orphaned schema sets and modules) + tags: + - cps-admin + summary: Clean the dataspace + operationId: cleanDataspace + parameters: + - $ref: 'components.yml#/components/parameters/apiVersionInPath' + - $ref: 'components.yml#/components/parameters/dataspaceNameInPath' + responses: + '204': + $ref: 'components.yml#/components/responses/NoContent' + '400': + $ref: 'components.yml#/components/responses/BadRequest' + '403': + $ref: 'components.yml#/components/responses/Forbidden' + '500': + $ref: 'components.yml#/components/responses/InternalServerError' diff --git a/cps-rest/docs/openapi/openapi.yml b/cps-rest/docs/openapi/openapi.yml index f4eab61875..c85bf7cac7 100644 --- a/cps-rest/docs/openapi/openapi.yml +++ b/cps-rest/docs/openapi/openapi.yml @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (C) 2021-2024 Nordix Foundation +# Copyright (C) 2021-2025 Nordix Foundation # Modifications Copyright (C) 2021 Pantheon.tech # Modifications Copyright (C) 2021 Bell Canada. # Modifications Copyright (C) 2022-2024 TechMahindra Ltd. @@ -61,6 +61,9 @@ paths: /{apiVersion}/admin/dataspaces/{dataspace-name}: $ref: 'cpsAdmin.yml#/adminDataspace' + /{apiVersion}/admin/dataspaces/{dataspace-name}/actions/clean: + $ref: 'cpsAdmin.yml#/adminCleanDataspace' + /v1/dataspaces/{dataspace-name}/anchors: $ref: 'cpsAdminV1Deprecated.yml#/anchorsByDataspace' diff --git a/cps-rest/pom.xml b/cps-rest/pom.xml index b04daf03bd..4e52b1b794 100644 --- a/cps-rest/pom.xml +++ b/cps-rest/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> @@ -62,10 +62,6 @@ <artifactId>spring-boot-starter-jetty</artifactId> </dependency> <dependency> - <groupId>org.springframework.retry</groupId> - <artifactId>spring-retry</artifactId> - </dependency> - <dependency> <groupId>org.springframework</groupId> <artifactId>spring-aspects</artifactId> </dependency> diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java index 675c0eaec4..4c6bd6cdc5 100755 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2023 Nordix Foundation + * Copyright (C) 2020-2025 Nordix Foundation * Modifications Copyright (C) 2020-2021 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -176,6 +176,20 @@ public class AdminRestController implements CpsAdminApi { } /** + * Clean the given dataspace of any orphaned (module) data. + * + * @param apiVersion api version + * @param dataspaceName dataspace name + * + * @return a {@Link ResponseEntity} of {@link HttpStatus} NO_CONTENT + */ + @Override + public ResponseEntity<Void> cleanDataspace(final String apiVersion, final String dataspaceName) { + cpsModuleService.deleteAllUnusedYangModuleData(dataspaceName); + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + /** * Create a new anchor. * * @param dataspaceName dataspace name diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy index 2335a5e770..0d189783fd 100755 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2020-2021 Bell Canada. - * Modifications Copyright (C) 2021-2022 Nordix Foundation + * Modifications Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,15 +23,8 @@ package org.onap.cps.rest.controller -import org.onap.cps.api.CpsAnchorService - -import static org.onap.cps.api.parameters.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post - import org.mapstruct.factory.Mappers +import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataspaceService import org.onap.cps.api.CpsModuleService import org.onap.cps.api.exceptions.AlreadyDefinedException @@ -51,6 +44,12 @@ import org.springframework.util.LinkedMultiValueMap import org.springframework.util.MultiValueMap import spock.lang.Specification +import static org.onap.cps.api.parameters.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post + @WebMvcTest(AdminRestController) class AdminRestControllerSpec extends Specification { @@ -79,7 +78,7 @@ class AdminRestControllerSpec extends Specification { def dataspace = new Dataspace(name: dataspaceName) def 'Create new dataspace with #scenario.'() { - when: 'post is invoked' + when: 'post is invoked on endpoint for creating a dataspace' def response = mvc.perform( post("/cps/api/${apiVersion}/dataspaces") @@ -97,7 +96,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create dataspace over existing with same name.'() { - given: 'an endpoint' + given: 'the endpoint to create a dataspace' def createDataspaceEndpoint = "$basePath/v1/dataspaces" and: 'the service method throws an exception indicating the dataspace is already defined' def thrownException = new AlreadyDefinedException(dataspaceName, new RuntimeException()) @@ -115,7 +114,7 @@ class AdminRestControllerSpec extends Specification { def 'Get a dataspace.'() { given: 'service method returns a dataspace' mockCpsDataspaceService.getDataspace(dataspaceName) >> dataspace - and: 'an endpoint' + and: 'the endpoint for getting a dataspace by name' def getDataspaceEndpoint = "$basePath/v1/admin/dataspaces/$dataspaceName" when: 'get dataspace API is invoked' def response = mvc.perform(get(getDataspaceEndpoint)).andReturn().response @@ -124,6 +123,17 @@ class AdminRestControllerSpec extends Specification { response.getContentAsString().contains(dataspaceName) } + def 'Clean a dataspace.'() { + given: 'service method returns a dataspace' + mockCpsDataspaceService.getDataspace(dataspaceName) >> dataspace + and: 'the endpoint for cleaning a dataspace' + def postCleanDataspaceEndpoint = "$basePath/v1/admin/dataspaces/$dataspaceName/actions/clean" + when: 'post is invoked on the clean dataspace endpoint' + def response = mvc.perform(post(postCleanDataspaceEndpoint)).andReturn().response + then: 'no content is returned' + response.status == HttpStatus.NO_CONTENT.value() + } + def 'Get all dataspaces.'() { given: 'service method returns all dataspace' mockCpsDataspaceService.getAllDataspaces() >> [dataspace, new Dataspace(name: "dataspace-test2")] @@ -173,8 +183,7 @@ class AdminRestControllerSpec extends Specification { .param('schema-set-name', schemaSetName)) .andReturn().response then: 'associated service method is invoked with expected parameters' - 1 * mockCpsModuleService.createSchemaSet(dataspaceName, schemaSetName, _) >> - { args -> yangResourceMapCapture = args[2] } + 1 * mockCpsModuleService.createSchemaSet(dataspaceName, schemaSetName, _) >> { args -> yangResourceMapCapture = args[2] } yangResourceMapCapture['assembly.yang'] == "fake assembly content 1\n" yangResourceMapCapture['component.yang'] == "fake component content 1\n" and: 'response code indicates success' @@ -208,7 +217,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create schema set from zip archive having #caseDescriptor.'() { - given: 'an endpoint' + given: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'zip archive having #caseDescriptor is uploaded with create schema set request' def response = @@ -228,7 +237,7 @@ class AdminRestControllerSpec extends Specification { def 'Create schema set from file with unsupported filename extension.'() { given: 'file with unsupported filename extension (.doc)' def multipartFile = createMultipartFile("filename.doc", "content") - and: 'an endpoint' + and: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'file uploaded with schema set create request' def response = @@ -242,7 +251,7 @@ class AdminRestControllerSpec extends Specification { } def 'Create schema set from #fileType file with IOException occurrence on processing.'() { - given: 'an endpoint' + given: 'the endpoint to create a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'file uploaded with schema set create request' def multipartFile = createMultipartFileForIOException(fileType) @@ -259,7 +268,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete schema set.'() { - given: 'an endpoint' + given: 'the endpoint for deleting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'delete schema set endpoint is invoked' def response = mvc.perform(delete(schemaSetEndpoint)).andReturn().response @@ -274,7 +283,7 @@ class AdminRestControllerSpec extends Specification { def thrownException = new SchemaSetInUseException(dataspaceName, schemaSetName) mockCpsModuleService.deleteSchemaSet(dataspaceName, schemaSetName, CASCADE_DELETE_PROHIBITED) >> { throw thrownException } - and: 'an endpoint' + and: 'the endpoint for deleting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'delete schema set endpoint is invoked' def response = mvc.perform(delete(schemaSetEndpoint)).andReturn().response @@ -286,7 +295,7 @@ class AdminRestControllerSpec extends Specification { given: 'service method returns a new schema set' mockCpsModuleService.getSchemaSet(dataspaceName, schemaSetName) >> new SchemaSet(name: schemaSetName, dataspaceName: dataspaceName) - and: 'an endpoint' + and: 'the endpoint for getting a schema set' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets/$schemaSetName" when: 'get schema set API is invoked' def response = mvc.perform(get(schemaSetEndpoint)).andReturn().response @@ -300,7 +309,7 @@ class AdminRestControllerSpec extends Specification { mockCpsModuleService.getSchemaSets(dataspaceName) >> [new SchemaSet(name: schemaSetName, dataspaceName: dataspaceName), new SchemaSet(name: "test-schemaset", dataspaceName: dataspaceName)] - and: 'an endpoint' + and: 'the endpoint for getting all schema sets' def schemaSetEndpoint = "$basePath/v1/dataspaces/$dataspaceName/schema-sets" when: 'get schema sets API is invoked' def response = mvc.perform(get(schemaSetEndpoint)).andReturn().response @@ -315,7 +324,7 @@ class AdminRestControllerSpec extends Specification { def requestParams = new LinkedMultiValueMap<>() requestParams.add('schema-set-name', schemaSetName) requestParams.add('anchor-name', anchorName) - when: 'post is invoked' + when: 'post is invoked on the create anchors endpoint' def response = mvc.perform( post("/cps/api/${apiVersion}/dataspaces/my_dataspace/anchors") @@ -332,10 +341,10 @@ class AdminRestControllerSpec extends Specification { 'V2 API' | 'v2' || '' } - def 'Get existing anchor.'() { - given: 'service method returns a list of anchors' + def 'Get existing anchors.'() { + given: 'service method returns a list of (one) anchors' mockCpsAnchorService.getAnchors(dataspaceName) >> [anchor] - and: 'an endpoint' + and: 'the endpoint for getting all anchors' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors" when: 'get all anchors API is invoked' def response = mvc.perform(get(anchorEndpoint)).andReturn().response @@ -348,7 +357,7 @@ class AdminRestControllerSpec extends Specification { given: 'service method returns an anchor' mockCpsAnchorService.getAnchor(dataspaceName, anchorName) >> new Anchor(name: anchorName, dataspaceName: dataspaceName, schemaSetName: schemaSetName) - and: 'an endpoint' + and: 'the endpoint for getting an anchor' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors/$anchorName" when: 'get anchor API is invoked' def response = mvc.perform(get(anchorEndpoint)).andReturn().response @@ -361,7 +370,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete anchor.'() { - given: 'an endpoint' + given: 'the endpoint for deleting an anchor' def anchorEndpoint = "$basePath/v1/dataspaces/$dataspaceName/anchors/$anchorName" when: 'delete method is invoked on anchor endpoint' def response = mvc.perform(delete(anchorEndpoint)).andReturn().response @@ -372,7 +381,7 @@ class AdminRestControllerSpec extends Specification { } def 'Delete dataspace.'() { - given: 'an endpoint' + given: 'the endpoint for deleting a dataspace' def dataspaceEndpoint = "$basePath/v1/dataspaces" when: 'delete dataspace endpoint is invoked' def response = mvc.perform(delete(dataspaceEndpoint) diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy index ca89fafe83..f2f962422f 100755 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/DataRestControllerSpec.groovy @@ -30,8 +30,8 @@ import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder -import org.onap.cps.api.model.DeltaReportBuilder +import org.onap.cps.impl.DataNodeBuilder +import org.onap.cps.impl.DeltaReportBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.DateTimeUtility import org.onap.cps.utils.JsonObjectMapper diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy index f29654c99f..2b5c471287 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsQueryService import org.onap.cps.api.parameters.PaginationOption -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.JsonObjectMapper import org.onap.cps.utils.PrefixResolver import org.spockframework.spring.SpringBean diff --git a/cps-ri/pom.xml b/cps-ri/pom.xml index 2492cb837e..aa5fad9ac2 100644 --- a/cps-ri/pom.xml +++ b/cps-ri/pom.xml @@ -3,7 +3,7 @@ ============LICENSE_START=======================================================
Copyright (C) 2020-2021 Pantheon.tech
Modifications Copyright (C) 2020-2021 Bell Canada
- Modifications Copyright (C) 2020-2024 Nordix Foundation
+ Modifications Copyright (C) 2020-2025 Nordix Foundation
================================================================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -26,14 +26,14 @@ <parent>
<groupId>org.onap.cps</groupId>
<artifactId>cps-parent</artifactId>
- <version>3.6.0-SNAPSHOT</version>
+ <version>3.6.1-SNAPSHOT</version>
<relativePath>../cps-parent/pom.xml</relativePath>
</parent>
<artifactId>cps-ri</artifactId>
<properties>
- <minimum-coverage>0.29</minimum-coverage>
+ <minimum-coverage>0.31</minimum-coverage>
<!-- Additional coverage is provided by integration-test module -->
</properties>
@@ -56,10 +56,6 @@ <artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<dependency>
- <groupId>org.springframework.retry</groupId>
- <artifactId>spring-retry</artifactId>
- </dependency>
- <dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aspects</artifactId>
</dependency>
diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java index 494d6919da..588a639ab8 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2024 Nordix Foundation. + * Copyright (C) 2020-2025 Nordix Foundation. * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -91,10 +91,10 @@ public class CpsAdminPersistenceServiceImpl implements CpsAdminPersistenceServic @Override public void createAnchor(final String dataspaceName, final String schemaSetName, final String anchorName) { - final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName); - final var schemaSetEntity = - schemaSetRepository.getByDataspaceAndName(dataspaceEntity, schemaSetName); - final var anchorEntity = AnchorEntity.builder() + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final SchemaSetEntity schemaSetEntity = schemaSetRepository + .getByDataspaceAndName(dataspaceEntity, schemaSetName); + final AnchorEntity anchorEntity = AnchorEntity.builder() .name(anchorName) .dataspace(dataspaceEntity) .schemaSet(schemaSetEntity) @@ -114,7 +114,7 @@ public class CpsAdminPersistenceServiceImpl implements CpsAdminPersistenceServic @Override public Collection<Anchor> getAnchors(final String dataspaceName) { - final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); final Collection<AnchorEntity> anchorEntities = anchorRepository.findAllByDataspace(dataspaceEntity); return anchorEntities.stream().map(CpsAdminPersistenceServiceImpl::toAnchor).collect(Collectors.toSet()); } @@ -154,14 +154,14 @@ public class CpsAdminPersistenceServiceImpl implements CpsAdminPersistenceServic @Transactional @Override public void deleteAnchor(final String dataspaceName, final String anchorName) { - final var anchorEntity = getAnchorEntity(dataspaceName, anchorName); + final AnchorEntity anchorEntity = getAnchorEntity(dataspaceName, anchorName); anchorRepository.delete(anchorEntity); } @Transactional @Override public void deleteAnchors(final String dataspaceName, final Collection<String> anchorNames) { - final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); anchorRepository.deleteAllByDataspaceAndNameIn(dataspaceEntity, anchorNames); } @@ -178,7 +178,7 @@ public class CpsAdminPersistenceServiceImpl implements CpsAdminPersistenceServic } private AnchorEntity getAnchorEntity(final String dataspaceName, final String anchorName) { - final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); return anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java index c43c8e2999..e102765a64 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2022-2023 TechMahindra Ltd. @@ -23,6 +23,7 @@ package org.onap.cps.ri; +import static org.onap.cps.api.CpsQueryService.NO_LIMIT; import static org.onap.cps.api.parameters.PaginationOption.NO_PAGINATION; import com.google.common.collect.ImmutableSet; @@ -37,6 +38,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -50,12 +52,12 @@ import org.onap.cps.api.exceptions.CpsPathException; import org.onap.cps.api.exceptions.DataNodeNotFoundException; import org.onap.cps.api.exceptions.DataNodeNotFoundExceptionBatch; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.api.parameters.PaginationOption; import org.onap.cps.cpspath.parser.CpsPathQuery; import org.onap.cps.cpspath.parser.CpsPathUtil; import org.onap.cps.cpspath.parser.PathParsingException; +import org.onap.cps.impl.DataNodeBuilder; import org.onap.cps.ri.models.AnchorEntity; import org.onap.cps.ri.models.DataspaceEntity; import org.onap.cps.ri.models.FragmentEntity; @@ -222,14 +224,44 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService description = "Time taken to query data nodes") public List<DataNode> queryDataNodes(final String dataspaceName, final String anchorName, final String cpsPath, final FetchDescendantsOption fetchDescendantsOption) { + return queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption, NO_LIMIT); + } + + @Override + public List<DataNode> queryDataNodes(final String dataspaceName, + final String anchorName, + final String cpsPath, + final FetchDescendantsOption fetchDescendantsOption, + final int queryResultLimit) { final AnchorEntity anchorEntity = getAnchorEntity(dataspaceName, anchorName); final CpsPathQuery cpsPathQuery = getCpsPathQuery(cpsPath); final Collection<FragmentEntity> fragmentEntities = - fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery); + fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery, queryResultLimit); return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); } @Override + public <T> Set<T> queryDataLeaf(final String dataspaceName, final String anchorName, final String cpsPath, + final Class<T> targetClass) { + final CpsPathQuery cpsPathQuery = getCpsPathQuery(cpsPath); + if (!cpsPathQuery.hasAttributeAxis()) { + throw new IllegalArgumentException( + "Only Cps Path Queries with attribute-axis are supported by queryDataLeaf"); + } + + final String attributeName = cpsPathQuery.getAttributeAxisAttributeName(); + final List<DataNode> dataNodes = queryDataNodes(dataspaceName, anchorName, cpsPath, + FetchDescendantsOption.OMIT_DESCENDANTS); + return dataNodes.stream() + .map(dataNode -> { + final Object attributeValue = dataNode.getLeaves().get(attributeName); + return targetClass.isInstance(attributeValue) ? targetClass.cast(attributeValue) : null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + } + + @Override @Timed(value = "cps.data.persistence.service.datanode.query.anchors", description = "Time taken to query data nodes across all anchors or list of anchors") public List<DataNode> queryDataNodesAcrossAnchors(final String dataspaceName, final String cpsPath, diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java index cf9fb021a6..dbc6c28ec5 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java @@ -36,7 +36,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -70,10 +69,6 @@ import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource; import org.opendaylight.yangtools.yang.parser.api.YangSyntaxErrorException; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.YangModelDependencyInfo; import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.retry.RetryContext; -import org.springframework.retry.annotation.Backoff; -import org.springframework.retry.annotation.Retryable; -import org.springframework.retry.support.RetrySynchronizationManager; import org.springframework.stereotype.Component; @Slf4j @@ -107,10 +102,9 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override public Collection<ModuleReference> getYangResourceModuleReferences(final String dataspaceName) { - final Set<YangResourceModuleReference> yangResourceModuleReferenceList = + final Collection<YangResourceModuleReference> yangResourceModuleReferences = yangResourceRepository.findAllModuleReferencesByDataspace(dataspaceName); - return yangResourceModuleReferenceList.stream().map(CpsModulePersistenceServiceImpl::toModuleReference) - .collect(Collectors.toList()); + return yangResourceModuleReferences.stream().map(CpsModulePersistenceServiceImpl::toModuleReference).toList(); } @Override @@ -154,23 +148,24 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - // A retry is made to store the schema set if it fails because of duplicated yang resource exception that - // can occur in case of specific concurrent requests. - @Retryable(retryFor = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = - @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) - public void storeSchemaSet(final String dataspaceName, final String schemaSetName, - final Map<String, String> moduleReferenceNameToContentMap) { - final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); - final Set<YangResourceEntity> yangResourceEntities = synchronizeYangResources(moduleReferenceNameToContentMap); - final SchemaSetEntity schemaSetEntity = new SchemaSetEntity(); - schemaSetEntity.setName(schemaSetName); - schemaSetEntity.setDataspace(dataspaceEntity); - schemaSetEntity.setYangResources(yangResourceEntities); - try { - schemaSetRepository.save(schemaSetEntity); - } catch (final DataIntegrityViolationException e) { - throw AlreadyDefinedException.forSchemaSet(schemaSetName, dataspaceName, e); - } + @Timed(value = "cps.module.persistence.schemaset.create", + description = "Time taken to store a schemaset (list of module references)") + public void createSchemaSet(final String dataspaceName, final String schemaSetName, + final Map<String, String> yangResourceContentPerName) { + final Set<YangResourceEntity> yangResourceEntities = synchronizeYangResources(yangResourceContentPerName); + createAndSaveSchemaSetEntity(dataspaceName, schemaSetName, yangResourceEntities); + } + + @Override + @Transactional + @Timed(value = "cps.module.persistence.schemaset.createFromNewAndExistingModules", + description = "Time taken to store a schemaset (from new and existing)") + public void createSchemaSetFromNewAndExistingModules(final String dataspaceName, final String schemaSetName, + final Map<String, String> newYangResourceContentPerName, + final Collection<ModuleReference> allModuleReferences) { + synchronizeYangResources(newYangResourceContentPerName); + final Set<YangResourceEntity> allYangResourceEntities = getYangResourceEntities(allModuleReferences); + createAndSaveSchemaSetEntity(dataspaceName, schemaSetName, allYangResourceEntities); } @Override @@ -189,26 +184,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - // A retry is made to store the schema set if it fails because of duplicated yang resource exception that - // can occur in case of specific concurrent requests. - @Retryable(retryFor = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = - @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) - @Timed(value = "cps.module.persistence.schemaset.store", - description = "Time taken to store a schemaset (list of module references") - public void storeSchemaSetFromModules(final String dataspaceName, final String schemaSetName, - final Map<String, String> newModuleNameToContentMap, - final Collection<ModuleReference> allModuleReferences) { - storeSchemaSet(dataspaceName, schemaSetName, newModuleNameToContentMap); - final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); - final SchemaSetEntity schemaSetEntity = - schemaSetRepository.getByDataspaceAndName(dataspaceEntity, schemaSetName); - final List<Integer> allYangResourceIds = - yangResourceRepository.getResourceIdsByModuleReferences(allModuleReferences); - yangResourceRepository.insertSchemaSetIdYangResourceId(schemaSetEntity.getId(), allYangResourceIds); - } - - @Override - @Transactional public void deleteSchemaSet(final String dataspaceName, final String schemaSetName) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); final SchemaSetEntity schemaSetEntity = @@ -225,20 +200,23 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Override @Transactional - public void updateSchemaSetFromModules(final String dataspaceName, final String schemaSetName, - final Map<String, String> newModuleNameToContentMap, - final Collection<ModuleReference> allModuleReferences) { + public void updateSchemaSetFromNewAndExistingModules(final String dataspaceName, final String schemaSetName, + final Map<String, String> newYangResourcesPerName, + final Collection<ModuleReference> allModuleReferences) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); final SchemaSetEntity schemaSetEntity = schemaSetRepository.getByDataspaceAndName(dataspaceEntity, schemaSetName); - storeAndLinkNewModules(newModuleNameToContentMap, schemaSetEntity); - updateAllModuleReferences(allModuleReferences, schemaSetEntity.getId()); + synchronizeYangResources(newYangResourcesPerName); + final Set<YangResourceEntity> allYangResourceEntities = getYangResourceEntities(allModuleReferences); + schemaSetEntity.setYangResources(allYangResourceEntities); + schemaSetRepository.save(schemaSetEntity); } @Override @Transactional - public void deleteAllUnusedYangModuleData() { - schemaSetRepository.deleteOrphanedSchemaSets(); + public void deleteAllUnusedYangModuleData(final String dataspaceName) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + schemaSetRepository.deleteOrphanedSchemaSets(dataspaceEntity.getId()); yangResourceRepository.deleteOrphanedYangResources(); } @@ -248,9 +226,35 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ return moduleReferenceRepository.identifyNewModuleReferences(moduleReferencesToCheck); } - private Set<YangResourceEntity> synchronizeYangResources( - final Map<String, String> moduleReferenceNameToContentMap) { - final Map<String, YangResourceEntity> checksumToEntityMap = moduleReferenceNameToContentMap.entrySet().stream() + private Set<YangResourceEntity> synchronizeYangResources(final Map<String, String> yangResourceContentPerName) { + final Map<String, YangResourceEntity> yangResourceEntitiesPerChecksum = + getYangResourceEntityPerChecksum(yangResourceContentPerName); + + final List<YangResourceEntity> existingYangResourceEntities = + yangResourceRepository.findAllByChecksumIn(yangResourceEntitiesPerChecksum.keySet()); + + existingYangResourceEntities.forEach(exist -> yangResourceEntitiesPerChecksum.remove(exist.getChecksum())); + final Collection<YangResourceEntity> newYangResourceEntities = yangResourceEntitiesPerChecksum.values(); + + if (!newYangResourceEntities.isEmpty()) { + try { + yangResourceRepository.saveAll(newYangResourceEntities); + } catch (final DataIntegrityViolationException dataIntegrityViolationException) { + convertExceptionIfNeeded(dataIntegrityViolationException, newYangResourceEntities); + } + } + + // return ALL yang resourceEntities + return ImmutableSet.<YangResourceEntity>builder() + .addAll(existingYangResourceEntities) + .addAll(newYangResourceEntities) + .build(); + } + + private static Map<String, YangResourceEntity> getYangResourceEntityPerChecksum( + final Map<String, String> yangResourceContentPerName) { + final Map<String, YangResourceEntity> yangResourceEntityPerChecksum = + yangResourceContentPerName.entrySet().stream() .map(entry -> { final String checksum = DigestUtils.sha256Hex(entry.getValue().getBytes(StandardCharsets.UTF_8)); final Map<String, String> moduleNameAndRevisionMap = createModuleNameAndRevisionMap(entry.getKey(), @@ -267,37 +271,22 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ YangResourceEntity::getChecksum, entity -> entity )); + return yangResourceEntityPerChecksum; + } - final List<YangResourceEntity> existingYangResourceEntities = - yangResourceRepository.findAllByChecksumIn(checksumToEntityMap.keySet()); - existingYangResourceEntities.forEach(yangFile -> checksumToEntityMap.remove(yangFile.getChecksum())); - - final Collection<YangResourceEntity> newYangResourceEntities = checksumToEntityMap.values(); - if (!newYangResourceEntities.isEmpty()) { - try { - yangResourceRepository.saveAll(newYangResourceEntities); - } catch (final DataIntegrityViolationException dataIntegrityViolationException) { - // Throw a CPS duplicated Yang resource exception if the cause of the error is a yang checksum - // database constraint violation. - // If it is not, then throw the original exception - final Optional<DuplicatedYangResourceException> convertedException = - convertToDuplicatedYangResourceException( - dataIntegrityViolationException, newYangResourceEntities); - convertedException.ifPresent( - e -> { - final RetryContext retryContext = RetrySynchronizationManager.getContext(); - int retryCount = retryContext == null ? 0 : retryContext.getRetryCount(); - log.warn("Cannot persist duplicated yang resource. System will attempt this method " - + "up to 5 times. Current retry count : {}", ++retryCount, e); - }); - throw convertedException.isPresent() ? convertedException.get() : dataIntegrityViolationException; - } + private void createAndSaveSchemaSetEntity(final String dataspaceName, + final String schemaSetName, + final Set<YangResourceEntity> yangResourceEntities) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final SchemaSetEntity schemaSetEntity = new SchemaSetEntity(); + schemaSetEntity.setName(schemaSetName); + schemaSetEntity.setDataspace(dataspaceEntity); + schemaSetEntity.setYangResources(yangResourceEntities); + try { + schemaSetRepository.save(schemaSetEntity); + } catch (final DataIntegrityViolationException e) { + throw AlreadyDefinedException.forSchemaSet(schemaSetName, dataspaceName, e); } - - return ImmutableSet.<YangResourceEntity>builder() - .addAll(existingYangResourceEntities) - .addAll(newYangResourceEntities) - .build(); } private static Map<String, String> createModuleNameAndRevisionMap(final String sourceName, final String source) { @@ -342,6 +331,14 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ return RevisionSourceIdentifier.create(sourceName); } + private void convertExceptionIfNeeded(final DataIntegrityViolationException dataIntegrityViolationException, + final Collection<YangResourceEntity> newYangResourceEntities) { + final Optional<DuplicatedYangResourceException> convertedException = + convertToDuplicatedYangResourceException( + dataIntegrityViolationException, newYangResourceEntities); + throw convertedException.isPresent() ? convertedException.get() : dataIntegrityViolationException; + } + /** * Convert the specified data integrity violation exception into a CPS duplicated Yang resource exception * if the cause of the error is a yang checksum database constraint violation. @@ -391,6 +388,13 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ return "no checksum found"; } + private Set<YangResourceEntity> getYangResourceEntities(final Collection<ModuleReference> moduleReferences) { + return moduleReferences.stream().map(moduleReference -> + yangResourceRepository + .findByModuleNameAndRevision(moduleReference.getModuleName(), moduleReference.getRevision())) + .collect(Collectors.toSet()); + } + private static ModuleReference toModuleReference( final YangResourceModuleReference yangResourceModuleReference) { return ModuleReference.builder() @@ -411,20 +415,4 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ .dataspaceName(schemaSetEntity.getDataspace().getName()).build(); } - private void storeAndLinkNewModules(final Map<String, String> newModuleNameToContentMap, - final SchemaSetEntity schemaSetEntity) { - final Set<YangResourceEntity> yangResourceEntities - = new HashSet<>(synchronizeYangResources(newModuleNameToContentMap)); - schemaSetEntity.setYangResources(yangResourceEntities); - schemaSetRepository.save(schemaSetEntity); - } - - private void updateAllModuleReferences(final Collection<ModuleReference> allModuleReferences, - final Integer schemaSetEntityId) { - yangResourceRepository.deleteSchemaSetYangResourceForSchemaSetId(schemaSetEntityId); - final List<Integer> allYangResourceIds = - yangResourceRepository.getResourceIdsByModuleReferences(allModuleReferences); - yangResourceRepository.insertSchemaSetIdYangResourceId(schemaSetEntityId, allYangResourceIds); - } - } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java index 3f3ca79900..bf354be024 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java @@ -51,13 +51,18 @@ public class FragmentQueryBuilder { private EntityManager entityManager; /** - * Create a sql query to retrieve by anchor(id) and cps path. + * Create a sql query to retrieve by anchor(id) and cps path with an optional queryResultLimit on results. * * @param anchorEntity the anchor * @param cpsPathQuery the cps path query to be transformed into a sql query + * @param queryResultLimit queryResultLimit number of returned entities + * (if the queryResultLimit is less than 1 the method returns all related entities) + * * @return a executable query object */ - public Query getQueryForAnchorAndCpsPath(final AnchorEntity anchorEntity, final CpsPathQuery cpsPathQuery) { + public Query getQueryForAnchorAndCpsPath(final AnchorEntity anchorEntity, + final CpsPathQuery cpsPathQuery, + final int queryResultLimit) { final StringBuilder sqlStringBuilder = new StringBuilder(); final Map<String, Object> queryParameters = new HashMap<>(); @@ -65,6 +70,7 @@ public class FragmentQueryBuilder { addWhereClauseForAnchor(anchorEntity, sqlStringBuilder, queryParameters); addNodeSearchConditions(cpsPathQuery, sqlStringBuilder, queryParameters, false); addSearchSuffix(cpsPathQuery, sqlStringBuilder, queryParameters); + addLimitClause(sqlStringBuilder, queryParameters, queryResultLimit); return getQuery(sqlStringBuilder.toString(), queryParameters, FragmentEntity.class); } @@ -219,6 +225,15 @@ public class FragmentQueryBuilder { } } + private static void addLimitClause(final StringBuilder sqlStringBuilder, + final Map<String, Object> queryParameters, + final int queryResultLimit) { + if (queryResultLimit > 0) { + sqlStringBuilder.append(" LIMIT :queryResultLimit"); + queryParameters.put("queryResultLimit", queryResultLimit); + } + } + private static Integer getTextValueAsInt(final CpsPathQuery cpsPathQuery) { try { return Integer.parseInt(cpsPathQuery.getTextFunctionConditionValue()); diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQuery.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQuery.java index 9c1929eaf7..4ee6555b79 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQuery.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQuery.java @@ -29,7 +29,9 @@ import org.onap.cps.ri.models.DataspaceEntity; import org.onap.cps.ri.models.FragmentEntity; public interface FragmentRepositoryCpsPathQuery { - List<FragmentEntity> findByAnchorAndCpsPath(AnchorEntity anchorEntity, CpsPathQuery cpsPathQuery); + + List<FragmentEntity> findByAnchorAndCpsPath(AnchorEntity anchorEntity, CpsPathQuery cpsPathQuery, + int queryResultLimit); List<FragmentEntity> findByDataspaceAndCpsPath(DataspaceEntity dataspaceEntity, CpsPathQuery cpsPathQuery, List<Long> anchorIds); diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQueryImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQueryImpl.java index e8c2725670..80fbe9b6cd 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQueryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepositoryCpsPathQueryImpl.java @@ -41,10 +41,15 @@ public class FragmentRepositoryCpsPathQueryImpl implements FragmentRepositoryCps @Override @Transactional public List<FragmentEntity> findByAnchorAndCpsPath(final AnchorEntity anchorEntity, - final CpsPathQuery cpsPathQuery) { - final Query query = fragmentQueryBuilder.getQueryForAnchorAndCpsPath(anchorEntity, cpsPathQuery); + final CpsPathQuery cpsPathQuery, + final int queryResultLimit) { + final Query query = fragmentQueryBuilder + .getQueryForAnchorAndCpsPath(anchorEntity, cpsPathQuery, queryResultLimit); final List<FragmentEntity> fragmentEntities = query.getResultList(); log.debug("Fetched {} fragment entities by anchor and cps path.", fragmentEntities.size()); + if (queryResultLimit > 0) { + log.debug("Result limited to {} entries", queryResultLimit); + } return fragmentEntities; } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java index b8dd7b755c..fdd72624ba 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetRepository.java @@ -83,8 +83,8 @@ public interface SchemaSetRepository extends JpaRepository<SchemaSetEntity, Inte */ @Modifying @Query(value = """ - DELETE FROM schema_set WHERE NOT EXISTS - (SELECT 1 FROM anchor WHERE anchor.schema_set_id = schema_set.id) + DELETE FROM schema_set WHERE schema_set.dataspace_id = :dataspaceId AND + NOT EXISTS (SELECT 1 FROM anchor WHERE anchor.schema_set_id = schema_set.id) """, nativeQuery = true) - void deleteOrphanedSchemaSets(); + void deleteOrphanedSchemaSets(@Param("dataspaceId") final int dataspaceId); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepository.java deleted file mode 100644 index 410dcc2e26..0000000000 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepository.java +++ /dev/null @@ -1,36 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * Copyright (C) 2021-2025 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ri.repository; - -import java.util.List; - -public interface SchemaSetYangResourceRepository { - - - /** - * Link yang resources (ids) with a schema set (id). - * - * @param schemaSetId the schema set id - * @param yangResourceIds list of yang resource ids - */ - void insertSchemaSetIdYangResourceId(final Integer schemaSetId, final List<Integer> yangResourceIds); - -} diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepositoryImpl.java deleted file mode 100644 index 989809af5b..0000000000 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/SchemaSetYangResourceRepositoryImpl.java +++ /dev/null @@ -1,58 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * Copyright (C) 2021-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ri.repository; - -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; -import java.sql.PreparedStatement; -import java.util.List; -import org.hibernate.Session; -import org.springframework.transaction.annotation.Transactional; - -@Transactional -public class SchemaSetYangResourceRepositoryImpl implements SchemaSetYangResourceRepository { - - private static final int MAX_INSERT_BATCH_SIZE = 100; - - @PersistenceContext - private EntityManager entityManager; - - @Override - public void insertSchemaSetIdYangResourceId(final Integer schemaSetId, final List<Integer> yangResourceIds) { - final Session session = entityManager.unwrap(Session.class); - session.doWork(connection -> { - try (PreparedStatement preparedStatement = connection.prepareStatement( - "INSERT INTO SCHEMA_SET_YANG_RESOURCES (SCHEMA_SET_ID, YANG_RESOURCE_ID) VALUES ( ?, ?)")) { - int sqlQueryCount = 1; - for (final int yangResourceId : yangResourceIds) { - preparedStatement.setInt(1, schemaSetId); - preparedStatement.setInt(2, yangResourceId); - preparedStatement.addBatch(); - if (sqlQueryCount % MAX_INSERT_BATCH_SIZE == 0 || sqlQueryCount == yangResourceIds.size()) { - preparedStatement.executeBatch(); - } - sqlQueryCount++; - } - } - }); - } -} - diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceNativeRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceNativeRepository.java deleted file mode 100644 index 2875511c1e..0000000000 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceNativeRepository.java +++ /dev/null @@ -1,31 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ri.repository; - -import java.util.Collection; -import java.util.List; -import org.onap.cps.api.model.ModuleReference; - -public interface YangResourceNativeRepository { - - List<Integer> getResourceIdsByModuleReferences(Collection<ModuleReference> moduleReferences); - -} diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceNativeRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceNativeRepositoryImpl.java deleted file mode 100644 index 34f1ee362a..0000000000 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceNativeRepositoryImpl.java +++ /dev/null @@ -1,69 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ri.repository; - -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; -import jakarta.persistence.Query; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.StringJoiner; -import lombok.extern.slf4j.Slf4j; -import org.hibernate.type.StandardBasicTypes; -import org.onap.cps.api.model.ModuleReference; -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; - -@Slf4j -@Repository -public class YangResourceNativeRepositoryImpl implements YangResourceNativeRepository { - - @PersistenceContext - private EntityManager entityManager; - - @Override - @Transactional - public List<Integer> getResourceIdsByModuleReferences(final Collection<ModuleReference> moduleReferences) { - if (moduleReferences.isEmpty()) { - return Collections.emptyList(); - } - final Query query = entityManager.createNativeQuery(getCombinedSelectSqlQuery(moduleReferences)) - .unwrap(org.hibernate.query.NativeQuery.class) - .addScalar("id", StandardBasicTypes.INTEGER); - final List<Integer> yangResourceIds = query.getResultList(); - if (yangResourceIds.size() != moduleReferences.size()) { - log.warn("ModuleReferences size : {} and QueryResult size : {}", moduleReferences.size(), - yangResourceIds.size()); - } - return yangResourceIds; - } - - private String getCombinedSelectSqlQuery(final Collection<ModuleReference> moduleReferences) { - final StringJoiner sqlQueryJoiner = new StringJoiner(" UNION ALL "); - moduleReferences.forEach(moduleReference -> - sqlQueryJoiner.add(String.format("SELECT id FROM yang_resource WHERE module_name='%s' and revision='%s'", - moduleReference.getModuleName(), - moduleReference.getRevision())) - ); - return sqlQueryJoiner.toString(); - } -} diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java index 628502f846..e36e376bc6 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/YangResourceRepository.java @@ -32,8 +32,10 @@ import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @Repository -public interface YangResourceRepository extends JpaRepository<YangResourceEntity, Integer>, - YangResourceNativeRepository, SchemaSetYangResourceRepository { +public interface YangResourceRepository extends JpaRepository<YangResourceEntity, Integer> { + + YangResourceEntity findByModuleNameAndRevision(@Param("moduleName") String moduleName, + @Param("revision") String revision); List<YangResourceEntity> findAllByChecksumIn(Collection<String> checksums); @@ -88,10 +90,6 @@ public interface YangResourceRepository extends JpaRepository<YangResourceEntity @Param("moduleName") String moduleName, @Param("revision") String revision); @Modifying - @Query(value = "DELETE FROM schema_set_yang_resources WHERE schema_set_id = :schemaSetId", nativeQuery = true) - void deleteSchemaSetYangResourceForSchemaSetId(@Param("schemaSetId") int schemaSetId); - - @Modifying @Query(value = """ DELETE FROM yang_resource WHERE NOT EXISTS (SELECT 1 FROM schema_set_yang_resources WHERE schema_set_yang_resources.yang_resource_id = yang_resource.id) diff --git a/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java index fa9feee1e7..9e89c8aed9 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/utils/CpsValidatorImpl.java @@ -27,7 +27,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.parameters.PaginationOption; -import org.onap.cps.impl.utils.CpsValidator; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Component; @Slf4j diff --git a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy index c818f3ba1f..e927922acf 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsDataPersistenceServiceImplSpec.groovy @@ -33,7 +33,7 @@ import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.exceptions.ConcurrencyException import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.model.DataNode -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import org.onap.cps.utils.JsonObjectMapper import org.springframework.dao.DataIntegrityViolationException import spock.lang.Specification diff --git a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy deleted file mode 100644 index 28a615b0e8..0000000000 --- a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceConcurrencySpec.groovy +++ /dev/null @@ -1,145 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022 Bell Canada. - * Modifications Copyright (C) 2021-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the 'License'); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ -package org.onap.cps.ri - -import org.hibernate.exception.ConstraintViolationException -import org.onap.cps.ri.models.DataspaceEntity -import org.onap.cps.ri.models.SchemaSetEntity -import org.onap.cps.ri.repository.DataspaceRepository -import org.onap.cps.ri.repository.ModuleReferenceRepository -import org.onap.cps.ri.repository.SchemaSetRepository -import org.onap.cps.ri.repository.YangResourceRepository -import org.onap.cps.spi.CpsAdminPersistenceService -import org.onap.cps.spi.CpsModulePersistenceService -import org.onap.cps.api.exceptions.DuplicatedYangResourceException -import org.onap.cps.api.model.ModuleReference -import org.spockframework.spring.SpringBean -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest -import org.springframework.dao.DataIntegrityViolationException -import org.springframework.retry.annotation.EnableRetry -import spock.lang.Specification - -import java.sql.SQLException - -@SpringBootTest(classes=[CpsModulePersistenceServiceImpl]) -@EnableRetry -class CpsModulePersistenceServiceConcurrencySpec extends Specification { - - @Autowired - CpsModulePersistenceService objectUnderTest - - @SpringBean - DataspaceRepository dataspaceRepository = Mock() - - @SpringBean - YangResourceRepository yangResourceRepository = Mock() - - @SpringBean - SchemaSetRepository schemaSetRepository = Mock() - - @SpringBean - CpsAdminPersistenceService cpsAdminPersistenceService = Mock() - - @SpringBean - ModuleReferenceRepository moduleReferenceRepository = Mock() - - def NEW_RESOURCE_NAME = 'some new resource' - def NEW_RESOURCE_CONTENT = 'module stores {\n' + - ' yang-version 1.1;\n' + - ' namespace "org:onap:ccsdk:sample";\n' + - '}' - - def newYangResourcesNameToContentMap = [(NEW_RESOURCE_NAME):NEW_RESOURCE_CONTENT] - - def yangResourceChecksum = 'b13faef573ed1374139d02c40d8ce09c80ea1dc70e63e464c1ed61568d48d539' - - def yangResourceChecksumDbConstraint = 'yang_resource_checksum_key' - - def sqlExceptionMessage = String.format('(checksum)=(%s)', yangResourceChecksum) - - def checksumIntegrityException = new DataIntegrityViolationException("checksum integrity exception", - new ConstraintViolationException('', new SQLException(sqlExceptionMessage), yangResourceChecksumDbConstraint)) - - def 'Store new schema set, maximum retries.'() { - given: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored' - objectUnderTest.storeSchemaSet('some dataspace', 'some new schema set', newYangResourcesNameToContentMap) - then: 'a duplicated yang resource exception is thrown ' - thrown(DuplicatedYangResourceException) - and: 'the system will attempt to save the data 5 times (because checksum integrity exception is thrown each time)' - 5 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - } - - def 'Store new schema set, succeed on third attempt.'() { - given: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored' - objectUnderTest.storeSchemaSet('some dataspace', 'some new schema set', newYangResourcesNameToContentMap) - then: 'no exception is thrown ' - noExceptionThrown() - and: 'the system will attempt to save the data 2 times with checksum integrity exception but then succeed' - 2 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - 1 * yangResourceRepository.saveAll(_) >> [] - } - - def 'Store schema set using modules, maximum retries.'() { - given: 'map of new modules, a list of existing modules, module reference' - def mapOfNewModules = [newModule1: 'module newmodule { yang-version 1.1; revision "2021-10-12" { } }'] - def moduleReferenceForExistingModule = new ModuleReference("test","2021-10-12") - def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule] - and: 'no pre-existing schemaset in database' - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - when: 'a new schemaset is stored from a module' - objectUnderTest.storeSchemaSetFromModules('some dataspace', 'some new schema set' , mapOfNewModules, listOfExistingModulesModuleReference) - then: 'a duplicated yang resource exception is thrown ' - thrown(DuplicatedYangResourceException) - and: 'the system will attempt to save the data 5 times (because checksum integrity exception is thrown each time)' - 5 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - } - - def 'Store schema set using modules, succeed on third attempt.'() { - given: 'map of new modules, a list of existing modules, module reference' - def mapOfNewModules = [newModule1: 'module newmodule { yang-version 1.1; revision "2021-10-12" { } }'] - def moduleReferenceForExistingModule = new ModuleReference("test","2021-10-12") - def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule] - and: 'no pre-existing schemaset in database' - def dataspaceEntity = new DataspaceEntity() - dataspaceRepository.getByName(_) >> new DataspaceEntity() - yangResourceRepository.findAllByChecksumIn(_) >> Collections.emptyList() - yangResourceRepository.getResourceIdsByModuleReferences(_) >> [] - and: 'can retrieve schemaset details after storing it' - def schemaSetEntity = new SchemaSetEntity() - schemaSetRepository.getByDataspaceAndName(dataspaceEntity, 'new schema set') >> schemaSetEntity - when: 'a new schemaset is stored from a module' - objectUnderTest.storeSchemaSetFromModules('some dataspace', 'new schema set' , mapOfNewModules, listOfExistingModulesModuleReference) - then: 'no exception is thrown ' - noExceptionThrown() - and: 'the system will attempt to save the data 2 times with checksum integrity exception but then succeed' - 2 * yangResourceRepository.saveAll(_) >> { throw checksumIntegrityException } - 1 * yangResourceRepository.saveAll(_) >> [] - } - -} diff --git a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceImplSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceImplSpec.groovy index 2915bc8e8c..9abfdbeb33 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceImplSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/ri/CpsModulePersistenceServiceImplSpec.groovy @@ -78,8 +78,8 @@ class CpsModulePersistenceServiceImplSpec extends Specification { and: 'persisting yang resource raises db constraint exception (in case of concurrent requests for example)' mockYangResourceRepository.saveAll(_) >> { throw dbException } when: 'attempt to store schema set ' - def newYangResourcesNameToContentMap = [(yangResourceName):yangResourceContent] - objectUnderTest.storeSchemaSet('my-dataspace', 'my-schema-set', newYangResourcesNameToContentMap) + def newYangResourceContentPerName = [(yangResourceName):yangResourceContent] + objectUnderTest.createSchemaSet('my-dataspace', 'my-schema-set', newYangResourceContentPerName) then: 'an #expectedThrownException is thrown' def e = thrown(expectedThrownException) assert e.getMessage().contains(expectedThrownExceptionMessage) @@ -96,7 +96,7 @@ class CpsModulePersistenceServiceImplSpec extends Specification { def schemaSetEntity = new SchemaSetEntity(id: 1) mockSchemaSetRepository.getByDataspaceAndName(_, _) >> schemaSetEntity when: 'schema set update is requested' - objectUnderTest.updateSchemaSetFromModules('my-dataspace', 'my-schemaset', [:], [new ModuleReference('some module name', 'some revision name')]) + objectUnderTest.updateSchemaSetFromNewAndExistingModules('my-dataspace', 'my-schemaset', [:], [new ModuleReference('some module name', 'some revision name')]) then: 'no exception is thrown ' noExceptionThrown() } diff --git a/cps-service/pom.xml b/cps-service/pom.xml index 7dda1327e0..1fe86cbb96 100644 --- a/cps-service/pom.xml +++ b/cps-service/pom.xml @@ -30,7 +30,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java b/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java index 345bc8825b..41713746b7 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java @@ -314,13 +314,13 @@ public interface CpsDataService { * @param dataspaceName source dataspace name * @param sourceAnchorName source anchor name * @param xpath xpath - * @param yangResourcesNameToContentMap YANG resources (files) map where key is a name and value is content + * @param yangResourceContentPerName YANG resources (files) map where key is a name and value is content * @param targetData target data to be compared in JSON string format * @param fetchDescendantsOption defines the scope of data to fetch: defaulted to INCLUDE_ALL_DESCENDANTS * @return list containing {@link DeltaReport} objects */ List<DeltaReport> getDeltaByDataspaceAnchorAndPayload(String dataspaceName, String sourceAnchorName, String xpath, - Map<String, String> yangResourcesNameToContentMap, + Map<String, String> yangResourceContentPerName, String targetData, FetchDescendantsOption fetchDescendantsOption); diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java index 81b6439efc..2494be4021 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java @@ -40,21 +40,21 @@ public interface CpsModuleService { * * @param dataspaceName dataspace name * @param schemaSetName schema set name - * @param yangResourcesNameToContentMap yang resources (files) as a mep where key is resource name + * @param yangResourceContentPerName yang resources (files) as a mep where key is resource name * and value is content */ void createSchemaSet(String dataspaceName, String schemaSetName, - Map<String, String> yangResourcesNameToContentMap); + Map<String, String> yangResourceContentPerName); /** * Create or upgrade a schema set from new modules and existing modules or only existing modules. - * @param dataspaceName Dataspace name - * @param schemaSetName schema set name - * @param newModuleNameToContentMap YANG resources map where key is a module name and value is content - * @param allModuleReferences All YANG resource module references + * @param dataspaceName Dataspace name + * @param schemaSetName schema set name + * @param yangResourceContentPerName YANG resources map where key is a name and value is content + * @param allModuleReferences All YANG resource module references */ void createSchemaSetFromModules(String dataspaceName, String schemaSetName, - Map<String, String> newModuleNameToContentMap, + Map<String, String> yangResourceContentPerName, Collection<ModuleReference> allModuleReferences); /** @@ -164,8 +164,11 @@ public interface CpsModuleService { Collection<ModuleReference> identifyNewModuleReferences(Collection<ModuleReference> moduleReferencesToCheck); /** - * Remove any Yang Resource Modules and Schema Sets from the DB that are no longer referenced by any anchor. + * Remove any Yang Resource Modules and Schema Sets from the given dataspace that are no longer referenced + * by any anchor. + * + * @param dataspaceName dataspace name */ - void deleteAllUnusedYangModuleData(); + void deleteAllUnusedYangModuleData(String dataspaceName); } diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsQueryService.java b/cps-service/src/main/java/org/onap/cps/api/CpsQueryService.java index d783b9ed0e..3044fe0a90 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsQueryService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsQueryService.java @@ -32,6 +32,8 @@ import org.onap.cps.api.parameters.PaginationOption; */ public interface CpsQueryService { + public static int NO_LIMIT = 0; + /** * Get data nodes for the given dataspace and anchor by cps path. * @@ -45,6 +47,20 @@ public interface CpsQueryService { Collection<DataNode> queryDataNodes(String dataspaceName, String anchorName, String cpsPath, FetchDescendantsOption fetchDescendantsOption); + /** + * Retrieves a collection of data nodes based on the specified CPS path query. + * + * @param dataspaceName the name of the dataspace (must not be null or empty) + * @param anchorName the name of the anchor (must not be null or empty) + * @param cpsPath the CPS path used for querying (must not be null or empty) + * @param fetchDescendantsOption specifies whether to include descendant nodes in the output + * @param queryResultLimit the maximum number of data nodes to return; if less than 1, returns all matching nodes + * + * @return a collection of matching {@link DataNode} instances (can be empty if no nodes are found) + */ + Collection<DataNode> queryDataNodes(String dataspaceName, String anchorName, + String cpsPath, FetchDescendantsOption fetchDescendantsOption, + int queryResultLimit); /** * Get data leaf for the given dataspace and anchor by cps path. diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java b/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java index be80b636ad..6597aa3908 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java +++ b/cps-service/src/main/java/org/onap/cps/api/model/DataNode.java @@ -26,20 +26,19 @@ import java.io.Serializable; import java.util.Collection; import java.util.Collections; import java.util.Map; -import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; -@Setter(AccessLevel.PROTECTED) +@Setter @Getter @EqualsAndHashCode +@NoArgsConstructor public class DataNode implements Serializable { private static final long serialVersionUID = 1482619410918597467L; - DataNode() {} - private String dataspace; private String schemaSetName; private String anchorName; diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java b/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java index df642628d0..761c6ad01d 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java +++ b/cps-service/src/main/java/org/onap/cps/api/model/DeltaReport.java @@ -23,21 +23,20 @@ package org.onap.cps.api.model; import com.fasterxml.jackson.annotation.JsonInclude; import java.io.Serializable; import java.util.Map; -import lombok.AccessLevel; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; -@Setter(AccessLevel.PROTECTED) +@Setter @Getter @JsonInclude(JsonInclude.Include.NON_NULL) +@NoArgsConstructor public class DeltaReport { public static final String CREATE_ACTION = "create"; public static final String REMOVE_ACTION = "remove"; public static final String REPLACE_ACTION = "replace"; - DeltaReport() {} - private String action; private String xpath; private Map<String, Serializable> sourceData; diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsAnchorServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsAnchorServiceImpl.java index fb22311128..f18ae74c73 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsAnchorServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsAnchorServiceImpl.java @@ -24,9 +24,9 @@ import java.util.Collection; import lombok.RequiredArgsConstructor; import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.model.Anchor; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsAdminPersistenceService; import org.onap.cps.spi.CpsDataPersistenceService; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Service; @Service diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java index f2513173a6..9f70ac9132 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDataServiceImpl.java @@ -42,15 +42,14 @@ import org.onap.cps.api.CpsDeltaService; import org.onap.cps.api.exceptions.DataValidationException; import org.onap.cps.api.model.Anchor; import org.onap.cps.api.model.DataNode; -import org.onap.cps.api.model.DataNodeBuilder; import org.onap.cps.api.model.DeltaReport; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.cpspath.parser.CpsPathUtil; import org.onap.cps.events.CpsDataUpdateEventsService; import org.onap.cps.events.model.Data.Operation; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsDataPersistenceService; import org.onap.cps.utils.ContentType; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.utils.DataMapUtils; import org.onap.cps.utils.JsonObjectMapper; import org.onap.cps.utils.PrefixResolver; @@ -231,7 +230,7 @@ public class CpsDataServiceImpl implements CpsDataService { @Override public List<DeltaReport> getDeltaByDataspaceAnchorAndPayload(final String dataspaceName, final String sourceAnchorName, final String xpath, - final Map<String, String> yangResourcesNameToContentMap, + final Map<String, String> yangResourceContentPerName, final String targetData, final FetchDescendantsOption fetchDescendantsOption) { @@ -244,7 +243,7 @@ public class CpsDataServiceImpl implements CpsDataService { new ArrayList<>(rebuildSourceDataNodes(xpath, sourceAnchor, sourceDataNodes)); final Collection<DataNode> targetDataNodes = - new ArrayList<>(buildTargetDataNodes(sourceAnchor, xpath, yangResourcesNameToContentMap, targetData)); + new ArrayList<>(buildTargetDataNodes(sourceAnchor, xpath, yangResourceContentPerName, targetData)); return cpsDeltaService.getDeltaReports(sourceDataNodesRebuilt, targetDataNodes); } @@ -381,12 +380,12 @@ public class CpsDataServiceImpl implements CpsDataService { } private Collection<DataNode> buildTargetDataNodes(final Anchor sourceAnchor, final String xpath, - final Map<String, String> yangResourcesNameToContentMap, + final Map<String, String> yangResourceContentPerName, final String targetData) { - if (yangResourcesNameToContentMap.isEmpty()) { + if (yangResourceContentPerName.isEmpty()) { return buildDataNodesWithAnchorAndXpath(sourceAnchor, xpath, targetData, ContentType.JSON); } else { - return buildDataNodesWithYangResourceAndXpath(yangResourcesNameToContentMap, xpath, + return buildDataNodesWithYangResourceAndXpath(yangResourceContentPerName, xpath, targetData, ContentType.JSON); } } @@ -455,12 +454,12 @@ public class CpsDataServiceImpl implements CpsDataService { } private Collection<DataNode> buildDataNodesWithParentNodeXpath( - final Map<String, String> yangResourcesNameToContentMap, final String xpath, + final Map<String, String> yangResourceContentPerName, final String xpath, final String nodeData, final ContentType contentType) { if (isRootNodeXpath(xpath)) { final ContainerNode containerNode = yangParser.parseData(contentType, nodeData, - yangResourcesNameToContentMap, PARENT_NODE_XPATH_FOR_ROOT_NODE_XPATH); + yangResourceContentPerName, PARENT_NODE_XPATH_FOR_ROOT_NODE_XPATH); final Collection<DataNode> dataNodes = new DataNodeBuilder() .withContainerNode(containerNode) .buildCollection(); @@ -471,7 +470,7 @@ public class CpsDataServiceImpl implements CpsDataService { } final String normalizedParentNodeXpath = CpsPathUtil.getNormalizedXpath(xpath); final ContainerNode containerNode = - yangParser.parseData(contentType, nodeData, yangResourcesNameToContentMap, normalizedParentNodeXpath); + yangParser.parseData(contentType, nodeData, yangResourceContentPerName, normalizedParentNodeXpath); final Collection<DataNode> dataNodes = new DataNodeBuilder() .withParentNodeXpath(normalizedParentNodeXpath) .withContainerNode(containerNode) @@ -497,18 +496,18 @@ public class CpsDataServiceImpl implements CpsDataService { } private Collection<DataNode> buildDataNodesWithYangResourceAndXpath( - final Map<String, String> yangResourcesNameToContentMap, final String xpath, + final Map<String, String> yangResourceContentPerName, final String xpath, final String nodeData, final ContentType contentType) { if (!isRootNodeXpath(xpath)) { final String parentNodeXpath = CpsPathUtil.getNormalizedParentXpath(xpath); if (parentNodeXpath.isEmpty()) { - return buildDataNodesWithParentNodeXpath(yangResourcesNameToContentMap, ROOT_NODE_XPATH, + return buildDataNodesWithParentNodeXpath(yangResourceContentPerName, ROOT_NODE_XPATH, nodeData, contentType); } - return buildDataNodesWithParentNodeXpath(yangResourcesNameToContentMap, parentNodeXpath, + return buildDataNodesWithParentNodeXpath(yangResourceContentPerName, parentNodeXpath, nodeData, contentType); } - return buildDataNodesWithParentNodeXpath(yangResourcesNameToContentMap, xpath, nodeData, contentType); + return buildDataNodesWithParentNodeXpath(yangResourceContentPerName, xpath, nodeData, contentType); } private static boolean isRootNodeXpath(final String xpath) { diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java index 15caa2276d..1a85147b64 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java @@ -27,8 +27,8 @@ import java.util.Collection; import lombok.RequiredArgsConstructor; import org.onap.cps.api.CpsDataspaceService; import org.onap.cps.api.model.Dataspace; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsAdminPersistenceService; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Service; @Service diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java index 7a9d142506..d532001aec 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDeltaServiceImpl.java @@ -32,7 +32,6 @@ import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.CpsDeltaService; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.model.DeltaReport; -import org.onap.cps.api.model.DeltaReportBuilder; import org.springframework.stereotype.Service; @Slf4j diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java index 7622ba5fe2..e50325c739 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsModuleServiceImpl.java @@ -36,8 +36,8 @@ import org.onap.cps.api.model.ModuleDefinition; import org.onap.cps.api.model.ModuleReference; import org.onap.cps.api.model.SchemaSet; import org.onap.cps.api.parameters.CascadeDeleteAllowed; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsModulePersistenceService; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder; import org.onap.cps.yang.YangTextSchemaSourceSet; import org.springframework.stereotype.Service; @@ -57,21 +57,21 @@ public class CpsModuleServiceImpl implements CpsModuleService { @Timed(value = "cps.module.service.schemaset.create", description = "Time taken to create (and store) a schemaset") public void createSchemaSet(final String dataspaceName, final String schemaSetName, - final Map<String, String> yangResourcesNameToContentMap) { + final Map<String, String> yangResourceContentPerName) { cpsValidator.validateNameCharacters(dataspaceName); - cpsModulePersistenceService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap); + cpsModulePersistenceService.createSchemaSet(dataspaceName, schemaSetName, yangResourceContentPerName); final YangTextSchemaSourceSet yangTextSchemaSourceSet = - timedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap); + timedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourceContentPerName); yangTextSchemaSourceSetCache.updateCache(dataspaceName, schemaSetName, yangTextSchemaSourceSet); } @Override public void createSchemaSetFromModules(final String dataspaceName, final String schemaSetName, - final Map<String, String> newModuleNameToContentMap, + final Map<String, String> yangResourceContentPerName, final Collection<ModuleReference> allModuleReferences) { cpsValidator.validateNameCharacters(dataspaceName); - cpsModulePersistenceService.storeSchemaSetFromModules(dataspaceName, schemaSetName, - newModuleNameToContentMap, allModuleReferences); + cpsModulePersistenceService.createSchemaSetFromNewAndExistingModules(dataspaceName, schemaSetName, + yangResourceContentPerName, allModuleReferences); } @Override @@ -83,7 +83,7 @@ public class CpsModuleServiceImpl implements CpsModuleService { @Override public SchemaSet getSchemaSet(final String dataspaceName, final String schemaSetName) { cpsValidator.validateNameCharacters(dataspaceName); - final var yangTextSchemaSourceSet = yangTextSchemaSourceSetCache + final YangTextSchemaSourceSet yangTextSchemaSourceSet = yangTextSchemaSourceSetCache .get(dataspaceName, schemaSetName); return SchemaSet.builder().name(schemaSetName).dataspaceName(dataspaceName) .moduleReferences(yangTextSchemaSourceSet.getModuleReferences()).build(); @@ -130,15 +130,14 @@ public class CpsModuleServiceImpl implements CpsModuleService { @Override public void upgradeSchemaSetFromModules(final String dataspaceName, final String schemaSetName, - final Map<String, String> newModuleNameToContentMap, + final Map<String, String> newYangResourceContentPerModule, final Collection<ModuleReference> allModuleReferences) { cpsValidator.validateNameCharacters(dataspaceName); - cpsModulePersistenceService.updateSchemaSetFromModules(dataspaceName, schemaSetName, - newModuleNameToContentMap, allModuleReferences); + cpsModulePersistenceService.updateSchemaSetFromNewAndExistingModules(dataspaceName, schemaSetName, + newYangResourceContentPerModule, allModuleReferences); yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName); } - @Override public Collection<ModuleReference> getYangResourceModuleReferences(final String dataspaceName) { cpsValidator.validateNameCharacters(dataspaceName); @@ -175,8 +174,9 @@ public class CpsModuleServiceImpl implements CpsModuleService { } @Override - public void deleteAllUnusedYangModuleData() { - cpsModulePersistenceService.deleteAllUnusedYangModuleData(); + public void deleteAllUnusedYangModuleData(final String dataspaceName) { + cpsValidator.validateNameCharacters(dataspaceName); + cpsModulePersistenceService.deleteAllUnusedYangModuleData(dataspaceName); } private boolean isCascadeDeleteProhibited(final CascadeDeleteAllowed cascadeDeleteAllowed) { diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsQueryServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsQueryServiceImpl.java index e534e0aea1..f27445f7c9 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsQueryServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsQueryServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2022-2023 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,8 +29,8 @@ import org.onap.cps.api.CpsQueryService; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.api.parameters.PaginationOption; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsDataPersistenceService; +import org.onap.cps.utils.CpsValidator; import org.springframework.stereotype.Service; @Service @@ -46,15 +46,29 @@ public class CpsQueryServiceImpl implements CpsQueryService { public Collection<DataNode> queryDataNodes(final String dataspaceName, final String anchorName, final String cpsPath, final FetchDescendantsOption fetchDescendantsOption) { - cpsValidator.validateNameCharacters(dataspaceName, anchorName); - return cpsDataPersistenceService.queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption); + return queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption, NO_LIMIT); + } + + @Override + @Timed(value = "cps.data.service.datanode.query", + description = "Time taken to query data nodes with a limit on results") + public Collection<DataNode> queryDataNodes(final String dataSpaceName, final String anchorName, + final String cpsPath, + final FetchDescendantsOption fetchDescendantsOption, + final int queryResultLimit) { + cpsValidator.validateNameCharacters(dataSpaceName, anchorName); + return cpsDataPersistenceService.queryDataNodes(dataSpaceName, + anchorName, + cpsPath, + fetchDescendantsOption, + queryResultLimit); } @Override public <T> Set<T> queryDataLeaf(final String dataspaceName, final String anchorName, final String cpsPath, final Class<T> targetClass) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); - throw new UnsupportedOperationException("Query by attribute-axis not implemented yet!"); + return cpsDataPersistenceService.queryDataLeaf(dataspaceName, anchorName, cpsPath, targetClass); } @Override diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DataNodeBuilder.java b/cps-service/src/main/java/org/onap/cps/impl/DataNodeBuilder.java index d509f53525..a78f3d9826 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DataNodeBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/impl/DataNodeBuilder.java @@ -20,7 +20,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model; +package org.onap.cps.impl; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -33,6 +33,7 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.exceptions.DataValidationException; +import org.onap.cps.api.model.DataNode; import org.onap.cps.utils.YangUtils; import org.opendaylight.yangtools.yang.common.Ordering; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; diff --git a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReportBuilder.java b/cps-service/src/main/java/org/onap/cps/impl/DeltaReportBuilder.java index a8e922f3df..fdc2e939d6 100644 --- a/cps-service/src/main/java/org/onap/cps/api/model/DeltaReportBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/impl/DeltaReportBuilder.java @@ -18,11 +18,12 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model; +package org.onap.cps.impl; import java.io.Serializable; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.onap.cps.api.model.DeltaReport; @Slf4j public class DeltaReportBuilder { diff --git a/cps-service/src/main/java/org/onap/cps/impl/YangTextSchemaSourceSetCache.java b/cps-service/src/main/java/org/onap/cps/impl/YangTextSchemaSourceSetCache.java index 688669c941..e7e7b1c5ce 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/YangTextSchemaSourceSetCache.java +++ b/cps-service/src/main/java/org/onap/cps/impl/YangTextSchemaSourceSetCache.java @@ -27,8 +27,8 @@ import io.micrometer.core.instrument.Metrics; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import lombok.RequiredArgsConstructor; -import org.onap.cps.impl.utils.CpsValidator; import org.onap.cps.spi.CpsModulePersistenceService; +import org.onap.cps.utils.CpsValidator; import org.onap.cps.yang.YangTextSchemaSourceSet; import org.onap.cps.yang.YangTextSchemaSourceSetBuilder; import org.springframework.cache.annotation.CacheConfig; diff --git a/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java b/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java index e864633f25..a80239039a 100644 --- a/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java +++ b/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -77,7 +77,7 @@ public abstract class AbstractModelLoader implements ModelLoader { } catch (final AlreadyDefinedException alreadyDefinedException) { log.warn("Creating new schema set failed as schema set already exists"); } catch (final Exception exception) { - log.error("Creating schema set failed: {} ", exception.getMessage()); + log.error("Creating schema set {} failed: {} ", schemaSetName, exception.getMessage()); throw new ModelOnboardingException("Creating schema set failed", exception.getMessage()); } } diff --git a/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java b/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java index 5be5b1e2e0..5be5fb0481 100644 --- a/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java +++ b/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2024 Nordix Foundation. + * Copyright (C) 2020-2025 Nordix Foundation. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2022-2023 TechMahindra Ltd. @@ -27,6 +27,7 @@ import java.io.Serializable; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import org.onap.cps.api.model.DataNode; import org.onap.cps.api.parameters.FetchDescendantsOption; import org.onap.cps.api.parameters.PaginationOption; @@ -185,6 +186,34 @@ public interface CpsDataPersistenceService { String cpsPath, FetchDescendantsOption fetchDescendantsOption); /** + * Get a datanode by cps path. + * + * @param dataspaceName dataspace name + * @param anchorName anchor name + * @param cpsPath cps path + * @param fetchDescendantsOption defines whether the descendants of the node(s) found by the query should be + * included in the output + * @param queryResultLimit limits the number of returned entities (if less than 1 returns all) + * + * @return the data nodes found i.e. 0 or more data nodes + */ + List<DataNode> queryDataNodes(String dataspaceName, + String anchorName, + String cpsPath, FetchDescendantsOption fetchDescendantsOption, + int queryResultLimit); + + /** + * Get data leaf for the given dataspace and anchor by cps path. + * + * @param dataspaceName dataspace name + * @param anchorName anchor name + * @param cpsPath cps path + * @param targetClass class of the expected data type + * @return a collection of data objects of expected type + */ + <T> Set<T> queryDataLeaf(String dataspaceName, String anchorName, String cpsPath, Class<T> targetClass); + + /** * Get a datanode by dataspace name and cps path across all anchors. * * @param dataspaceName dataspace name diff --git a/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java b/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java index b1f8aad88f..02e1b6c754 100755 --- a/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java +++ b/cps-service/src/main/java/org/onap/cps/spi/CpsModulePersistenceService.java @@ -36,34 +36,37 @@ public interface CpsModulePersistenceService { /** * Stores Schema Set. * - * @param dataspaceName dataspace name - * @param schemaSetName schema set name - * @param yangResourcesNameToContentMap YANG resources (files) map where key is a name and value is content + * @param dataspaceName dataspace name + * @param schemaSetName schema set name + * @param yangResourceContentPerName a map of YANG resources map where key is a name and value is content */ - void storeSchemaSet(String dataspaceName, String schemaSetName, Map<String, String> yangResourcesNameToContentMap); + void createSchemaSet(String dataspaceName, String schemaSetName, Map<String, String> yangResourceContentPerName); /** * Stores a new schema set from new modules and existing modules. * - * @param dataspaceName Dataspace name - * @param schemaSetName Schema set name - * @param newModuleNameToContentMap YANG resources map where key is a module name and value is content - * @param allModuleReferences All YANG resources module references + * @param dataspaceName dataspace name + * @param schemaSetName Schema set name + * @param newYangResourceContentPerName a map of only the new YANG resources + * the key is a name and value is its content + * @param allModuleReferences all YANG resources module references */ - void storeSchemaSetFromModules(String dataspaceName, String schemaSetName, - Map<String, String> newModuleNameToContentMap, Collection<ModuleReference> allModuleReferences); + void createSchemaSetFromNewAndExistingModules(String dataspaceName, String schemaSetName, + Map<String, String> newYangResourceContentPerName, + Collection<ModuleReference> allModuleReferences); /** * Update an existing schema set from new modules and existing modules. * - * @param dataspaceName Dataspace name + * @param dataspaceName dataspace name * @param schemaSetName Schema set name - * @param newModuleNameToContentMap YANG resources map where key is a module name and value is content - * @param allModuleReferences All YANG resources module references + * @param newYangResourcesPerName a map of only the new YANG resources + * the key is a module name and value is its content + * @param allModuleReferences all YANG resources module references */ - void updateSchemaSetFromModules(final String dataspaceName, final String schemaSetName, - final Map<String, String> newModuleNameToContentMap, - final Collection<ModuleReference> allModuleReferences); + void updateSchemaSetFromNewAndExistingModules(String dataspaceName, String schemaSetName, + Map<String, String> newYangResourcesPerName, + Collection<ModuleReference> allModuleReferences); /** * Checks whether a schema set exists in the specified dataspace. @@ -146,9 +149,11 @@ public interface CpsModulePersistenceService { String moduleName, String moduleRevision); /** - * Remove any unused Yang Resource Modules and Schema Sets. + * Remove any unused Yang Resource Modules and Schema Sets from the given dataspace. + * + * @param dataspaceName dataspace name */ - void deleteAllUnusedYangModuleData(); + void deleteAllUnusedYangModuleData(String dataspaceName); /** * Identify new module references from those returned by a node compared to what is in CPS already. diff --git a/cps-service/src/main/java/org/onap/cps/impl/utils/CpsValidator.java b/cps-service/src/main/java/org/onap/cps/utils/CpsValidator.java index 75bcf126a4..93f51ee58d 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/utils/CpsValidator.java +++ b/cps-service/src/main/java/org/onap/cps/utils/CpsValidator.java @@ -18,7 +18,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.impl.utils; +package org.onap.cps.utils; import org.onap.cps.api.parameters.PaginationOption; diff --git a/cps-service/src/main/java/org/onap/cps/utils/YangParser.java b/cps-service/src/main/java/org/onap/cps/utils/YangParser.java index 08f450e2f1..5dfeb2fb3f 100644 --- a/cps-service/src/main/java/org/onap/cps/utils/YangParser.java +++ b/cps-service/src/main/java/org/onap/cps/utils/YangParser.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation. + * Copyright (C) 2024-2025 Nordix Foundation. * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,17 +71,17 @@ public class YangParser { /** * Parses data into (normalized) ContainerNode according to schema context for the given yang resource. * - * @param nodeData data string - * @param yangResourcesNameToContentMap yang resource to content map - * @return the NormalizedNode object + * @param nodeData data string + * @param yangResourceContentPerName yang resource content per name + * @return the NormalizedNode object */ @Timed(value = "cps.utils.yangparser.nodedata.with.parent.with.yangResourceMap.parse", description = "Time taken to parse node data with a parent") public ContainerNode parseData(final ContentType contentType, final String nodeData, - final Map<String, String> yangResourcesNameToContentMap, + final Map<String, String> yangResourceContentPerName, final String parentNodeXpath) { - final SchemaContext schemaContext = getSchemaContext(yangResourcesNameToContentMap); + final SchemaContext schemaContext = getSchemaContext(yangResourceContentPerName); return yangParserHelper.parseData(contentType, nodeData, schemaContext, parentNodeXpath, VALIDATE_AND_PARSE); } @@ -114,8 +114,8 @@ public class YangParser { anchor.getSchemaSetName()).getSchemaContext(); } - private SchemaContext getSchemaContext(final Map<String, String> yangResourcesNameToContentMap) { - return timedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap) + private SchemaContext getSchemaContext(final Map<String, String> yangResourceContentPerName) { + return timedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourceContentPerName) .getSchemaContext(); } diff --git a/cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java b/cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java index 013faff0c8..9b2ac944dc 100644 --- a/cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java @@ -30,8 +30,8 @@ public class TimedYangTextSchemaSourceSetBuilder { @Timed(value = "cps.yangtextschemasourceset.build", description = "Time taken to build a yang text schema source set") public YangTextSchemaSourceSet getYangTextSchemaSourceSet( - final Map<String, String> yangResourcesNameToContentMap) { - return YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap); + final Map<String, String> yangResourceContentPerName) { + return YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName); } } diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsAnchorServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsAnchorServiceImplSpec.groovy index 22f5c9f83e..d78c8bb47f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsAnchorServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsAnchorServiceImplSpec.groovy @@ -21,7 +21,7 @@ package org.onap.cps.impl -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsAdminPersistenceService import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.api.exceptions.ModuleNamesNotFoundException diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy index 3ea859ae6d..abcda6c696 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataServiceImplSpec.groovy @@ -31,7 +31,7 @@ import org.onap.cps.TestUtils import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDeltaService import org.onap.cps.events.CpsDataUpdateEventsService -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.exceptions.ConcurrencyException @@ -40,7 +40,6 @@ import org.onap.cps.api.exceptions.DataValidationException import org.onap.cps.api.exceptions.SessionManagerException import org.onap.cps.api.exceptions.SessionTimeoutException import org.onap.cps.api.model.Anchor -import org.onap.cps.api.model.DataNodeBuilder import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import org.onap.cps.utils.PrefixResolver @@ -249,10 +248,10 @@ class CpsDataServiceImplSpec extends Specification { def 'Get delta between anchor and payload with user provided schema #scenario'() { given: 'user provided schema set ' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') - setupSchemaSetMocksForDelta(yangResourcesNameToContentMap) + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') + setupSchemaSetMocksForDelta(yangResourceContentPerName) when: 'attempt to get delta between an anchor and a JSON payload' - objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, yangResourcesNameToContentMap, jsonData, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) + objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, yangResourceContentPerName, jsonData, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) then: 'dataspacename and anchor names are validated' 1 * mockCpsValidator.validateNameCharacters(['some-dataspace', 'some-anchor']) and: 'source data nodes are fetched using appropriate persistence layer method' @@ -286,10 +285,10 @@ class CpsDataServiceImplSpec extends Specification { def 'Delta between anchor and payload error scenario #scenario'() { given: 'schema set for given anchor and dataspace references bookstore model' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') - setupSchemaSetMocksForDelta(yangResourcesNameToContentMap) + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') + setupSchemaSetMocksForDelta(yangResourceContentPerName) when: 'attempt to get delta between anchor and payload' - objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, yangResourcesNameToContentMap, jsonData, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) + objectUnderTest.getDeltaByDataspaceAnchorAndPayload(dataspaceName, anchorName, xpath, yangResourceContentPerName, jsonData, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) then: 'expected exception is thrown' thrown(DataValidationException) where: 'following parameters were used' @@ -655,11 +654,11 @@ class CpsDataServiceImplSpec extends Specification { mockYangTextSchemaSourceSet.getSchemaContext() >> schemaContext } - def setupSchemaSetMocksForDelta(Map<String, String> yangResourcesNameToContentMap) { + def setupSchemaSetMocksForDelta(Map<String, String> yangResourceContentPerName) { def mockYangTextSchemaSourceSet = Mock(YangTextSchemaSourceSet) - mockTimedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap) >> mockYangTextSchemaSourceSet + mockTimedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourceContentPerName) >> mockYangTextSchemaSourceSet mockYangTextSchemaSourceSetCache.get(_, _) >> mockYangTextSchemaSourceSet - def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName).getSchemaContext() mockYangTextSchemaSourceSet.getSchemaContext() >> schemaContext } diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy index 468fe76d41..97f6fba4d3 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy @@ -21,7 +21,7 @@ package org.onap.cps.impl -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsAdminPersistenceService import org.onap.cps.api.model.Dataspace import spock.lang.Specification diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy index ce871621e5..48db53c882 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsModuleServiceImplSpec.groovy @@ -25,7 +25,7 @@ package org.onap.cps.impl import org.onap.cps.TestUtils import org.onap.cps.api.CpsAnchorService -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsModulePersistenceService import org.onap.cps.api.exceptions.DuplicatedYangResourceException import org.onap.cps.api.exceptions.ModelValidationException @@ -56,7 +56,7 @@ class CpsModuleServiceImplSpec extends Specification { when: 'Create schema set method is invoked' objectUnderTest.createSchemaSet('someDataspace', 'schemaSetName@with Special!Characters', [:]) then: 'Parameters are validated and processing is delegated to persistence service' - 1 * mockCpsModulePersistenceService.storeSchemaSet('someDataspace', 'schemaSetName@with Special!Characters', [:]) + 1 * mockCpsModulePersistenceService.createSchemaSet('someDataspace', 'schemaSetName@with Special!Characters', [:]) and: 'the CpsValidator is called on the dataspaceName' 1 * mockCpsValidator.validateNameCharacters('someDataspace') } @@ -68,16 +68,16 @@ class CpsModuleServiceImplSpec extends Specification { when: 'create schema set from modules method is invoked' objectUnderTest.createSchemaSetFromModules('someDataspaceName', 'someSchemaSetName', [newModule: 'newContent'], listOfExistingModulesModuleReference) then: 'processing is delegated to persistence service' - 1 * mockCpsModulePersistenceService.storeSchemaSetFromModules('someDataspaceName', 'someSchemaSetName', [newModule: 'newContent'], listOfExistingModulesModuleReference) + 1 * mockCpsModulePersistenceService.createSchemaSetFromNewAndExistingModules('someDataspaceName', 'someSchemaSetName', [newModule: 'newContent'], listOfExistingModulesModuleReference) and: 'the CpsValidator is called on the dataspaceName' 1 * mockCpsValidator.validateNameCharacters('someDataspaceName') } def 'Create schema set from invalid resources'() { given: 'Invalid yang resource as name-to-content map' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('invalid.yang') + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('invalid.yang') when: 'Create schema set method is invoked' - objectUnderTest.createSchemaSet('someDataspace', 'someSchemaSet', yangResourcesNameToContentMap) + objectUnderTest.createSchemaSet('someDataspace', 'someSchemaSet', yangResourceContentPerName) then: 'Model validation exception is thrown' thrown(ModelValidationException) } @@ -85,7 +85,7 @@ class CpsModuleServiceImplSpec extends Specification { def 'Create schema set with duplicate yang resource exception in persistence layer.'() { given: 'the persistence layer throws an duplicated yang resource exception' def originalException = new DuplicatedYangResourceException('name', '123', null) - mockCpsModulePersistenceService.storeSchemaSet(*_) >> { throw originalException } + mockCpsModulePersistenceService.createSchemaSet(*_) >> { throw originalException } when: 'attempt to create schema set' objectUnderTest.createSchemaSet('someDataspace', 'someSchemaSet', [:]) then: 'the same duplicated yang resource exception is thrown (up)' @@ -98,9 +98,9 @@ class CpsModuleServiceImplSpec extends Specification { def 'Get schema set by name and dataspace.'() { given: 'an already present schema set' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') and: 'yang resource cache returns the expected schema set' - mockYangTextSchemaSourceSetCache.get('someDataspace', 'schemaSetName@with Special!Characters') >> YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + mockYangTextSchemaSourceSetCache.get('someDataspace', 'schemaSetName@with Special!Characters') >> YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName) when: 'get schema set method is invoked' def result = objectUnderTest.getSchemaSet('someDataspace', 'schemaSetName@with Special!Characters') then: 'the correct schema set is returned' @@ -252,11 +252,11 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsValidator.validateNameCharacters('some-dataspace-name', 'some-anchor-name') } - def 'Delete all unused yang module data.'() { + def 'Delete unused yang module data for a dataspace.'() { when: 'deleting unused yang module data' - objectUnderTest.deleteAllUnusedYangModuleData() - then: 'it is delegated to the module persistence service' - 1 * mockCpsModulePersistenceService.deleteAllUnusedYangModuleData() + objectUnderTest.deleteAllUnusedYangModuleData('some-dataspace-name') + then: 'it is delegated to the module persistence service with the correct parameters' + 1 * mockCpsModulePersistenceService.deleteAllUnusedYangModuleData('some-dataspace-name') } def 'Schema set exists.'() { diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsQueryServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsQueryServiceImplSpec.groovy index 80db83b27a..9db4aa4c3e 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsQueryServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsQueryServiceImplSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2023 Nordix Foundation + * Copyright (C) 2021-2025 Nordix Foundation * Modifications Copyright (C) 2023 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +21,8 @@ package org.onap.cps.impl - -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.api.CpsQueryService +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.parameters.PaginationOption @@ -42,7 +42,7 @@ class CpsQueryServiceImplSpec extends Specification { when: 'queryDataNodes is invoked' objectUnderTest.queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption) then: 'the persistence service is called once with the correct parameters' - 1 * mockCpsDataPersistenceService.queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption) + 1 * mockCpsDataPersistenceService.queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption, CpsQueryService.NO_LIMIT) and: 'the CpsValidator is called on the dataspaceName, schemaSetName and anchorName' 1 * mockCpsValidator.validateNameCharacters(dataspaceName, anchorName) where: 'all fetch descendants options are supported' @@ -50,6 +50,21 @@ class CpsQueryServiceImplSpec extends Specification { FetchDescendantsOption.DIRECT_CHILDREN_ONLY, new FetchDescendantsOption(10)] } + def 'Query data nodes by cps path with limit.'() { + given: 'a dataspace name, an anchor name and a cps path' + def dataspaceName = 'some-dataspace' + def anchorName = 'some-anchor' + def cpsPath = '/cps-path' + def fetchDescendantsOption = FetchDescendantsOption.OMIT_DESCENDANTS + def myLimit = 123 + when: 'queryDataNodes (with limit) is invoked' + objectUnderTest.queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption, myLimit) + then: 'the persistence service is called once with the correct parameters' + 1 * mockCpsDataPersistenceService.queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption, myLimit) + and: 'the CpsValidator is called on the dataspaceName, schemaSetName and anchorName' + 1 * mockCpsValidator.validateNameCharacters(dataspaceName, anchorName) + } + def 'Query data nodes across all anchors by cps path with #fetchDescendantsOption.'() { given: 'a dataspace name, an anchor name and a cps path' def dataspaceName = 'some-dataspace' @@ -73,11 +88,10 @@ class CpsQueryServiceImplSpec extends Specification { 1 * mockCpsDataPersistenceService.countAnchorsForDataspaceAndCpsPath("some-dataspace", "/cps-path") } - // TODO will be implemented in CPS-2416 def 'Query data leaf.'() { when: 'a query for a specific leaf is executed' objectUnderTest.queryDataLeaf('some-dataspace', 'some-anchor', '/cps-path/@id', Object.class) then: 'solution is not implemented yet' - thrown(UnsupportedOperationException) + 1 * mockCpsDataPersistenceService.queryDataLeaf('some-dataspace', 'some-anchor', '/cps-path/@id', Object.class) } } diff --git a/cps-service/src/test/groovy/org/onap/cps/api/model/DataNodeBuilderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/DataNodeBuilderSpec.groovy index 24c78864a5..1597d45761 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/model/DataNodeBuilderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/DataNodeBuilderSpec.groovy @@ -19,10 +19,11 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model +package org.onap.cps.impl import org.onap.cps.TestUtils import org.onap.cps.api.exceptions.DataValidationException +import org.onap.cps.api.model.DataNode import org.onap.cps.utils.ContentType import org.onap.cps.utils.DataMapUtils import org.onap.cps.utils.YangParserHelper diff --git a/cps-service/src/test/groovy/org/onap/cps/api/model/DeltaReportBuilderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/DeltaReportBuilderSpec.groovy index 94e3ed5c26..2decefff21 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/model/DeltaReportBuilderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/DeltaReportBuilderSpec.groovy @@ -18,7 +18,7 @@ * ============LICENSE_END========================================================= */ -package org.onap.cps.api.model +package org.onap.cps.impl import spock.lang.Specification diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/E2ENetworkSliceSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/E2ENetworkSliceSpec.groovy index 4ab71f7228..db5b4f104e 100755 --- a/cps-service/src/test/groovy/org/onap/cps/impl/E2ENetworkSliceSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/E2ENetworkSliceSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation. + * Copyright (C) 2021-2025 Nordix Foundation. * Modifications Copyright (C) 2021-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022-2024 TechMahindra Ltd. @@ -28,7 +28,7 @@ import org.onap.cps.TestUtils import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDeltaService import org.onap.cps.events.CpsDataUpdateEventsService -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.spi.CpsModulePersistenceService import org.onap.cps.api.model.Anchor @@ -66,32 +66,32 @@ class E2ENetworkSliceSpec extends Specification { def 'E2E model can be parsed by CPS.'() { given: 'Valid yang resource as name-to-content map' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap( 'ietf/ietf-inet-types@2013-07-15.yang', 'ietf/ietf-yang-types@2013-07-15.yang', 'e2e/basic/ran-network2020-08-06.yang' ) when: 'Create schema set method is invoked' - cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourceContentPerName) then: 'Parameters are validated and processing is delegated to persistence service' - 1 * mockModuleStoreService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + 1 * mockModuleStoreService.createSchemaSet(dataspaceName, schemaSetName, yangResourceContentPerName) } def 'E2E Coverage Area-Tracking Area & TA-Cell mapping model can be parsed by CPS.'() { given: 'Valid yang resource as name-to-content map' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap( 'e2e/basic/cps-cavsta-onap-internal2021-01-28.yang') when: 'Create schema set method is invoked' - cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + cpsModuleServiceImpl.createSchemaSet(dataspaceName, schemaSetName, yangResourceContentPerName) then: 'Parameters are validated and processing is delegated to persistence service' - 1 * mockModuleStoreService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap) + 1 * mockModuleStoreService.createSchemaSet(dataspaceName, schemaSetName, yangResourceContentPerName) } def 'E2E Coverage Area-Tracking Area & TA-Cell mapping data can be parsed by CPS.'() { given: 'Valid yang resource as name-to-content map' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap( 'e2e/basic/cps-cavsta-onap-internal2021-01-28.yang') - def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName).getSchemaContext() def dataNodeStored and : 'a valid json is provided for the model' def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-Cavsta-Data.txt') @@ -99,7 +99,7 @@ class E2ENetworkSliceSpec extends Specification { mockCpsAnchorService.getAnchor(dataspaceName, anchorName) >> new Anchor().builder().name(anchorName).schemaSetName(schemaSetName).dataspaceName(dataspaceName).build() mockYangTextSchemaSourceSetCache.get(dataspaceName, schemaSetName) >> - YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName) mockModuleStoreService.getYangSchemaResources(dataspaceName, schemaSetName) >> schemaContext when: 'saveData method is invoked' cpsDataServiceImpl.saveData(dataspaceName, anchorName, jsonData, noTimestamp) @@ -123,15 +123,15 @@ class E2ENetworkSliceSpec extends Specification { def 'E2E Coverage Area-Tracking Area & TA-Cell mapping data can be parsed for RAN inventory.'() { def dataNodeStored given: 'valid yang resource as name-to-content map' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap( 'e2e/basic/cps-ran-inventory@2021-01-28.yang') - def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName).getSchemaContext() and : 'a valid json is provided for the model' def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-ran-inventory-data.json') and : 'all the further dependencies are mocked ' mockCpsAnchorService.getAnchor('someDataspace', 'someAnchor') >> new Anchor().builder().name('someAnchor').schemaSetName('someSchemaSet').dataspaceName(dataspaceName).build() - mockYangTextSchemaSourceSetCache.get('someDataspace', 'someSchemaSet') >> YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + mockYangTextSchemaSourceSetCache.get('someDataspace', 'someSchemaSet') >> YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName) mockModuleStoreService.getYangSchemaResources('someDataspace', 'someSchemaSet') >> schemaContext when: 'saveData method is invoked' cpsDataServiceImpl.saveData('someDataspace', 'someAnchor', jsonData, noTimestamp) @@ -161,7 +161,7 @@ class E2ENetworkSliceSpec extends Specification { def 'E2E RAN Schema Model.'(){ given: 'yang resources' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap( + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap( 'ietf/ietf-inet-types@2013-07-15.yang', 'ietf/ietf-yang-types@2013-07-15.yang', 'e2e/basic/cps-ran-schema-model@2021-05-19.yang' @@ -169,7 +169,7 @@ class E2ENetworkSliceSpec extends Specification { and : 'json data' def jsonData = TestUtils.getResourceFileContent('e2e/basic/cps-ran-schema-model-data-v4.json') expect: 'schema context is built with no exception indicating the schema set being valid ' - def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap).getSchemaContext() + def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName).getSchemaContext() and: 'data is parsed with no exception indicating the model match' new YangParserHelper().parseData(ContentType.JSON, jsonData, schemaContext, '', false) != null } diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/YangTextSchemaSourceSetCacheSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/YangTextSchemaSourceSetCacheSpec.groovy index fe49d04f3c..e71782ce4f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/YangTextSchemaSourceSetCacheSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/YangTextSchemaSourceSetCacheSpec.groovy @@ -22,7 +22,7 @@ package org.onap.cps.impl import org.onap.cps.TestUtils -import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.utils.CpsValidator import org.onap.cps.spi.CpsModulePersistenceService import org.onap.cps.yang.YangTextSchemaSourceSet import org.onap.cps.yang.YangTextSchemaSourceSetBuilder @@ -65,13 +65,13 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { given: 'cache is empty' yangResourceCacheImpl.clear() and: 'a schema set exists' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') - def expectedYangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') + def expectedYangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName) when: 'schema-set information is asked' def result = objectUnderTest.get('my-dataspace', 'my-schemaset') then: 'information fetched from cps module persistence' 1 * mockModuleStoreService.getYangSchemaResources('my-dataspace', 'my-schemaset') - >> yangResourcesNameToContentMap + >> yangResourceContentPerName and: 'stored in the cache' def cachedValue = getCachedValue('my-dataspace', 'my-schemaset') assert cachedValue.getModuleReferences() == expectedYangTextSchemaSourceSet.getModuleReferences() @@ -83,8 +83,8 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { def 'Cache Hit: Respond from cache'() { given: 'a schema set exists' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') - def expectedYangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') + def expectedYangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName) and: 'stored in cache' yangResourceCacheImpl.put(getCacheKey('my-dataspace', 'my-schemaset'), expectedYangTextSchemaSourceSet) when: 'schema-set information is asked' @@ -97,8 +97,8 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { def 'Cache Update: when no data exist in the cache'() { given: 'a schema set exists' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') - def yangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') + def yangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName) when: 'cache is updated' objectUnderTest.updateCache('my-dataspace', 'my-schemaset', yangTextSchemaSourceSet) then: 'cached value is same as expected' @@ -110,8 +110,8 @@ class YangTextSchemaSourceSetCacheSpec extends Specification { def 'Cache Evict:with invalid #scenario'() { given: 'a schema set exists in cache' - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') - def yangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap) + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') + def yangTextSchemaSourceSet = YangTextSchemaSourceSetBuilder.of(yangResourceContentPerName) yangResourceCacheImpl.put(getCacheKey('my-dataspace', 'my-schemaset'), yangTextSchemaSourceSet) def cachedValue = getCachedValue('my-dataspace', 'my-schemaset') assert cachedValue.getModuleReferences() == yangTextSchemaSourceSet.getModuleReferences() diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy index bb0f5b0911..6ff41c128f 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/DataMapUtilsSpec.groovy @@ -22,7 +22,7 @@ package org.onap.cps.utils -import org.onap.cps.api.model.DataNodeBuilder +import org.onap.cps.impl.DataNodeBuilder import spock.lang.Specification class DataMapUtilsSpec extends Specification { diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy index a2fadb7e9f..cb7a16cb84 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/YangParserSpec.groovy @@ -40,7 +40,7 @@ class YangParserSpec extends Specification { def objectUnderTest = new YangParser(mockYangParserHelper, mockYangTextSchemaSourceSetCache, mockTimedYangTextSchemaSourceSetBuilder) def anchor = new Anchor(dataspaceName: 'my dataspace', schemaSetName: 'my schema') - def yangResourcesNameToContentMap = TestUtils.getYangResourcesAsMap('bookstore.yang') + def yangResourceContentPerName = TestUtils.getYangResourcesAsMap('bookstore.yang') def mockYangTextSchemaSourceSet = Mock(YangTextSchemaSourceSet) def mockSchemaContext = Mock(SchemaContext) def containerNodeFromYangUtils = Mock(ContainerNode) @@ -91,9 +91,9 @@ class YangParserSpec extends Specification { def 'Parsing data with yang resource to context map.'() { given: 'the schema source set for the yang resource map is returned' - mockTimedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap) >> mockYangTextSchemaSourceSet + mockTimedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourceContentPerName) >> mockYangTextSchemaSourceSet when: 'parsing some json data' - def result = objectUnderTest.parseData(ContentType.JSON, 'some json', yangResourcesNameToContentMap, noParent) + def result = objectUnderTest.parseData(ContentType.JSON, 'some json', yangResourceContentPerName, noParent) then: 'the yang parser helper always returns a container node' 1 * mockYangParserHelper.parseData(ContentType.JSON, 'some json', mockSchemaContext, noParent, validateAndParse) >> containerNodeFromYangUtils and: 'the result is the same container node as return from yang utils' diff --git a/csit/tests/cps-trust-level/cps-trust-level.robot b/csit/tests/cps-trust-level/cps-trust-level.robot index 810bcf4d12..98ec665a6c 100644 --- a/csit/tests/cps-trust-level/cps-trust-level.robot +++ b/csit/tests/cps-trust-level/cps-trust-level.robot @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,11 +46,11 @@ Register data node Should Be Equal As Strings ${response.status_code} 200 Verify notification - ${group_id}= Create Consumer auto_offset_reset=earliest - Subscribe Topic topics=cm-events group_id=${group_id} - ${result}= Poll group_id=${group_id} only_value=False poll_attempts=5 - ${headers} Set Variable ${result[0].headers()} - ${payload} Set Variable ${result[0].value()} + ${group_id}= Create Consumer auto_offset_reset=earliest + Subscribe Topic topics=ncmp-inventory-events group_id=${group_id} + ${result}= Poll group_id=${group_id} only_value=False poll_attempts=5 + ${headers} Set Variable ${result[0].headers()} + ${payload} Set Variable ${result[0].value()} FOR ${header_key_value_pair} IN @{headers} Compare Header Values ${header_key_value_pair[0]} ${header_key_value_pair[1]} "ce_specversion" "1.0" Compare Header Values ${header_key_value_pair[0]} ${header_key_value_pair[1]} "ce_source" "NCMP" diff --git a/docker-compose/config/endurance.env b/docker-compose/config/endurance.env index be337219cf..e46bd5429d 100644 --- a/docker-compose/config/endurance.env +++ b/docker-compose/config/endurance.env @@ -1,6 +1,8 @@ DB_CONTAINER_NAME=endurance-dbpostgresql DB_PORT=5433 +POSTGRES_EXPORTER_PORT_RANGE=9187-9188 + NGINX_CONTAINER_NAME=endurance-nginx-loadbalancer CPS_CORE_PORT=8884 CPS_PORT_RANGE=8798-8799 diff --git a/docker-compose/config/grafana/cps-database-pool.json b/docker-compose/config/grafana/cps-database-pool.json new file mode 100644 index 0000000000..37782af71a --- /dev/null +++ b/docker-compose/config/grafana/cps-database-pool.json @@ -0,0 +1,886 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "expr": "resets(process_uptime_seconds{application=\"$application\", region=\"$region\", instance=\"$instance\"}[1m]) > 0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Restart Detection", + "showIn": 0, + "step": "1m", + "tagKeys": "restart-tag", + "tags": [], + "textFormat": "uptime reset", + "titleFormat": "Restart", + "type": "tags" + } + ] + }, + "description": "HikariCP & JDBC Dashboard (Micrometer.io)", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 16, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "title": "JDBC Connections", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 6, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jdbc_connections_min{application=\"$application\", region=~\"$region\", instance=~\"$instance\", name=~\"$jdbc_connection_name\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Min", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 7, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jdbc_connections_active{application=\"$application\", region=~\"$region\", instance=~\"$instance\", name=~\"$jdbc_connection_name\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Active", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 8, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jdbc_connections_max{application=\"$application\", region=~\"$region\", instance=~\"$instance\", name=~\"$jdbc_connection_name\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 2, + "panels": [], + "title": "Hikari Connections", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 21, + "x": 0, + "y": 5 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "hikaricp_connections_active{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active connections", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "hikaricp_connections_idle{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Idle connections", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "hikaricp_connections_pending{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pending threads", + "refId": "B" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 5 + }, + "id": 12, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "hikaricp_connections_max{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 9 + }, + "id": 13, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "hikaricp_connections_min{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Min", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 13 + }, + "id": 17, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "hikaricp_connections_timeout_total{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total Timeout", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(hikaricp_connections_usage_seconds_sum{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}[5m]) / irate(hikaricp_connections_usage_seconds_count{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Usage time", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(hikaricp_connections_creation_seconds_sum{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}[5m]) / irate(hikaricp_connections_creation_seconds_count{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Creation time", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(hikaricp_connections_acquire_seconds_sum{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}[5m]) / irate(hikaricp_connections_acquire_seconds_count{application=\"$application\", region=~\"$region\", instance=~\"$instance\", pool=~\"$hikaricp_pool_name\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Acquire time", + "refId": "B" + } + ], + "title": "Connections Time", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [ + "prometheus", + "hikaricp", + "micrometer", + "spring boot", + "jdbc" + ], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "", + "includeAll": false, + "label": "Application", + "name": "application", + "options": [], + "query": "label_values(application)", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "", + "includeAll": true, + "label": "Region", + "name": "region", + "options": [], + "query": "label_values(jdbc_connections_min{application=\"$application\"}, region)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "129.192.80.24:9998", + "value": "129.192.80.24:9998" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(jdbc_connections_min{application=\"$application\", region=~\"$region\"}, instance)", + "includeAll": true, + "label": "Instance", + "name": "instance", + "options": [], + "query": "label_values(jdbc_connections_min{application=\"$application\", region=~\"$region\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "dataSource", + "value": "dataSource" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(jdbc_connections_min{application=\"$application\", region=~\"$region\", instance=~\"$instance\"}, name)", + "includeAll": false, + "label": "JDBC Connection Name", + "name": "jdbc_connection_name", + "options": [], + "query": "label_values(jdbc_connections_min{application=\"$application\", region=~\"$region\", instance=~\"$instance\"}, name)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "CpsDatabasePool", + "value": "CpsDatabasePool" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(hikaricp_connections_min{application=\"$application\", region=~\"$region\", instance=~\"$instance\"}, pool)", + "includeAll": false, + "label": "Hikari Pool Name", + "name": "hikaricp_pool_name", + "options": [], + "query": "label_values(hikaricp_connections_min{application=\"$application\", region=~\"$region\", instance=~\"$instance\"}, pool)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cps Database Pool", + "uid": "wdV6wx7iz", + "version": 1, + "weekStart": "" +}
\ No newline at end of file diff --git a/docker-compose/config/grafana/data-dashboard.json b/docker-compose/config/grafana/data-dashboard.json new file mode 100644 index 0000000000..4fbb65ee5b --- /dev/null +++ b/docker-compose/config/grafana/data-dashboard.json @@ -0,0 +1,276 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "NCMP Rest Interfaces", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "http_server_requests_seconds_count{instance=\"$Instance\", job=\"$Job\", status=~\"201|200|203|204\", uri=~\"/ncmp/v1/ch/id-searches|/ncmp/v1/ch/searches|/ncmp/v1/ch/{cm-handle}/data/ds/{datastore-name}|/ncmp/v1/data\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Inventory API calls by URI and STATUS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum({error=\"OutOfMemoryError\", instance=\"$Instance\", job=\"$Job\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Out of Memory Errors", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [ + "ncmp", + "data", + "api" + ], + "templating": { + "list": [ + { + "current": { + "text": "129.192.80.24:9998", + "value": "129.192.80.24:9998" + }, + "definition": "label_values(instance)", + "label": "Instance", + "name": "Instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "cps-and-ncmp-kpi", + "value": "cps-and-ncmp-kpi" + }, + "definition": "label_values(job)", + "label": "job", + "name": "Job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "2025-01-24T10:09:19.223Z", + "to": "2025-01-24T10:32:39.091Z" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Data REST Interfaces", + "uid": "aeavdgvjyt2iob", + "version": 1, + "weekStart": "" +}
\ No newline at end of file diff --git a/docker-compose/config/grafana/lcm-state-dashboard.json b/docker-compose/config/grafana/inventory-dashboard.json index 5339b038d7..959995fa69 100644 --- a/docker-compose/config/grafana/lcm-state-dashboard.json +++ b/docker-compose/config/grafana/inventory-dashboard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 3, "links": [], "panels": [ { @@ -179,111 +179,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, "y": 9 }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "cmHandlesByState{state=\"ADVISED\", instance=\"$Instance\", job=\"$Job\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "ADVISED", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 3, + "id": 6, "options": { "legend": { "calcs": [], @@ -301,7 +201,7 @@ { "disableTextWrap": false, "editorMode": "builder", - "expr": "cmHandlesByState{state=\"READY\", instance=\"$Instance\", job=\"$Job\"}", + "expr": "http_server_requests_seconds_count{instance=\"$Instance\", job=\"$Job\", status=~\"201|200\", uri=\"/ncmpInventory/v1/ch\"}", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", @@ -310,7 +210,7 @@ "useBackend": false } ], - "title": "READY", + "title": "Inventory API calls by URI and STATUS", "type": "timeseries" }, { @@ -375,11 +275,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, "y": 17 }, - "id": 2, + "id": 7, "options": { "legend": { "calcs": [], @@ -397,7 +297,7 @@ { "disableTextWrap": false, "editorMode": "builder", - "expr": "cmHandlesByState{state=\"LOCKED\", instance=\"$Instance\", job=\"$Job\"}", + "expr": "sum({error=\"OutOfMemoryError\", instance=\"$Instance\", job=\"$Job\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", @@ -406,115 +306,23 @@ "useBackend": false } ], - "title": "LOCKED", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "cmHandlesByState{state=\"DELETING\", instance=\"$Instance\", job=\"$Job\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "DELETING", + "title": "Out of Memory Errors", "type": "timeseries" } ], "preload": false, "schemaVersion": 40, - "tags": [], + "tags": [ + "lcm-state", + "inventory", + "ncmp" + ], "templating": { "list": [ { "current": { - "text": "172.17.0.1:8799", - "value": "172.17.0.1:8799" + "text": "129.192.80.24:9998", + "value": "129.192.80.24:9998" }, "definition": "label_values(instance)", "label": "Instance", @@ -531,8 +339,8 @@ }, { "current": { - "text": "cps-and-ncmp-endurance", - "value": "cps-and-ncmp-endurance" + "text": "cps-and-ncmp-kpi", + "value": "cps-and-ncmp-kpi" }, "definition": "label_values(job)", "label": "job", @@ -555,8 +363,8 @@ }, "timepicker": {}, "timezone": "browser", - "title": "LCM State", - "uid": "ae9zcowku03k0d", + "title": "Inventory REST Interfaces", + "uid": "beao8xrt6qjnkc", "version": 1, "weekStart": "" }
\ No newline at end of file diff --git a/docker-compose/config/grafana/jvm-micrometer-dashboard.json b/docker-compose/config/grafana/jvm-micrometer-dashboard.json index 8f7747c596..9f09d2e2cb 100644 --- a/docker-compose/config/grafana/jvm-micrometer-dashboard.json +++ b/docker-compose/config/grafana/jvm-micrometer-dashboard.json @@ -22,6 +22,7 @@ }, "enable": true, "expr": "resets(process_uptime_seconds{application=\"$application\", instance=\"$instance\"}[1m]) > 0", + "hide": false, "iconColor": "rgba(255, 96, 96, 1)", "name": "Restart Detection", "showIn": 0, @@ -35,17 +36,12 @@ "description": "Dashboard for Micrometer instrumented applications (Java, Spring Boot, Micronaut)", "editable": true, "fiscalYearStartMonth": 0, - "gnetId": 4701, "graphTooltip": 1, - "id": 1, + "id": 3, "links": [], "panels": [ { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, @@ -54,15 +50,6 @@ }, "id": 139, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "Quick Facts", "type": "row" }, @@ -73,6 +60,10 @@ }, "fieldConfig": { "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, "decimals": 1, "mappings": [ { @@ -127,7 +118,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -153,6 +144,10 @@ }, "fieldConfig": { "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, "mappings": [ { "options": { @@ -206,7 +201,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -292,7 +287,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -387,7 +382,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -407,10 +402,6 @@ }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, @@ -419,15 +410,6 @@ }, "id": 140, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "I/O Overview", "type": "row" }, @@ -448,6 +430,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -513,6 +496,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -546,6 +530,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -642,6 +627,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -675,6 +661,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -740,6 +727,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -787,6 +775,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -852,6 +841,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -931,10 +921,6 @@ }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, @@ -943,15 +929,6 @@ }, "id": 141, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "JVM Memory", "type": "row" }, @@ -972,6 +949,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1038,6 +1016,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1097,6 +1076,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1163,6 +1143,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1223,6 +1204,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1289,6 +1271,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1348,6 +1331,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1414,6 +1398,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1468,10 +1453,6 @@ }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, @@ -1480,15 +1461,6 @@ }, "id": 142, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "JVM Misc", "type": "row" }, @@ -1509,6 +1481,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1577,6 +1550,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1637,6 +1611,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1704,6 +1679,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1750,6 +1726,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1817,6 +1794,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1890,6 +1868,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2046,6 +2025,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -2080,6 +2060,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2148,6 +2129,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -2181,6 +2163,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2324,6 +2307,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -2360,6 +2344,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2428,6 +2413,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -2463,10 +2449,6 @@ }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, @@ -2476,15 +2458,6 @@ "id": 143, "panels": [], "repeat": "persistence_counts", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "JVM Memory Pools (Heap)", "type": "row" }, @@ -2536,8 +2509,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2551,7 +2523,7 @@ }, "gridPos": { "h": 7, - "w": 8, + "w": 24, "x": 0, "y": 36 }, @@ -2572,6 +2544,7 @@ } }, "repeat": "jvm_memory_pool_heap", + "repeatDirection": "h", "targets": [ { "datasource": { @@ -2624,10 +2597,6 @@ }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, @@ -2636,66 +2605,29 @@ }, "id": 144, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "JVM Memory Pools (Non-Heap)", "type": "row" }, { - "aliasColors": {}, - "autoMigrateFrom": "graph", - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "rightLogBase": 1 + "fieldConfig": { + "defaults": {}, + "overrides": [] }, "gridPos": { "h": 7, - "w": 8, + "w": 24, "x": 0, "y": 44 }, "id": 78, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "maxPerRow": 3, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", + "options": {}, "repeat": "jvm_memory_pool_nonheap", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "repeatDirection": "h", "targets": [ { "datasource": { @@ -2743,104 +2675,39 @@ "step": 1800 } ], - "thresholds": [], "title": "$jvm_memory_pool_nonheap", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "timeseries", - "x-axis": true, - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "y-axis": true, - "y_formats": [ - "mbytes", - "short" - ], - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ] + "type": "timeseries" }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 51 }, "id": 145, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "Garbage Collection", "type": "row" }, { - "aliasColors": {}, - "autoMigrateFrom": "graph", - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fill": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 0, - "y": 59 + "y": 52 }, "id": 98, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "options": {}, "targets": [ { "datasource": { @@ -2855,72 +2722,26 @@ "refId": "A" } ], - "thresholds": [], "title": "Collections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "show": true - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "autoMigrateFrom": "graph", - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fill": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 8, - "y": 59 + "y": 52 }, "id": 101, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "options": {}, "targets": [ { "datasource": { @@ -2949,72 +2770,26 @@ "refId": "B" } ], - "thresholds": [], "title": "Pause Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "show": true - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "autoMigrateFrom": "graph", - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fill": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 16, - "y": 59 + "y": 52 }, "id": 99, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "options": {}, "targets": [ { "datasource": { @@ -3041,103 +2816,39 @@ "refId": "B" } ], - "thresholds": [], "title": "Allocated/Promoted", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ] + "type": "timeseries" }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 66 + "y": 59 }, "id": 146, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "Classloading", "type": "row" }, { - "aliasColors": {}, - "autoMigrateFrom": "graph", - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "rightLogBase": 1 + "fieldConfig": { + "defaults": {}, + "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 67 + "y": 60 }, "id": 37, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "options": {}, "targets": [ { "datasource": { @@ -3153,84 +2864,26 @@ "step": 1200 } ], - "thresholds": [], "title": "Classes loaded", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "timeseries", - "x-axis": true, - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "y-axis": true, - "y_formats": [ - "short", - "short" - ], - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "autoMigrateFrom": "graph", - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "rightLogBase": 1 + "fieldConfig": { + "defaults": {}, + "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 67 + "y": 60 }, "id": 38, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "options": {}, "targets": [ { "datasource": { @@ -3248,115 +2901,42 @@ "step": 1200 } ], - "thresholds": [], "title": "Class delta", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "timeseries", - "x-axis": true, - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "y-axis": true, - "y_formats": [ - "ops", - "short" - ], - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ] + "type": "timeseries" }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 74 + "y": 67 }, "id": 147, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], "title": "Buffer Pools", "type": "row" }, { - "aliasColors": {}, - "autoMigrateFrom": "graph", - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fill": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, - "w": 8, + "w": 24, "x": 0, - "y": 75 + "y": 68 }, "id": 131, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "maxPerRow": 3, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", + "options": {}, "repeat": "jvm_buffer_pool", - "seriesOverrides": [ - { - "alias": "count", - "yaxis": 2 - }, - { - "alias": "buffers", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "repeatDirection": "h", "targets": [ { "datasource": { @@ -3393,46 +2973,23 @@ "refId": "C" } ], - "thresholds": [], "title": "$jvm_buffer_pool", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "timeseries", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "decimals": 0, - "format": "short", - "label": "", - "logBase": 1, - "min": "0", - "show": true - } - ] + "type": "timeseries" } ], + "preload": false, "refresh": "auto", - "schemaVersion": 39, - "tags": [], + "schemaVersion": 40, + "tags": [ + "jvm", + "prometheus", + "micrometer" + ], "templating": { "list": [ { "current": { "isNone": true, - "selected": false, "text": "None", "value": "" }, @@ -3441,26 +2998,17 @@ "uid": "PBFA97CFB590B2093" }, "definition": "", - "hide": 0, "includeAll": false, "label": "Application", - "multi": false, "name": "application", "options": [], "query": "label_values(application)", "refresh": 2, "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allFormat": "glob", "current": { - "selected": false, "text": "docker-compose-cps-and-ncmp-1:8080", "value": "docker-compose-cps-and-ncmp-1:8080" }, @@ -3469,27 +3017,17 @@ "uid": "PBFA97CFB590B2093" }, "definition": "", - "hide": 0, "includeAll": false, "label": "Instance", - "multi": false, - "multiFormat": "glob", "name": "instance", "options": [], "query": "label_values(jvm_memory_used_bytes{application=\"$application\"}, instance)", "refresh": 2, "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allFormat": "glob", "current": { - "selected": false, "text": "All", "value": "$__all" }, @@ -3501,24 +3039,16 @@ "hide": 2, "includeAll": true, "label": "JVM Memory Pools Heap", - "multi": false, - "multiFormat": "glob", "name": "jvm_memory_pool_heap", "options": [], "query": "label_values(jvm_memory_used_bytes{application=\"$application\", instance=\"$instance\", area=\"heap\"},id)", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allFormat": "glob", "current": { - "selected": false, "text": "All", "value": "$__all" }, @@ -3530,24 +3060,16 @@ "hide": 2, "includeAll": true, "label": "JVM Memory Pools Non-Heap", - "multi": false, - "multiFormat": "glob", "name": "jvm_memory_pool_nonheap", "options": [], "query": "label_values(jvm_memory_used_bytes{application=\"$application\", instance=\"$instance\", area=\"nonheap\"},id)", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 2, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allFormat": "glob", "current": { - "selected": false, "text": "All", "value": "$__all" }, @@ -3559,19 +3081,13 @@ "hide": 2, "includeAll": true, "label": "JVM Buffer Pools", - "multi": false, - "multiFormat": "glob", "name": "jvm_buffer_pool", "options": [], "query": "label_values(jvm_buffer_memory_used_bytes{application=\"$application\", instance=\"$instance\"},id)", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, @@ -3579,35 +3095,10 @@ "from": "now-30m", "to": "now" }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, + "timepicker": {}, "timezone": "browser", "title": "JVM (Micrometer)", "uid": "bdvp1kgecrda8f", "version": 1, "weekStart": "" -} +}
\ No newline at end of file diff --git a/docker-compose/config/grafana/postgresql-statistics-dashboard.json b/docker-compose/config/grafana/postgresql-statistics-dashboard.json new file mode 100644 index 0000000000..afc428f8cf --- /dev/null +++ b/docker-compose/config/grafana/postgresql-statistics-dashboard.json @@ -0,0 +1,2239 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for PostgreSQL Statistics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "title": "Settings", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_static{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 54, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_connections{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max Connections", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 1 + }, + "id": 56, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Shared Buffers", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 1 + }, + "id": 58, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Effective Cache", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 1 + }, + "id": 60, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Maintenance Work Mem", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 66, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Work Mem", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 32, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max WAL Size", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 62, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Random Page Cost", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 70, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_seq_page_cost{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Seq Page Cost", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 4 + }, + "id": 64, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max Worker Processes", + "type": "stat" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 4 + }, + "id": 68, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max Parallel Workers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 72, + "panels": [], + "title": "Database", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 72, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "editorMode": "code", + "expr": "pg_database_size_bytes{instance=\"$instance\", datname=\"$db\"}", + "legendFormat": "size", + "range": true, + "refId": "A" + } + ], + "title": "Size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 36, + "panels": [], + "title": "Connection / Transaction Statistics", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_stat_activity_count{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "commits", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "rollbacks", + "refId": "B" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rows", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_fetched{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SELECT (index scan)", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_returned{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SELECT (table scan)", + "refId": "B" + } + ], + "title": "Read Stats", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rows", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_inserted{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "INSERT", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_updated{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UPDATE", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_tup_deleted{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DELETE", + "refId": "C" + } + ], + "title": "Change Stats", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_stat_activity_max_tx_duration{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max_tx_duration [{{state}}]", + "refId": "A" + } + ], + "title": "Longest Transaction", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=\"$db\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=\"$db\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=\"$db\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Cache Hit Rate", + "refId": "A" + } + ], + "title": "Cache Hit Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 50, + "panels": [], + "title": "misc", + "type": "row" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 0, + "y": 44 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_backend_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_backend", + "refId": "A" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_alloc_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_alloc", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "backend_fsync", + "refId": "C" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_checkpoint", + "refId": "D" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_buffers_clean_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "buffers_clean", + "refId": "E" + } + ], + "title": "Buffers (bgwriter)", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 44 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "conflicts", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "deadlocks", + "refId": "A" + } + ], + "title": "Conflicts/Deadlocks", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 17, + "x": 0, + "y": 50 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "pg_locks_count{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "title": "Lock Tables", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 50 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Temp Bytes", + "refId": "A" + } + ], + "title": "Temp File", + "type": "timeseries" + }, + { + "datasource": "PBFA97CFB590B2093", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", + "refId": "B" + }, + { + "datasource": "PBFA97CFB590B2093", + "exemplar": true, + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", + "refId": "A" + } + ], + "title": "Checkpoint Stats", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [ + "postgresql", + "postgres-exporter" + ], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(pg_up, instance)", + "includeAll": false, + "label": "instance", + "name": "instance", + "options": [], + "query": { + "query": "label_values(pg_up, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "cpsdb", + "value": "cpsdb" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(pg_stat_database_tup_fetched{datname!~\"template.*|postgres\",instance=\"$instance\"},datname)", + "includeAll": false, + "label": "Database", + "name": "db", + "options": [], + "query": { + "query": "label_values(pg_stat_database_tup_fetched{datname!~\"template.*|postgres\",instance=\"$instance\"},datname)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PostgreSQL Statistics", + "uid": "OpKZVIAMz", + "version": 1, + "weekStart": "" +}
\ No newline at end of file diff --git a/docker-compose/config/prometheus.yml b/docker-compose/config/prometheus.yml index 8db07c8d05..1beea6ebce 100644 --- a/docker-compose/config/prometheus.yml +++ b/docker-compose/config/prometheus.yml @@ -20,4 +20,10 @@ scrape_configs: static_configs: - targets: - '172.17.0.1:8798' - - '172.17.0.1:8799'
\ No newline at end of file + - '172.17.0.1:8799' + +- job_name: 'postgres' + static_configs: + - targets: + - '172.17.0.1:9187' + - '172.17.0.1:9188'
\ No newline at end of file diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index c9df8b9889..8e42bc58be 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -68,7 +68,8 @@ services: ONAP_OTEL_EXPORTER_ENDPOINT: http://jaeger-service:4317 POLICY_SERVICE_ENABLED: 'false' POLICY_SERVICE_DEFAULT_DECISION: 'deny from env' - JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=75.0 -XX:MaxRAMPercentage=75.0" + CPS_MONITORING_MICROMETER_JVM_EXTRAS: 'true' + JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0" ### DEBUG: Uncomment next line to enable java debugging ### JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 restart: unless-stopped @@ -80,7 +81,8 @@ services: resources: limits: cpus: '3' - memory: 2G + memory: 3G + memswap_limit: 3G nginx: container_name: ${NGINX_CONTAINER_NAME:-nginx-loadbalancer} @@ -95,7 +97,7 @@ services: ### if kafka is not required comment out zookeeper and kafka ### zookeeper: - image: confluentinc/cp-zookeeper:6.2.1 + image: confluentinc/cp-zookeeper:7.8.0 container_name: ${ZOOKEEPER_CONTAINER_NAME:-zookeeper} ports: - ${ZOOKEEPER_PORT:-2181}:2181 @@ -103,7 +105,7 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 kafka: - image: confluentinc/cp-kafka:6.2.1 + image: confluentinc/cp-kafka:7.8.0 container_name: ${KAFKA_CONTAINER_NAME:-kafka} ports: - ${KAFKA_PORT:-9092}:9092 @@ -150,7 +152,7 @@ services: KAFKA_BOOTSTRAP_SERVER: kafka:29092 NCMP_CONSUMER_GROUP_ID: ncmp-group NCMP_ASYNC_M2M_TOPIC: ncmp-async-m2m - MODULE_INITIAL_PROCESSING_DELAY_MS: 120000 + MODULE_INITIAL_PROCESSING_DELAY_MS: 180000 MODULE_REFERENCES_DELAY_MS: 100 MODULE_RESOURCES_DELAY_MS: 1000 READ_DATA_FOR_CM_HANDLE_DELAY_MS: 300 @@ -199,7 +201,10 @@ services: volumes: - ./config/grafana/provisioning/:/etc/grafana/provisioning/ - ./config/grafana/jvm-micrometer-dashboard.json:/var/lib/grafana/dashboards/jvm-micrometer-dashboard.json - - ./config/grafana/lcm-state-dashboard.json:/var/lib/grafana/dashboards/lcm-state-dashboard.json + - ./config/grafana/inventory-dashboard.json:/var/lib/grafana/dashboards/inventory-dashboard.json + - ./config/grafana/data-dashboard.json:/var/lib/grafana/dashboards/data-dashboard.json + - ./config/grafana/postgresql-statistics-dashboard.json:/var/lib/grafana/dashboards/postgresql-statistics-dashboard.json + - ./config/grafana/cps-database-pool.json:/var/lib/grafana/dashboards/cps-database-pool.json - grafana:/var/lib/grafana environment: - GF_SECURITY_ADMIN_PASSWORD=admin @@ -228,6 +233,15 @@ services: profiles: - tracing + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter + environment: + - DATA_SOURCE_NAME=postgresql://${DB_USERNAME:-cps}:${DB_PASSWORD:-cps}@${DB_CONTAINER_NAME:-dbpostgresql}:5432/postgres?sslmode=disable + ports: + - ${POSTGRES_EXPORTER_PORT_RANGE:-9187-9188}:9187 + depends_on: + - dbpostgresql + volumes: grafana: driver: local diff --git a/docs/admin-guide.rst b/docs/admin-guide.rst index 4a40f9b29c..9009de21c9 100644 --- a/docs/admin-guide.rst +++ b/docs/admin-guide.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2022 Nordix Foundation +.. Copyright (C) 2021-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _adminGuide: @@ -12,6 +12,40 @@ CPS Admin Guide .. toctree:: :maxdepth: 1 +Regular Maintenance +=================== +This section details tasks that an administrator of the CPS application should execute on regular basis +to ensure optimum working of CPS. + +Dataspace Clean Up +------------------ +Certain data in the CPS database might not be explicitly removed after it is no longer required ('orphaned data'). +For example, schema sets and their associated unique module resources no longer used by any anchor because of model upgrades. +This data would unnecessarily take up space and could eventually affect the performance of the DB if it is not deleted. +How often this needs to be done depends on how often schema sets are being deprecated. +Typically once per month should suffice. + +To remove orphaned data in a given dataspace use the following post request: + +.. code:: + + http://<cps-component-service-name>:<cps-port>/v2/admin/dataspaces/<dataspace-name>/actions/clean + +for example + +.. code-block:: bash + + curl --location --request POST 'http://cps:8080/admin/datsaspaces/bookstore/actions/clean' \ + --header 'Content-Type: application/json; charset=utf-8' + + Response : HTTP Status 204 + +For more details refer to the CPS-Core API: :doc:`design`. + +.. note:: + NCMP has no specific maintenance tasks but it will also build up orphaned data when CM Handles get updated and or deleted. + To delete this data execute the above procedure for the dataspace named 'NFP-Operational'. + Logging Configuration ===================== @@ -192,17 +226,15 @@ Prometheus Metrics can be checked at the following endpoint http://<cps-component-service-name>:8080/actuator/prometheus -Hazelcast ---------- +Heap Dump and Thread Dump +------------------------- -Hazelcast cluster state and health check can be seen using the below endpoints +On demand heap dump and thread dump generation using the below endpoints. .. code:: - http://<cps-component-service-name>:<member-port>/hazelcast/health - http://<cps-component-service-name>:<member-port>/hazelcast/rest/management/cluster/state - -See also : :ref:`cps_common_distributed_datastructures` + http://<cps-component-service-name>:8080/actuator/heapdump + http://<cps-component-service-name>:8080/actuator/threaddump Naming Validation ----------------- @@ -210,9 +242,8 @@ Naming Validation As part of the Kohn 3.1.0 release, CPS has added validation to the names of the following components: - Dataspace names - - Schema Set names - Anchor names - - Cm-Handle identifiers + - CM Handle identifiers The following characters along with spaces are no longer valid for naming of these components. diff --git a/docs/api/swagger/cps/openapi.yaml b/docs/api/swagger/cps/openapi.yaml index c84609b638..f6baadc8ea 100644 --- a/docs/api/swagger/cps/openapi.yaml +++ b/docs/api/swagger/cps/openapi.yaml @@ -330,6 +330,65 @@ paths: summary: Get a dataspace tags: - cps-admin + /{apiVersion}/admin/dataspaces/{dataspace-name}/actions/clean: + post: + description: Clean the dataspace (remove orphaned schema sets and modules) + operationId: cleanDataspace + parameters: + - description: apiVersion + in: path + name: apiVersion + required: true + schema: + default: v2 + enum: + - v1 + - v2 + type: string + - description: dataspace-name + in: path + name: dataspace-name + required: true + schema: + example: my-dataspace + type: string + responses: + "204": + content: {} + description: No Content + "400": + content: + application/json: + example: + status: 400 + message: Bad Request + details: The provided request is not valid + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Bad Request + "403": + content: + application/json: + example: + status: 403 + message: Request Forbidden + details: This request is forbidden + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Forbidden + "500": + content: + application/json: + example: + status: 500 + message: Internal Server Error + details: Internal Server Error occurred + schema: + $ref: '#/components/schemas/ErrorMessage' + description: Internal Server Error + summary: Clean the dataspace + tags: + - cps-admin /v1/dataspaces/{dataspace-name}/anchors: post: deprecated: true diff --git a/docs/cm-handle-lcm-events.rst b/docs/cm-handle-lcm-events.rst index 8446834c31..38339e206a 100644 --- a/docs/cm-handle-lcm-events.rst +++ b/docs/cm-handle-lcm-events.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmHandleLcmEvents: @@ -15,7 +15,7 @@ CM Handle Lifecycle Management (LCM) Events Introduction ============ -LCM events for CM Handles are published when a CM Handle is created, deleted or another change in the cm handle state occurs. +LCM events for CM Handles are published when a CM Handle is created, deleted or another change in the CM Handle state occurs. **3 possible event types:** @@ -55,7 +55,7 @@ Event payload varies based on the type of event. **CREATE** -Event payload for this event contains the properties of the new cm handle created. +Event payload for this event contains the properties of the new CM Handle created. *Create event payload prototype* @@ -77,7 +77,7 @@ Event payload for this event contains the properties of the new cm handle create **UPDATE** -Event payload for this event contains the difference in state and properties of the cm handle. +Event payload for this event contains the difference in state and properties of the CM Handle. *Update event payload prototype* @@ -106,7 +106,7 @@ Event payload for this event contains the difference in state and properties of **DELETE** -Event payload for this event contains the identifier of the deleted cm handle. +Event payload for this event contains the identifier of the deleted CM Handle. *Delete event payload prototype* @@ -114,4 +114,4 @@ Event payload for this event contains the identifier of the deleted cm handle. "event": { "cmHandleId" : "cmhandle-001", - }
\ No newline at end of file + } diff --git a/docs/cps-events.rst b/docs/cps-events.rst index 47aa73f12e..1097af9a12 100644 --- a/docs/cps-events.rst +++ b/docs/cps-events.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2022-2023 Nordix Foundation +.. Copyright (C) 2022-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cpsEvents: @@ -16,9 +16,9 @@ CPS Events cm-notification-subscriptions.rst .. note:: - Legacy async response on a client supplied topic for single cm handle data request are no longer supported. Click link below for the legacy specification. + Legacy async response on a client supplied topic for single CM Handle data request are no longer supported. Click link below for the legacy specification. .. toctree:: :maxdepth: 0 - ncmp-async-events.rst
\ No newline at end of file + ncmp-async-events.rst diff --git a/docs/cps-ncmp-message-status-codes.rst b/docs/cps-ncmp-message-status-codes.rst index e0a3f0308b..799838ae44 100644 --- a/docs/cps-ncmp-message-status-codes.rst +++ b/docs/cps-ncmp-message-status-codes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023-2024 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _dataOperationMessageStatusCodes: @@ -16,9 +16,9 @@ CPS-NCMP Message Status Codes +-----------------+------------------------------------------------------+-----------------------------------+ | 1 | ACCEPTED | CM Data Notification Subscription | +-----------------+------------------------------------------------------+-----------------------------------+ - | 100 | cm handle id(s) is(are) not found | All features | + | 100 | CM Handle id(s) is(are) not found | All features | +-----------------+------------------------------------------------------+-----------------------------------+ - | 101 | cm handle(s) not ready | Data Operation | + | 101 | CM Handle(s) not ready | Data Operation | +-----------------+------------------------------------------------------+-----------------------------------+ | 102 | dmi plugin service is not responding | Data Operation | +-----------------+------------------------------------------------------+-----------------------------------+ @@ -30,9 +30,9 @@ CPS-NCMP Message Status Codes +-----------------+------------------------------------------------------+-----------------------------------+ | 108 | Unknown error | All features | +-----------------+------------------------------------------------------+-----------------------------------+ - | 109 | cm-handle already exists | Inventory | + | 109 | CM Handle already exists | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ - | 110 | cm-handle has an invalid character(s) in id | Inventory | + | 110 | CM Handle has an invalid character(s) in id | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ | 111 | alternate id already associated | Inventory | +-----------------+------------------------------------------------------+-----------------------------------+ diff --git a/docs/cps-scheduled-processes.rst b/docs/cps-scheduled-processes.rst index c204e6ca0a..9af9a81c0c 100644 --- a/docs/cps-scheduled-processes.rst +++ b/docs/cps-scheduled-processes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2022 Nordix Foundation +.. Copyright (C) 2022-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING @@ -21,17 +21,17 @@ The following section is a list of the current scheduled processes running withi Module Sync ----------- The module sync is a user :ref:`configurable timed process<additional-cps-ncmp-customizations>`, -which is set to search for CM-Handles within CPS with an *'ADVISED'* state. -Once the CM-Handle is processed by the module sync, the CM-Handle state is then set to *'READY'*, if the process completes successfully. -If for any reason the module sync fails, the CM-Handle state will then be set to *'LOCKED'*, +which is set to search for CM Handles within CPS with an *'ADVISED'* state. +Once the CM Handle is processed by the module sync, the CM Handle state is then set to *'READY'*, if the process completes successfully. +If for any reason the module sync fails, the CM Handle state will then be set to *'LOCKED'*, and the reason for the lock will also be stored within CPS. -CM-Handles in the *'LOCKED'* state will be retried when the system has availability. CM-Handles in a *'LOCKED'* -state are processed by the retry mechanism, by setting CM-Handle state back to *'ADVISED'* so the next sync cycle will process those again. +CM Handles in the *'LOCKED'* state will be retried when the system has availability. CM Handles in a *'LOCKED'* +state are processed by the retry mechanism, by setting CM Handle state back to *'ADVISED'* so the next sync cycle will process those again. Data Sync --------- The data sync is a user :ref:`configurable timed process<additional-cps-ncmp-customizations>`, -which is set to search for CM-Handles with a sync state of *'UNSYNCHRONIZED'*. -Once the CM-Handle(s) with a sync state of *'UNSYNCHRONIZED'* is processed by the data sync, -the CM-Handle sync state is then set to *'SYNCHRONIZED'*, if the process completes successfully. -If the data sync fails, the CM-Handle sync state will remain as *'UNSYNCHRONIZED'*, and will be re-attempted. +which is set to search for CM Handles with a sync state of *'UNSYNCHRONIZED'*. +Once the CM Handle(s) with a sync state of *'UNSYNCHRONIZED'* is processed by the data sync, +the CM Handle sync state is then set to *'SYNCHRONIZED'*, if the process completes successfully. +If the data sync fails, the CM Handle sync state will remain as *'UNSYNCHRONIZED'*, and will be re-attempted. diff --git a/docs/deployment.rst b/docs/deployment.rst index 940bc50923..840ab8e116 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -22,14 +22,24 @@ set appropriately. For example, given a database with 2GB of memory, 512MB is a CPS and NCMP Configuration ========================== +CPU and Memory Requirements +--------------------------- + +The following are minimum requirements for NCMP: + +* For 20,000 CM-handles: 2 CPUs and 2 GB RAM per instance, with 70% heap allocation. +* For 50,000 CM-handles: 3 CPUs and 3 GB RAM per instance, with 70% heap allocation. + JVM Memory Allocation +^^^^^^^^^^^^^^^^^^^^^ -Allocating 75% of the container's memory to the JVM heap ensures efficient memory management. -This helps the JVM make the best use of the allocated resources while leaving enough memory for other processes. +When running with 2 GB or more memory per instance, allocating 70% of the JVM memory to the heap ensures efficient +memory management. It is not recommended to go above 70%. .. code-block:: yaml - JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=75.0 -XX:MaxRAMPercentage=75.0" + JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0" + Load balancer configuration =========================== @@ -191,21 +201,16 @@ Any spring supported property can be configured by providing in ``config.additio | Property | Description | Default Value | +===========================================+=========================================================================================================+===============================+ | config.appUserName | User name used by cps-core service to configure the authentication for REST API it exposes. | ``cpsuser`` | -| | | | | | This is the user name to be used by cps-core REST clients to authenticate themselves. | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.appUserPassword | Password used by cps-core service to configure the authentication for REST API it exposes. | Not defined | -| | | | | | If not defined, the password is generated when deploying the application. | | -| | | | | | See also :ref:`cps_common_credentials_retrieval`. | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | postgres.config.pgUserName | Internal user name used by cps-core to connect to its own database. | ``cps`` | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | postgres.config.pgUserPassword | Internal password used by cps-core to connect to its own database. | Not defined | -| | | | | | If not defined, the password is generated when deploying the application. | | -| | | | | | See also :ref:`cps_common_credentials_retrieval`. | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | postgres.config.pgDatabase | Database name used by cps-core | ``cpsdb`` | @@ -225,28 +230,24 @@ Any spring supported property can be configured by providing in ``config.additio +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security protocol. | ``SASL_PLAINTEXT`` | | spring.kafka.security.protocol | Some possible values are: | | -| | | | | | * ``PLAINTEXT`` | | | | * ``SASL_PLAINTEXT``, for authentication | | | | * ``SASL_SSL``, for authentication and encryption | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL mechanism. Required for SASL_PLAINTEXT and SASL_SSL protocols. | Not defined | | spring.kafka.properties. | Some possible values are: | | -| sasl.mechanism | | | -| | * ``PLAIN``, for PLAINTEXT | | +| sasl.mechanism | * ``PLAIN``, for PLAINTEXT | | | | * ``SCRAM-SHA-512``, for SSL | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL JAAS configuration. Required for SASL_PLAINTEXT and SASL_SSL protocols. | Not defined | | spring.kafka.properties. | Some possible values are: | | -| sasl.jaas.config | | | -| | * ``org.apache.kafka.common.security.plain.PlainLoginModule required username="..." password="...";``, | | -| | for PLAINTEXT | | +| sasl.jaas.config | * ``org.apache.kafka.common.security.plain.PlainLoginModule required username="..." password="...";``, | | +| | for PLAINTEXT | | | | * ``org.apache.kafka.common.security.scram.ScramLoginModule required username="..." password="...";``, | | -| | for SSL | | +| | for SSL | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL SSL store type. Required for SASL_SSL protocol. | Not defined | | spring.kafka.ssl.trust-store-type | Some possible values are: | | -| | | | | | * ``JKS`` | | +-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | config.eventPublisher. | Kafka security SASL SSL store file location. Required for SASL_SSL protocol. | Not defined | @@ -293,35 +294,84 @@ Any spring supported property can be configured by providing in ``config.additio Additional CPS-NCMP Customizations ================================== -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.dmiPluginUserName | User name used by cps-core to authenticate themselves for using ncmp-dmi-plugin service. | ``dmiuser`` | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.dmiPluginUserPassword | Internal password used by cps-core to connect to ncmp-dmi-plugin service. | Not defined | -| | | | -| | If not defined, the password is generated when deploying the application. | | -| | | | -| | See also :ref:`cps_common_credentials_retrieval`. | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.ncmp.timers | Specifies the delay in milliseconds in which the module sync watch dog will wake again after finishing. | ``5000`` | -| .advised-modules-sync.sleep-time-ms | | | -| | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.ncmp.timers | Specifies the delay in milliseconds in which the data sync watch dog will wake again after finishing. | ``30000`` | -| .cm-handle-data-sync.sleep-time-ms | | | -| | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum time in seconds, to wait for establishing a connection for the HTTP Client. | ``30`` | -| .connectionTimeoutInSeconds | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum number of connections allowed per route in the HTTP client. | ``50`` | -| .maximumConnectionsPerRoute | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the maximum total number of connections that can be held by the HTTP client. | ``100`` | -| .maximumConnectionsTotal | | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| config.additional.ncmp.dmi.httpclient | Specifies the duration in seconds for the threshold, after which idle connections will be evicted | ``5`` | -| .idleConnectionEvictionThresholdInSeconds | from the connection pool by the HTTP client. | | -+-------------------------------------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ + ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| Property | Description | Default Value | ++=================================================+=======================================================================================+=================================+ +| config.dmiPluginUserName | User name used by cps-core to authenticate themselves for using ncmp-dmi-plugin | ``dmiuser`` | +| | service. | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.dmiPluginUserPassword | Internal password used by cps-core to connect to ncmp-dmi-plugin service. | Not defined | +| | If not defined, the password is generated when deploying the application. | | +| | See also :ref:`cps_common_credentials_retrieval`. | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.ncmp.timers | Specifies the delay in milliseconds in which the module sync watch dog will wake again| ``5000`` | +| .advised-modules-sync.sleep-time-ms | after finishing. | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.ncmp.timers | Specifies the delay in milliseconds in which the data sync watch dog will wake again | ``30000`` | +| .cm-handle-data-sync.sleep-time-ms | after finishing. | | +| | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Maximum size (in MB) of the in-memory buffer for HTTP response data. | ``16`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .maximumInMemorySizeInMegabytes | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Maximum number of simultaneous connections allowed in the connection pool. | ``100`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .maximumConnectionsTotal | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Maximum number of pending requests when the connection pool is full. | ``50`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .pendingAcquireMaxCount | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Specifies the maximum time in seconds, to wait for establishing a connection for the | ``30`` | +| .[app] | HTTP Client. | | +| .httpclient | | | +| .[services] | | | +| .connectionTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Timeout (in seconds) for reading data from the server after the connection is | ``30`` | +| .[app] | established. | | +| .httpclient | | | +| .[services] | | | +| .readTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Timeout (in seconds) for writing data to the server. | ``30`` | +| .[app] | | | +| .httpclient | | | +| .[services] | | | +| .writeTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp | Total timeout (in seconds) for receiving a complete response, including all processing| ``60`` | +| .[app] | stages. | | +| .httpclient | | | +| .[services] | | | +| .responseTimeoutInSeconds | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | Enables or disables the policy-executor feature. | ``false`` | +| .enabled | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | The default (fallback) decision in case a problem with the external service occurs. | ``allow`` | +| .defaultDecision | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | The server address for the external policy executor service. | ``http://policy-executor-stub`` | +| .server.address | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ +| config.additional.ncmp.policy-executor | The port used for the external policy executor service. | ``8093`` | +| .server.port | | | ++-------------------------------------------------+---------------------------------------------------------------------------------------+---------------------------------+ + +.. note:: + + - [app] : can be 'policy-executor' or 'dmi'. + - [services] 'all-services' for 'policy-executor'. + - [services] 'data-services' and 'model-services' for 'dmi'. CPS-Core Docker Installation ============================ @@ -342,13 +392,13 @@ Below are the list of distributed datastructures that we have. +--------------+------------------------------------+-----------------------------------------------------------+ | Component | Data Structure Name | Use | +==============+====================================+===========================================================+ -| cps-ncmp | moduleSyncStartedOnCmHandles | Watchdog process to register cm handles. | +| cps-ncmp | moduleSyncStartedOnCmHandles | Watchdog process to register CM Handles. | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | dataSyncSemaphores | Watchdog process to sync data from the nodes. | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | moduleSyncWorkQueue | Queue used internally for workers to pick the task. | +--------------+------------------------------------+-----------------------------------------------------------+ -| cps-ncmp | trustLevelPerCmHandle | Stores the trust level per cm handle id | +| cps-ncmp | trustLevelPerCmHandle | Stores the trust level per CM Handle id | +--------------+------------------------------------+-----------------------------------------------------------+ | cps-ncmp | trustLevelPerDmiPlugin | Stores the trust level for the dmi-plugins. | +--------------+------------------------------------+-----------------------------------------------------------+ diff --git a/docs/modeling.rst b/docs/modeling.rst index 7ebf6fecd5..65e4aa97ca 100644 --- a/docs/modeling.rst +++ b/docs/modeling.rst @@ -1,7 +1,7 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 .. Copyright (C) 2021 Pantheon.tech -.. Modifications Copyright (C) 2021-2023 Nordix Foundation +.. Modifications Copyright (C) 2021-2025 Nordix Foundation .. _modeling: .. toctree:: @@ -89,31 +89,31 @@ Note: Although additional-properties are present in the model of the dmi-registr Basic Concepts -------------- -- **CM-Handle** represents an instance a modeled Network Function(node) in ONAP. +- **CM Handle** represents an instance a modeled Network Function(node) in ONAP. These are stored as Anchors within CPS-Core. - - **CM-Handle States** are used to represent the potential states in which a CM-Handle can transition between. + - **CM Handle States** are used to represent the potential states in which a CM Handle can transition between. - The 5 possible CM-Handle states are: ADVISED, READY, LOCKED, DELETING, DELETED + The 5 possible CM Handle states are: ADVISED, READY, LOCKED, DELETING, DELETED - **ADVISED** indicates that a CM-Handle has been registered successfully, and is waiting for the module synchronization process to sync the CM-Handle. + **ADVISED** indicates that a CM Handle has been registered successfully, and is waiting for the module synchronization process to sync the CM Handle. - **READY** indicates that the CM-Handle has been synced successfully. + **READY** indicates that the CM Handle has been synced successfully. - **LOCKED** indicates that the CM-Handle has not synced successfully. A retry mechanism within CPS will set the state back to ADVISED after a set time. + **LOCKED** indicates that the CM Handle has not synced successfully. A retry mechanism within CPS will set the state back to ADVISED after a set time. - **DELETING** indicates that the CM-Handle is currently being deleted. + **DELETING** indicates that the CM Handle is currently being deleted. - **DELETED** indicates that the CM-Handle has been deleted successfully. + **DELETED** indicates that the CM Handle has been deleted successfully. - - **Data-sync state** is the state of the data synchronization process of the CM-Handle + - **Data-sync state** is the state of the data synchronization process of the CM Handle There are 3 possibles states: NONE_REQUESTED, UNSYNCHRONIZED, SYNCHRONIZED **NONE_REQUESTED** indicates that the data sync is not requested by the user - **UNSYNCHRONIZED** indicates the cm-handle is waiting for the data sync watchdog operation to carry out the sync process + **UNSYNCHRONIZED** indicates the CM Handle is waiting for the data sync watchdog operation to carry out the sync process **SYNCHRONIZED** indicates the watchdog process has finished the data synchronization successfully diff --git a/docs/ncmp-data-operation.rst b/docs/ncmp-data-operation.rst index 10c3bfaca5..e0b7bb88cd 100644 --- a/docs/ncmp-data-operation.rst +++ b/docs/ncmp-data-operation.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023-2024 Nordix Foundation +.. Copyright (C) 2023-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmHandleDataOperation: @@ -15,7 +15,7 @@ Data Operations Endpoint Introduction ============ -For all data operations on cm handle(s), we have a post endpoint: +For all data operations on CM Handle(s), we have a post endpoint: - /ncmp/v1/data?topic={client-topic-name} forward request to it's dmi plugin service. @@ -50,7 +50,7 @@ This endpoint executes data operation for given array of operations: | | | implementation. For ONAP DMI Plugin it will be RESTConf paths but it can| | | | really be anything. | +--------------------------+-------------+-------------------------------------------------------------------------+ - | targetIds | Yes | List of cm handle references | + | targetIds | Yes | List of CM Handle references | +--------------------------+-------------+-------------------------------------------------------------------------+ The status codes used in the events resulting from these operations are defined here: @@ -156,4 +156,4 @@ DMI Service 2 (POST) : `http://{dmi-host-name}:{dmi-port}/dmi/v1/data?topic=my-t Above examples are for illustration purposes only. Please refer to link below for latest schema. -:download:`Data operation event schema <schemas/data-operation-event-schema-1.0.0.json>`
\ No newline at end of file +:download:`Data operation event schema <schemas/data-operation-event-schema-1.0.0.json>` diff --git a/docs/overview.rst b/docs/overview.rst index 19ab8b4847..bc99214e2e 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -1,6 +1,8 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021 Pantheon.tech, Nordix Foundation +.. Copyright (C) 2021 Pantheon.tech +.. Modifications Copyright (C) 2021-2025 Nordix Foundation + .. _overview: CPS Overview @@ -45,7 +47,7 @@ even though CPS-Core could be deployed without the NCMP extension. NCMP-DMI-Plugin --------------- -The Data-Model-Inventory (DMI) Plugin is a rest interface used to synchronize CM-Handles data between CPS and DMI through the DMI-Plugin. +The Data-Model-Inventory (DMI) Plugin is a rest interface used to synchronize CM Handles data between CPS and DMI through the DMI-Plugin. This is built previously from the CPS-NF-Proxy component. CPS Project diff --git a/docs/policy-executor.rst b/docs/policy-executor.rst index b934a579b1..712b4fcf38 100644 --- a/docs/policy-executor.rst +++ b/docs/policy-executor.rst @@ -1,11 +1,10 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2024 Nordix Foundation +.. Copyright (C) 2024-2025 Nordix Foundation -.. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING +.. DO NOT CHANGE THIS LABEL - EVEN THOUGH IT GIVES A WARNING .. _policy_executor: - Policy Executor ############### @@ -15,7 +14,16 @@ Policy Executor Introduction ============ -Work In Progress: This feature is not yet completed and does not affect current NCMP functionality. +The Policy Executor feature can be used to connect an external system to make decisions on CM write operation. +When the feature is enabled, NCMP will first call the configured external system and depending on the response, return an error or continue. +The details of the interface can be found in the ':ref:`policy_executor_consumed_apis`' section. + +This feature is available on 'legacy data interface' for operation on a single cm handle: "/v1/ch/{cm-handle}/data/ds/{datastore-name}" and only applies to "ncmp-datastore:passthrough-running". + +By default, the feature is not enabled. This is controlled by 'config.additional.ncmp.policy-executor.enabled' and other deployment parameters in the same group to enable it. See :ref:`additional-cps-ncmp-customizations` + +.. DO NOT CHANGE THIS LABEL - EVEN THOUGH IT GIVES A WARNING +.. _policy_executor_consumed_apis: Consumed APIs ------------- diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 9c825e4d35..d2ed1b8044 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2024 Nordix Foundation +.. Copyright (C) 2021-2025 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _release_notes: @@ -15,6 +15,29 @@ CPS Release Notes .. * * * PARIS * * * .. ==================== +Version: 3.6.1 +============== + +Release Data +------------ + ++--------------------------------------+--------------------------------------------------------+ +| **CPS Project** | | +| | | ++--------------------------------------+--------------------------------------------------------+ +| **Docker images** | onap/cps-and-ncmp:3.6.1 | +| | | ++--------------------------------------+--------------------------------------------------------+ +| **Release designation** | 3.6.1 Paris | +| | | ++--------------------------------------+--------------------------------------------------------+ +| **Release date** | Not yet released | +| | | ++--------------------------------------+--------------------------------------------------------+ + +Features +-------- + Version: 3.6.0 ============== @@ -31,15 +54,19 @@ Release Data | **Release designation** | 3.6.0 Paris | | | | +--------------------------------------+--------------------------------------------------------+ -| **Release date** | Not yet released | +| **Release date** | 2025 January 29 | | | | +--------------------------------------+--------------------------------------------------------+ Bug Fixes --------- + - `CPS-2563 <https://lf-onap.atlassian.net/browse/CPS-2563>`_ Fix for internal error code during duplicated registration. + - `CPS-2576 <https://lf-onap.atlassian.net/browse/CPS-2576>`_ Fix for cm handle stuck in LOCKED state during registration. Features -------- + - `CPS-2249 <https://lf-onap.atlassian.net/browse/CPS-2249>`_ NCMP to support Conflict Handling. + - `CPS-2540 <https://lf-onap.atlassian.net/browse/CPS-2540>`_ One schemaset per module set tag. .. ==================== @@ -68,21 +95,19 @@ Release Data Bug Fixes --------- -3.5.5 - `CPS-2509 <https://lf-onap.atlassian.net/browse/CPS-2509>`_ Fix module endpoints using alternate identifier. - `CPS-2517 <https://lf-onap.atlassian.net/browse/CPS-2517>`_ Make Content-Type header default to JSON for CPS APIs. - - `CPS-2530 <https://lf-onap.atlassian.net/browse/CPS-2530>`_ NCMP Modules API giving empty response on READY cm handles if two sub systems discovered in parallel. + - `CPS-2530 <https://lf-onap.atlassian.net/browse/CPS-2530>`_ NCMP Modules API giving empty response on READY CM Handles if two sub systems discovered in parallel. Features -------- -3.5.5 - `CPS-2009 <https://lf-onap.atlassian.net/browse/CPS-2009>`_ Update legacy NCMP APIs interfaces to support alternate id. - `CPS-2082 <https://lf-onap.atlassian.net/browse/CPS-2082>`_ Support XML content type to data node APIs in cps-core. - `CPS-2433 <https://lf-onap.atlassian.net/browse/CPS-2433>`_ Remove traces of unmaintained CPS-TBDMT repository. - `CPS-2436 <https://lf-onap.atlassian.net/browse/CPS-2436>`_ CM Avc Event to publish source key to target key while forwarding. - `CPS-2445 <https://lf-onap.atlassian.net/browse/CPS-2445>`_ Expose CPS and NCMP version information using git plugin. - `CPS-2451 <https://lf-onap.atlassian.net/browse/CPS-2451>`_ Removing oparent from CPS-NCMP and ONAP DMI Plugin repository. - - `CPS-2478 <https://lf-onap.atlassian.net/browse/CPS-2478>`_ Optimized Cm Handle Registration and De-Registration use case. + - `CPS-2478 <https://lf-onap.atlassian.net/browse/CPS-2478>`_ Optimized CM Handle Registration and De-Registration use case. - `CPS-2507 <https://lf-onap.atlassian.net/browse/CPS-2507>`_ Upgrade liquibase to 4.30.0 version. Performance @@ -111,12 +136,10 @@ Release Data Bug Fixes --------- -3.5.4 - `CPS-2403 <https://lf-onap.atlassian.net/browse/CPS-2403>`_ Improve lock handling and queue management during CM-handle Module Sync. Features -------- -3.5.4 - `CPS-2408 <https://lf-onap.atlassian.net/browse/CPS-2408>`_ One Hazelcast instance per JVM to manage the distributed data structures. Version: 3.5.3 @@ -141,7 +164,6 @@ Release Data Bug Fixes --------- -3.5.3 - `CPS-2353 <https://lf-onap.atlassian.net/browse/CPS-2353>`_ Slow cmHandle registration when we use moduleSetTag, alternateId and dataProducerIdentifier - `CPS-2395 <https://lf-onap.atlassian.net/browse/CPS-2395>`_ Retry mechanism (with back off algorithm) is removed with more frequent watchdog poll - `CPS-2409 <https://lf-onap.atlassian.net/browse/CPS-2409>`_ Return NONE for get effective trust level api if the trust level caches empty (restart case) @@ -150,9 +172,6 @@ Bug Fixes Features -------- -3.5.3 - - `CPS-2247 <https://lf-onap.atlassian.net/browse/CPS-2247>`_ Policy Executor: Invoke Policy Executor and handle 'deny' response - - `CPS-2412 <https://lf-onap.atlassian.net/browse/CPS-2412>`_ Policy Executor: handle errors - `CPS-2417 <https://lf-onap.atlassian.net/browse/CPS-2417>`_ Remove Hazelcast cache for prefix resolver @@ -178,14 +197,12 @@ Release Data Bug Fixes --------- -3.5.2 - `CPS-2306 <https://lf-onap.atlassian.net/browse/CPS-2306>`_ Update response message for data validation failure and make it consistent across APIs - `CPS-2319 <https://lf-onap.atlassian.net/browse/CPS-2319>`_ Fix "Create a node" and "Add List Elements" APIs response code - `CPS-2372 <https://lf-onap.atlassian.net/browse/CPS-2372>`_ Blank alternate ID overwrites existing one Features -------- -3.5.2 - `CPS-1812 <https://lf-onap.atlassian.net/browse/CPS-1812>`_ CM Data Subscriptions ( Create, Delete and Merging ) with positive scenarios - `CPS-2326 <https://lf-onap.atlassian.net/browse/CPS-2326>`_ Uplift liquibase-core dependency to 4.28.0 - `CPS-2353 <https://lf-onap.atlassian.net/browse/CPS-2353>`_ Improve registration performance with moduleSetTag @@ -213,14 +230,12 @@ Release Data Bug Fixes --------- -3.5.1 - `CPS-2302 <https://lf-onap.atlassian.net/browse/CPS-2302>`_ Fix handling of special characters in moduleSetTag. Features -------- -3.5.1 - `CPS-2121 <https://lf-onap.atlassian.net/browse/CPS-2121>`_ Enabled http client prometheus metrics and manage high cardinality using URL template. - - `CPS-2289 <https://lf-onap.atlassian.net/browse/CPS-2289>`_ Support for CPS Path Query in NCMP Inventory Cm Handle Search. + - `CPS-2289 <https://lf-onap.atlassian.net/browse/CPS-2289>`_ Support for CPS Path Query in NCMP Inventory CM Handle Search. Version: 3.5.0 ============== @@ -242,13 +257,8 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.5.0 - Features -------- -3.5.0 - `CPS-989 <https://lf-onap.atlassian.net/browse/CPS-989>`_ Replace RestTemplate with WebClient. - `CPS-2172 <https://lf-onap.atlassian.net/browse/CPS-2172>`_ Support for OpenTelemetry Tracing. @@ -278,12 +288,10 @@ Release Data Bug Fixes --------- -3.4.9 - `CPS-2211 <https://lf-onap.atlassian.net/browse/CPS-2211>`_ Toggle switch to disable CPS Core change events if not used by application. Set CPS_CHANGE_EVENT_NOTIFICATIONS_ENABLED environment variable for the same. Features -------- -3.4.9 - `CPS-1836 <https://lf-onap.atlassian.net/browse/CPS-1836>`_ Delta between anchor and JSON payload. Version: 3.4.8 @@ -308,15 +316,11 @@ Release Data Bug Fixes --------- -3.4.8 - `CPS-2186 <https://lf-onap.atlassian.net/browse/CPS-2186>`_ Report async task failures to client topic during data operations request - `CPS-2190 <https://lf-onap.atlassian.net/browse/CPS-2190>`_ Improve performance of NCMP module searches - `CPS-2194 <https://lf-onap.atlassian.net/browse/CPS-2194>`_ Added defaults for CPS and DMI username and password - `CPS-2204 <https://lf-onap.atlassian.net/browse/CPS-2204>`_ Added error handling for yang module upgrade operation -Features --------- - Version: 3.4.7 ============== @@ -339,12 +343,10 @@ Release Data Bug Fixes --------- -3.4.7 - `CPS-2150 <https://lf-onap.atlassian.net/browse/CPS-2150>`_ Fix for Async task execution failed by TimeoutException. Features -------- -3.4.7 - `CPS-2061 <https://lf-onap.atlassian.net/browse/CPS-2061>`_ Liquibase Steps Condensing and Cleanup. - `CPS-2101 <https://lf-onap.atlassian.net/browse/CPS-2101>`_ Uplift Spring Boot to 3.2.4 version. @@ -370,7 +372,6 @@ Release Data Bug Fixes --------- -3.4.6 - `CPS-2126 <https://lf-onap.atlassian.net/browse/CPS-2126>`_ Passing HTTP Authorization Bearer Token to DMI Plugins. @@ -406,10 +407,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.4.5 - Features -------- @@ -438,7 +435,6 @@ Release Data Bug Fixes --------- -3.4.4 - `CPS-2027 <https://lf-onap.atlassian.net/browse/CPS-2027>`_ Upgrade Yang modules using module set tag functionalities fix Features @@ -469,7 +465,6 @@ Release Data Bug Fixes --------- -3.4.3 - `CPS-2000 <https://lf-onap.atlassian.net/browse/CPS-2000>`_ Fix for Schema object cache not being distributed. - `CPS-2027 <https://lf-onap.atlassian.net/browse/CPS-2027>`_ Fixes for upgrade yang modules using module set tag. - `CPS-2070 <https://lf-onap.atlassian.net/browse/CPS-2070>`_ Add retry interval for Kafka consumer. @@ -506,11 +501,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.4.2 - - Features -------- - `CPS-1638 <https://lf-onap.atlassian.net/browse/CPS-1638>`_ Introduce trust level for CM handle. @@ -530,7 +520,7 @@ Known Limitations, Issues and Workarounds For upgrading, CPS uses Liquibase for database upgrades. In order to enable Hibernate write batching (`CPS-1795 <https://lf-onap.atlassian.net/browse/CPS-1795>`_), a change to the database entity ID generation is required. As such, *this release does not fully support In-Service Software Upgrade* - CPS will not store new DataNodes and -NCMP will not register new CM-handles during an upgrade with old and new versions of CPS running concurrently. +NCMP will not register new CM Handles during an upgrade with old and new versions of CPS running concurrently. Other operations (read, update, delete) are not impacted. @@ -556,7 +546,6 @@ Release Data Bug Fixes --------- -3.4.1 - `CPS-1979 <https://lf-onap.atlassian.net/browse/CPS-1979>`_ Bug fix for Invalid topic name suffix. Features @@ -594,7 +583,6 @@ Release Data Bug Fixes --------- -3.4.0 - `CPS-1956 <https://lf-onap.atlassian.net/browse/CPS-1956>`_ Bug fix for No yang resources stored during cmhandle discovery. .. ======================== @@ -623,13 +611,9 @@ Release Data Bug Fixes --------- -3.3.9 - `CPS-1923 <https://lf-onap.atlassian.net/browse/CPS-1923>`_ CPS and NCMP changed management endpoint and port from /manage to /actuator and port same as cps application port. - `CPS-1933 <https://lf-onap.atlassian.net/browse/CPS-1933>`_ Setting up the class loader explicitly in hazelcast config. -Features --------- - Version: 3.3.8 ============== @@ -650,10 +634,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.8 - Features -------- - `CPS-1888 <https://lf-onap.atlassian.net/browse/CPS-1888>`_ Uplift Spring Boot to 3.1.2. @@ -680,7 +660,6 @@ Release Data Bug Fixes --------- -3.3.7 - `CPS-1866 <https://lf-onap.atlassian.net/browse/CPS-1866>`_ Fix ClassDefNotFoundError in opendaylight Yang parser Features @@ -713,7 +692,6 @@ Release Data Bug Fixes --------- -3.3.6 - `CPS-1841 <https://lf-onap.atlassian.net/browse/CPS-1841>`_ Update of top-level data node fails with exception - `CPS-1842 <https://lf-onap.atlassian.net/browse/CPS-1842>`_ Replace event-id with correlation-id for data read operation cloud event @@ -743,10 +721,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.5 - Features -------- - `CPS-1760 <https://lf-onap.atlassian.net/browse/CPS-1760>`_ Improve handling of special characters in Cps Paths @@ -771,10 +745,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.4 - Features -------- - `CPS-1767 <https://lf-onap.atlassian.net/browse/CPS-1767>`_ Upgrade CPS to java 17 @@ -799,13 +769,9 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.3 - Features -------- - - `CPS-1515 <https://lf-onap.atlassian.net/browse/CPS-1515>`_ Support Multiple CM-Handles for NCMP Get Operation + - `CPS-1515 <https://lf-onap.atlassian.net/browse/CPS-1515>`_ Support Multiple CM Handles for NCMP Get Operation - `CPS-1675 <https://lf-onap.atlassian.net/browse/CPS-1675>`_ Persistence write performance improvement(s) - `CPS-1745 <https://lf-onap.atlassian.net/browse/CPS-1745>`_ Upgrade to Openapi 3.0.3 @@ -831,7 +797,6 @@ Release Data Bug Fixes --------- -3.3.2 - `CPS-1716 <https://lf-onap.atlassian.net/browse/CPS-1716>`_ NCMP: Java Heap OutOfMemory errors and slow registration in case of 20k cmhandles Features @@ -862,11 +827,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.1 - - None - Features -------- - `CPS-1272 <https://lf-onap.atlassian.net/browse/CPS-1272>`_ Add Contains operation to CPS Path @@ -895,11 +855,6 @@ Release Data | | | +--------------------------------------+--------------------------------------------------------+ -Bug Fixes ---------- -3.3.0 - - None - Features -------- - `CPS-1215 <https://lf-onap.atlassian.net/browse/CPS-1215>`_ Add OR operation for CPS Path @@ -931,7 +886,6 @@ Release Data Bug Fixes --------- -3.2.6 - `CPS-1526 <https://lf-onap.atlassian.net/browse/CPS-1526>`_ Fix response message for PATCH operation - `CPS-1563 <https://lf-onap.atlassian.net/browse/CPS-1563>`_ Fix 500 response error on id-searches with empty parameters @@ -961,13 +915,8 @@ Release Data Bug Fixes --------- -3.2.5 - `CPS-1537 <https://lf-onap.atlassian.net/browse/CPS-1537>`_ Introduce control switch for model loader functionality. -Features --------- - - None - Version: 3.2.4 ============== @@ -1265,7 +1214,7 @@ Features - `CPS-869 <https://lf-onap.atlassian.net/browse/CPS-869>`_ Apply Standardized logging fields to adhere to ONAP Best practice REQ-1072 - `CPS-870 <https://lf-onap.atlassian.net/browse/CPS-870>`_ Align CPS-Core output with SDN-C output (add module name) - `CPS-875 <https://lf-onap.atlassian.net/browse/CPS-875>`_ CM Handle State: Watchdog-process that syncs 'ADVISED' CM Handles - - `CPS-877 <https://lf-onap.atlassian.net/browse/CPS-877>`_ CM Handle State: Exclude any CM-Handles from queries/operations that are not in state 'READY' + - `CPS-877 <https://lf-onap.atlassian.net/browse/CPS-877>`_ CM Handle State: Exclude any CM Handles from queries/operations that are not in state 'READY' - `CPS-899 <https://lf-onap.atlassian.net/browse/CPS-899>`_ Start and stop sessions on Java API - `CPS-909 <https://lf-onap.atlassian.net/browse/CPS-909>`_ Separate NCMP endpoint for ch/{cm-handle}/properties and ch/{cm-handle}/state - `CPS-917 <https://lf-onap.atlassian.net/browse/CPS-917>`_ Structured Errors response for passthrough use-cases in NCMP @@ -1278,7 +1227,7 @@ Features - `CPS-1099 <https://lf-onap.atlassian.net/browse/CPS-1099>`_ Expose simplified 'external' lock reason enum state over REST interface - `CPS-1101 <https://lf-onap.atlassian.net/browse/CPS-1101>`_ Introducing the DELETING and DELETED Cmhandle State - `CPS-1102 <https://lf-onap.atlassian.net/browse/CPS-1102>`_ Register the Cmhandle Sends Advised State notification. - - `CPS-1133 <https://lf-onap.atlassian.net/browse/CPS-1133>`_ Enable/Disable Data Sync for Cm Handle + - `CPS-1133 <https://lf-onap.atlassian.net/browse/CPS-1133>`_ Enable/Disable Data Sync for CM Handle - `CPS-1136 <https://lf-onap.atlassian.net/browse/CPS-1136>`_ DMI Audit Support (get all CM Handles for a registered DMI) @@ -1379,27 +1328,27 @@ Features - `CPS-559 <https://lf-onap.atlassian.net/browse/CPS-559>`_ Define response objects (schemas) in cps-ncmp - `CPS-636 <https://lf-onap.atlassian.net/browse/CPS-636>`_ Update operation for datastore pass through running - `CPS-638 <https://lf-onap.atlassian.net/browse/CPS-638>`_ Delete operation for datastore pass through running - - `CPS-677 <https://lf-onap.atlassian.net/browse/CPS-677>`_ Support 'public' Cm Handle Properties - - `CPS-741 <https://lf-onap.atlassian.net/browse/CPS-741>`_ Re sync after removing cm handles + - `CPS-677 <https://lf-onap.atlassian.net/browse/CPS-677>`_ Support 'public' CM Handle Properties + - `CPS-741 <https://lf-onap.atlassian.net/browse/CPS-741>`_ Re sync after removing CM Handles - `CPS-777 <https://lf-onap.atlassian.net/browse/CPS-777>`_ Ensure all DMI operations use POST method - `CPS-780 <https://lf-onap.atlassian.net/browse/CPS-780>`_ Add examples for parameters, request and response in openapi yaml for cps-core - `CPS-789 <https://lf-onap.atlassian.net/browse/CPS-789>`_ CPS Data Updated Event Schema V2 to support delete operation - `CPS-791 <https://lf-onap.atlassian.net/browse/CPS-791>`_ CPS-Core sends delete notification event - - `CPS-817 <https://lf-onap.atlassian.net/browse/CPS-817>`_ Create Endpoint For Get Cm Handles (incl. public properties) By Name + - `CPS-817 <https://lf-onap.atlassian.net/browse/CPS-817>`_ Create Endpoint For Get CM Handles (incl. public properties) By Name - `CPS-837 <https://lf-onap.atlassian.net/browse/CPS-837>`_ Add Remove and Update properties (DMI and Public) as part of CM Handle Registration update Bug Fixes --------- - - `CPS-762 <https://lf-onap.atlassian.net/browse/CPS-762>`_ Query cm handles for module names returns incorrect cm handle identifiers + - `CPS-762 <https://lf-onap.atlassian.net/browse/CPS-762>`_ Query CM Handles for module names returns incorrect CM Handle identifiers - `CPS-788 <https://lf-onap.atlassian.net/browse/CPS-788>`_ Yang Resource formatting is incorrect - - `CPS-783 <https://lf-onap.atlassian.net/browse/CPS-783>`_ Remove cm handle does not completely remove all cm handle information + - `CPS-783 <https://lf-onap.atlassian.net/browse/CPS-783>`_ Remove CM Handle does not completely remove all CM Handle information - `CPS-841 <https://lf-onap.atlassian.net/browse/CPS-841>`_ Upgrade log4j to 2.17.1 as recommended by ONAP SECCOM - `CPS-856 <https://lf-onap.atlassian.net/browse/CPS-856>`_ Retry mechanism not working for concurrent CmHandle registration - `CPS-867 <https://lf-onap.atlassian.net/browse/CPS-867>`_ Database port made configurable through env variable DB_PORT - `CPS-886 <https://lf-onap.atlassian.net/browse/CPS-886>`_ Fragment handling decreasing performance for large number of cmHandles - `CPS-887 <https://lf-onap.atlassian.net/browse/CPS-887>`_ Increase performance of cmHandle registration for large number of schema sets in DB - - `CPS-892 <https://lf-onap.atlassian.net/browse/CPS-892>`_ Fixed the response code during CM-Handle Registration from 201 CREATED to 204 NO_CONTENT + - `CPS-892 <https://lf-onap.atlassian.net/browse/CPS-892>`_ Fixed the response code during CM Handle Registration from 201 CREATED to 204 NO_CONTENT - `CPS-893 <https://lf-onap.atlassian.net/browse/CPS-893>`_ NCMP Java API depends on NCMP-Rest-API (cyclic) through json properties on Java API Known Limitations, Issues and Workarounds @@ -1407,9 +1356,9 @@ Known Limitations, Issues and Workarounds *System Limitations* -Null can no longer be passed within the dmi plugin service names when registering a cm handle, as part of +Null can no longer be passed within the dmi plugin service names when registering a CM Handle, as part of `CPS-837 <https://lf-onap.atlassian.net/browse/CPS-837>`_ null is now used to indicate if a property should be removed as part -of cm handle registration. +of CM Handle registration. The Absolute path to list with integer key will not work. Please refer `CPS-961 <https://lf-onap.atlassian.net/browse/CPS-961>`_ for more information. diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 531d353b0c..5a00e8dd6e 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -23,7 +23,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index b9b1c162e7..453dbca2bd 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -22,6 +22,8 @@ package org.onap.cps.integration.base import com.hazelcast.map.IMap +import java.time.OffsetDateTime +import java.util.concurrent.BlockingQueue import okhttp3.mockwebserver.MockWebServer import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService @@ -44,6 +46,7 @@ import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncService import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.onap.cps.ncmp.impl.utils.AlternateIdMatcher import org.onap.cps.ri.repository.DataspaceRepository +import org.onap.cps.ri.repository.SchemaSetRepository import org.onap.cps.ri.utils.SessionManager import org.onap.cps.utils.JsonObjectMapper import org.springframework.beans.factory.annotation.Autowired @@ -59,10 +62,6 @@ import org.springframework.test.web.servlet.MockMvc import org.testcontainers.spock.Testcontainers import spock.lang.Shared import spock.lang.Specification -import spock.util.concurrent.PollingConditions - -import java.time.OffsetDateTime -import java.util.concurrent.BlockingQueue @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, classes = [CpsDataspaceService]) @Testcontainers @@ -102,6 +101,12 @@ abstract class CpsIntegrationSpecBase extends Specification { SessionManager sessionManager @Autowired + DataspaceRepository dataspaceRepository + + @Autowired + SchemaSetRepository schemaSetRepository + + @Autowired ParameterizedCmHandleQueryService networkCmProxyCmHandleQueryService @Autowired @@ -153,16 +158,16 @@ abstract class CpsIntegrationSpecBase extends Specification { static NO_ALTERNATE_ID = '' static GENERAL_TEST_DATASPACE = 'generalTestDataspace' static BOOKSTORE_SCHEMA_SET = 'bookstoreSchemaSet' - static MODULE_SYNC_WAIT_TIME_IN_SECONDS = 2 static initialized = false def now = OffsetDateTime.now() + enum ModuleNameStrategy { UNIQUE, OVERLAPPING } + def setup() { if (!initialized) { cpsDataspaceService.createDataspace(GENERAL_TEST_DATASPACE) createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE) - cpsAnchorService.createAnchor(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET, 'owner-of-bookstore-schema-set-do-not-delete') initialized = true } mockDmiServer1.setDispatcher(dmiDispatcher1) @@ -182,7 +187,7 @@ abstract class CpsIntegrationSpecBase extends Specification { mockDmiServer1.shutdown() mockDmiServer2.shutdown() mockPolicyServer.shutdown() - cpsModuleService.deleteAllUnusedYangModuleData() + cpsModuleService.deleteAllUnusedYangModuleData('NFP-Operational') } def static readResourceDataFile(filename) { @@ -203,18 +208,18 @@ abstract class CpsIntegrationSpecBase extends Specification { return nodeCount } - def getBookstoreYangResourcesNameToContentMap() { + def getBookstoreyangResourceContentPerName() { def bookstoreModelFileContent = readResourceDataFile('bookstore/bookstore.yang') def bookstoreTypesFileContent = readResourceDataFile('bookstore/bookstore-types.yang') return [bookstore: bookstoreModelFileContent, bookstoreTypes: bookstoreTypesFileContent] } def createStandardBookStoreSchemaSet(targetDataspace) { - cpsModuleService.createSchemaSet(targetDataspace, BOOKSTORE_SCHEMA_SET, getBookstoreYangResourcesNameToContentMap()) + cpsModuleService.createSchemaSet(targetDataspace, BOOKSTORE_SCHEMA_SET, getBookstoreyangResourceContentPerName()) } def createStandardBookStoreSchemaSet(targetDataspace, targetSchemaSet) { - cpsModuleService.createSchemaSet(targetDataspace, targetSchemaSet, getBookstoreYangResourcesNameToContentMap()) + cpsModuleService.createSchemaSet(targetDataspace, targetSchemaSet, getBookstoreyangResourceContentPerName()) } def dataspaceExists(dataspaceName) { @@ -254,9 +259,7 @@ abstract class CpsIntegrationSpecBase extends Specification { def registerCmHandle(dmiPlugin, cmHandleId, moduleSetTag, alternateId) { registerCmHandleWithoutWaitForReady(dmiPlugin, cmHandleId, moduleSetTag, alternateId) moduleSyncWatchdog.moduleSyncAdvisedCmHandles() - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - CmHandleState.READY == networkCmProxyInventoryFacade.getCmHandleCompositeState(cmHandleId).cmHandleState - }) + CmHandleState.READY == networkCmProxyInventoryFacade.getCmHandleCompositeState(cmHandleId).cmHandleState } def registerCmHandleWithoutWaitForReady(dmiPlugin, cmHandleId, moduleSetTag, alternateId) { @@ -265,9 +268,14 @@ abstract class CpsIntegrationSpecBase extends Specification { } def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset) { + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy.UNIQUE) + } + + def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset, ModuleNameStrategy moduleNameStrategy ) { def cmHandles = [] def id = offset - def moduleReferences = (1..200).collect { "${moduleSetTag}Module${it}" } + def modulePrefix = moduleNameStrategy.OVERLAPPING.equals(moduleNameStrategy) ? 'same' : moduleSetTag + def moduleReferences = (1..200).collect { "${modulePrefix}Module${it}" } (1..numberOfCmHandles).each { def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: "ch-${id}", moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID) cmHandles.add(ncmpServiceCmHandle) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy index c790521627..11c0ff2743 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/DmiDispatcher.groovy @@ -29,6 +29,7 @@ import org.springframework.http.HttpStatus import org.springframework.http.MediaType import java.util.regex.Matcher +import java.util.stream.Collectors import static org.onap.cps.integration.base.CpsIntegrationSpecBase.readResourceDataFile @@ -81,7 +82,7 @@ class DmiDispatcher extends Dispatcher { // get module resources for a CM-handle case ~'^/dmi/v1/ch/(.*)/moduleResources$': def cmHandleId = Matcher.lastMatcher[0][1] - return getModuleResourcesResponse(cmHandleId) + return getModuleResourcesResponse(request, cmHandleId) // pass-through data operation for a CM-handle case ~'^/dmi/v1/ch/(.*)/data/ds/(.*)$': @@ -124,8 +125,13 @@ class DmiDispatcher extends Dispatcher { return mockResponseWithBody(HttpStatus.OK, moduleReferences) } - def getModuleResourcesResponse(cmHandleId) { - def moduleResources = '[' + getModuleNamesForCmHandle(cmHandleId).collect { + def getModuleResourcesResponse(request, cmHandleId) { + def moduleResourcesRequest = jsonSlurper.parseText(request.getBody().readUtf8()) + def requestedModuleNames = moduleResourcesRequest.get('data').get('modules').collect{it.get('name')} + def candidateModuleNames = getModuleNamesForCmHandle(cmHandleId) + def moduleNames = candidateModuleNames.stream().filter(candidate -> requestedModuleNames.contains(candidate)).toList() + + def moduleResources = '[' + moduleNames.collect { MODULE_RESOURCES_RESPONSE_TEMPLATE.replaceAll("<MODULE_NAME>", it) }.join(',') + ']' return mockResponseWithBody(HttpStatus.OK, moduleResources) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy index 2bd5a4a1be..ca321119ea 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/AnchorServiceIntegrationSpec.groovy @@ -59,17 +59,17 @@ class AnchorServiceIntegrationSpec extends FunctionalSpecBase { and: '1 anchor with "other" schema set is created' createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE, 'otherSchemaSet') objectUnderTest.createAnchor(GENERAL_TEST_DATASPACE, 'otherSchemaSet', 'anchor3') - then: 'there are 4 anchors in the general test database' - assert objectUnderTest.getAnchors(GENERAL_TEST_DATASPACE).size() == 4 - and: 'there are 3 anchors associated with bookstore schema set' - assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET).size() == 3 + then: 'there are 3 anchors in the general test database' + assert objectUnderTest.getAnchors(GENERAL_TEST_DATASPACE).size() == 3 + and: 'there are 2 anchors associated with bookstore schema set' + assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET).size() == 2 and: 'there is 1 anchor associated with other schema set' assert objectUnderTest.getAnchorsBySchemaSetName(GENERAL_TEST_DATASPACE, 'otherSchemaSet').size() == 1 } def 'Querying anchor(name)s (depends on previous test!).'() { - expect: 'there are now 4 anchors using the "stores" module (both schema sets use the same modules) ' - assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'bookstore-types']).size() == 4 + expect: 'there are now 3 anchors using the "stores" module (both schema sets use the same modules)' + assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'bookstore-types']).size() == 3 and: 'there are no anchors using both "stores" and a "unused-model"' assert objectUnderTest.queryAnchorNames(GENERAL_TEST_DATASPACE, ['stores', 'unused-model']).size() == 0 } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy index 178b0227ca..47a332adc9 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -32,8 +32,6 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsDataspaceService } - def cleanup() { cpsModuleService.deleteAllUnusedYangModuleData() } - def 'Dataspace CRUD operations.'() { when: 'a dataspace is created' objectUnderTest.createDataspace('newDataspace') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy index d8010875c1..49201e8f22 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy @@ -53,25 +53,23 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { } """ - def newYangResourcesNameToContentMap = [:] - def moduleReferences = [] + def yangResourceContentPerName = [:] + def allModuleReferences = [] def noNewModules = [:] def bookstoreModelFileContent = readResourceDataFile('bookstore/bookstore.yang') def bookstoreTypesFileContent = readResourceDataFile('bookstore/bookstore-types.yang') def setup() { objectUnderTest = cpsModuleService } - def cleanup() { objectUnderTest.deleteAllUnusedYangModuleData() } - /* C R E A T E S C H E M A S E T U S E - C A S E S */ def 'Create new schema set from yang resources with #scenario'() { given: 'a new schema set with #numberOfModules modules' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(numberOfNewModules) + populateYangResourceContentPerNameAndAllModuleReferences(numberOfNewModules) when: 'the new schema set is created' - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', yangResourceContentPerName) then: 'the number of module references has increased by #numberOfNewModules' def yangResourceModuleReferences = objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1) originalNumberOfModuleReferences + numberOfNewModules == yangResourceModuleReferences.size() @@ -95,39 +93,43 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Create new schema set from modules with #scenario'() { given: 'a new schema set with #numberOfNewModules modules' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(numberOfNewModules) + populateYangResourceContentPerNameAndAllModuleReferences(numberOfNewModules) and: 'add existing module references (optional)' - moduleReferences.addAll(existingModuleReferences) + allModuleReferences.addAll(existingModuleReferences) when: 'the new schema set is created' def schemaSetName = "NewSchemaWith${numberOfNewModules}Modules" - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, schemaSetName, newYangResourcesNameToContentMap, moduleReferences) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, schemaSetName, yangResourceContentPerName, allModuleReferences) and: 'associated with a new anchor' cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, schemaSetName, 'newAnchor') then: 'the new anchor has the correct number of modules' def yangResourceModuleReferences = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'newAnchor') assert expectedNumberOfModulesForAnchor == yangResourceModuleReferences.size() + and: 'the schema set has the correct number of modules too' + def dataspaceEntity = dataspaceRepository.getByName(FUNCTIONAL_TEST_DATASPACE_1) + def schemaSetEntity = schemaSetRepository.getByDataspaceAndName(dataspaceEntity, schemaSetName) + assert expectedNumberOfModulesForAnchor == schemaSetEntity.yangResources.size() cleanup: objectUnderTest.deleteSchemaSetsWithCascade(FUNCTIONAL_TEST_DATASPACE_1, [ schemaSetName.toString() ]) where: 'the following module references are provided' scenario | numberOfNewModules | existingModuleReferences || expectedNumberOfModulesForAnchor 'empty schema set' | 0 | [ ] || 0 - 'one existing module' | 0 | [bookStoreModuleReference ] || 1 + 'one existing module' | 0 | [ bookStoreModuleReference ] || 1 'two new modules' | 2 | [ ] || 2 - 'two new modules, one existing' | 2 | [bookStoreModuleReference ] || 3 + 'two new modules, one existing' | 2 | [ bookStoreModuleReference ] || 3 'over max batch size #modules' | 101 | [ ] || 101 'two valid, one invalid module' | 2 | [ new ModuleReference('NOT EXIST','IRRELEVANT') ] || 2 } def 'Duplicate schema content.'() { given: 'a map of yang resources' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) + populateYangResourceContentPerNameAndAllModuleReferences(1) when: 'a new schema set is created' - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', yangResourceContentPerName) then: 'the dataspace has one new module (reference)' def numberOfModuleReferencesAfterFirstSchemaSetHasBeenAdded = objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1).size() assert numberOfModuleReferencesAfterFirstSchemaSetHasBeenAdded == originalNumberOfModuleReferences + 1 when: 'a second new schema set is created' - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema2', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema2', yangResourceContentPerName) then: 'the dataspace has no additional module (reference)' assert numberOfModuleReferencesAfterFirstSchemaSetHasBeenAdded == objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1).size() cleanup: @@ -136,8 +138,8 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Attempt to create schema set, error scenario: #scenario.'() { when: 'attempt to store schema set #schemaSetName in dataspace #dataspaceName' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(0) - objectUnderTest.createSchemaSet(dataspaceName, schemaSetName, newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(0) + objectUnderTest.createSchemaSet(dataspaceName, schemaSetName, yangResourceContentPerName) then: 'an #expectedException is thrown' thrown(expectedException) where: 'the following data is used' @@ -148,7 +150,7 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Attempt to create duplicate schema set from modules.'() { when: 'attempt to store duplicate schema set from modules' - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_SCHEMA_SET, newYangResourcesNameToContentMap, []) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_SCHEMA_SET, yangResourceContentPerName, []) then: 'an Already Defined Exception is thrown' thrown(AlreadyDefinedException) } @@ -194,12 +196,12 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Identifying new module references with #scenario'() { when: 'identifyNewModuleReferences is called' - def result = objectUnderTest.identifyNewModuleReferences(moduleReferences) + def result = objectUnderTest.identifyNewModuleReferences(allModuleReferences) then: 'the correct module references are returned' assert result.size() == expectedResult.size() assert result.containsAll(expectedResult) where: 'the following data is used' - scenario | moduleReferences || expectedResult + scenario | allModuleReferences || expectedResult 'just new module references' | [new ModuleReference('new1', 'r1'), new ModuleReference('new2', 'r1')] || [new ModuleReference('new1', 'r1'), new ModuleReference('new2', 'r1')] 'one new module,one existing reference' | [new ModuleReference('new1', 'r1'), bookStoreModuleReference] || [new ModuleReference('new1', 'r1')] 'no new module references' | [bookStoreModuleReference] || [] @@ -218,8 +220,8 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Retrieve all schema sets.'() { given: 'an extra schema set is stored' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(1) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchema1', yangResourceContentPerName) when: 'all schema sets are retrieved' def result = objectUnderTest.getSchemaSets(FUNCTIONAL_TEST_DATASPACE_1) then: 'the result contains all expected schema sets' @@ -235,8 +237,8 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Delete schema sets with(out) cascade.'() { given: 'a schema set' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(1) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', yangResourceContentPerName) and: 'optionally create anchor for the schema set' if (associateWithAnchor) { cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet', 'newAnchor') @@ -263,11 +265,11 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def 'Delete schema sets with shared resources.'() { given: 'a new schema set' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(1) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet1', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(1) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet1', yangResourceContentPerName) and: 'another schema set which shares one yang resource (module)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences(2) - objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet2', newYangResourcesNameToContentMap) + populateYangResourceContentPerNameAndAllModuleReferences(2) + objectUnderTest.createSchemaSet(FUNCTIONAL_TEST_DATASPACE_1, 'newSchemaSet2', yangResourceContentPerName) when: 'all schema sets are retrieved' def moduleRevisions = objectUnderTest.getYangResourceModuleReferences(FUNCTIONAL_TEST_DATASPACE_1).revision then: 'both modules (revisions) are present' @@ -301,23 +303,23 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { U P G R A D E */ - def 'Upgrade schema set (with existing and new modules, no matching module set tag in NCMP)'() { + def 'Upgrade schema set [with existing and new modules, no matching module set tag in NCMP]'() { given: 'an anchor and schema set with 2 modules (to be upgraded)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('original', 2) - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', newYangResourcesNameToContentMap, []) + populateYangResourceContentPerNameAndAllModuleReferences('original', 2) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', yangResourceContentPerName, allModuleReferences) cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', 'targetAnchor') def yangResourceModuleReferencesBeforeUpgrade = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'targetAnchor') assert yangResourceModuleReferencesBeforeUpgrade.size() == 2 assert yangResourceModuleReferencesBeforeUpgrade.containsAll([new ModuleReference('original_0','2000-01-01'),new ModuleReference('original_1','2001-01-01')]) and: 'two new 2 modules (from node)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('new', 2) + populateYangResourceContentPerNameAndAllModuleReferences('new', 2) def newModuleReferences = [new ModuleReference('new_0','2000-01-01'),new ModuleReference('new_1','2001-01-01')] and: 'a list of all module references (normally retrieved from node)' - def allModuleReferences = [] - allModuleReferences.add(bookStoreModuleReference) - allModuleReferences.addAll(newModuleReferences) + def allOtherModuleReferences = [] + allOtherModuleReferences.add(bookStoreModuleReference) + allOtherModuleReferences.addAll(newModuleReferences) when: 'the schema set is upgraded' - objectUnderTest.upgradeSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', newYangResourcesNameToContentMap, allModuleReferences) + objectUnderTest.upgradeSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', yangResourceContentPerName, allOtherModuleReferences) then: 'the new anchor has the correct new and existing modules' def yangResourceModuleReferencesAfterUpgrade = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'targetAnchor') assert yangResourceModuleReferencesAfterUpgrade.size() == 3 @@ -327,18 +329,19 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { objectUnderTest.deleteSchemaSetsWithCascade(FUNCTIONAL_TEST_DATASPACE_1, ['targetSchema']) } - def 'Upgrade existing schema set from another anchor (used in NCMP for matching module set tag)'() { + def 'Upgrade existing schema set from another anchor [used in NCMP for matching module set tag]'() { given: 'an anchor and schema set with 1 module (target)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('target', 1) - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', newYangResourcesNameToContentMap, []) + populateYangResourceContentPerNameAndAllModuleReferences('target', 1) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', yangResourceContentPerName, allModuleReferences) cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', 'targetAnchor') def moduleReferencesBeforeUpgrade = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'targetAnchor') assert moduleReferencesBeforeUpgrade.size() == 1 and: 'another anchor and schema set with 2 other modules (source for upgrade)' - populateNewYangResourcesNameToContentMapAndAllModuleReferences('source', 2) - objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'sourceSchema', newYangResourcesNameToContentMap, []) + populateYangResourceContentPerNameAndAllModuleReferences('source', 2) + objectUnderTest.createSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'sourceSchema', yangResourceContentPerName, allModuleReferences) cpsAnchorService.createAnchor(FUNCTIONAL_TEST_DATASPACE_1, 'sourceSchema', 'sourceAnchor') - assert objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'sourceAnchor').size() == 2 + def yangResourcesModuleReferences = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'sourceAnchor') + assert yangResourcesModuleReferences.size() == 2 when: 'the target schema is upgraded using the module references from the source anchor' def moduleReferencesFromSourceAnchor = objectUnderTest.getYangResourcesModuleReferences(FUNCTIONAL_TEST_DATASPACE_1, 'sourceAnchor') objectUnderTest.upgradeSchemaSetFromModules(FUNCTIONAL_TEST_DATASPACE_1, 'targetSchema', noNewModules, moduleReferencesFromSourceAnchor) @@ -356,17 +359,19 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { H E L P E R M E T H O D S */ - def populateNewYangResourcesNameToContentMapAndAllModuleReferences(numberOfModules) { - populateNewYangResourcesNameToContentMapAndAllModuleReferences('name', numberOfModules) + def populateYangResourceContentPerNameAndAllModuleReferences(numberOfModules) { + populateYangResourceContentPerNameAndAllModuleReferences('name', numberOfModules) } - def populateNewYangResourcesNameToContentMapAndAllModuleReferences(namePrefix, numberOfModules) { + def populateYangResourceContentPerNameAndAllModuleReferences(namePrefix, numberOfModules) { + yangResourceContentPerName.clear() + allModuleReferences.clear() numberOfModules.times { def uniqueName = namePrefix + '_' + it def uniqueRevision = String.valueOf(2000 + it) + '-01-01' - moduleReferences.add(new ModuleReference(uniqueName, uniqueRevision)) + allModuleReferences.add(new ModuleReference(uniqueName, uniqueRevision)) def uniqueContent = NEW_RESOURCE_CONTENT.replace(NEW_RESOURCE_REVISION, uniqueRevision).replace('module test_module', 'module '+uniqueName) - newYangResourcesNameToContentMap.put(uniqueRevision, uniqueContent) + yangResourceContentPerName.put(uniqueName, uniqueContent) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy index e4d75aa378..42fb964d52 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 Nordix Foundation * Modifications Copyright (C) 2023-2025 TechMahindra Ltd * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -27,7 +27,6 @@ import org.onap.cps.integration.base.FunctionalSpecBase import org.onap.cps.api.parameters.FetchDescendantsOption import org.onap.cps.api.parameters.PaginationOption import org.onap.cps.api.exceptions.CpsPathException -import spock.lang.Ignore import static org.onap.cps.api.parameters.FetchDescendantsOption.DIRECT_CHILDREN_ONLY import static org.onap.cps.api.parameters.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS @@ -57,7 +56,6 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { 'the AND is used where result does not exist' | '//books[@lang="English" and @price=1000]' || 0 | [] } - @Ignore // TODO will be implemented in CPS-2416 def 'Query data leaf using CPS path for #scenario.'() { when: 'query data leaf for bookstore container' def result = objectUnderTest.queryDataLeaf(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_ANCHOR_1, cpsPath, Object.class) @@ -70,7 +68,6 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { 'non-existing path' | '/non-existing/@title' || 0 } - @Ignore def 'Query data leaf with type #leafType using CPS path.'() { given: 'a cps path query for two books, returning only #leafName' def cpsPath = '//books[@title="Matilda" or @title="Good Omens"]/@' + leafName @@ -85,7 +82,6 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { 'editions' | List.class || [[1988, 2000], [2006]] } - @Ignore def 'Query data leaf using CPS path with ancestor axis.'() { given: 'a cps path query that will return the names of the categories of two books' def cpsPath = '//books[@title="Matilda" or @title="Good Omens"]/ancestor::categories/@name' @@ -461,4 +457,17 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { and: 'the queried nodes have expected bookstore names' assert result.anchorName.toSet() == [BOOKSTORE_ANCHOR_1, BOOKSTORE_ANCHOR_2].toSet() } + + def 'Query with a limit of #limit.' () { + when: + def result = objectUnderTest.queryDataNodes(FUNCTIONAL_TEST_DATASPACE_1, BOOKSTORE_ANCHOR_1, '/bookstore/categories', OMIT_DESCENDANTS, limit) + then: 'the expected number of nodes is returned' + assert countDataNodesInTree(result) == expectedNumberOfResults + where: 'the following parameters are used' + limit || expectedNumberOfResults + 1 || 1 + 2 || 2 + 0 || 5 + -1 || 5 + } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy index 6f063fb222..41798cb1f1 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleCreateSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package org.onap.cps.integration.functional.ncmp +import java.time.Duration import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.common.serialization.StringDeserializer import org.onap.cps.integration.KafkaTestContainer @@ -32,9 +33,6 @@ import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle import org.onap.cps.ncmp.events.lcm.v1.LcmEvent import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.api.inventory.models.LockReasonCategory -import spock.util.concurrent.PollingConditions - -import java.time.Duration class CmHandleCreateSpec extends CpsIntegrationSpecBase { @@ -73,9 +71,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handle goes to READY state after module sync' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(uniqueId).cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(uniqueId).cmHandleState and: 'the CM-handle has expected modules' assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(uniqueId).moduleName.sort() @@ -113,11 +109,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handle goes to LOCKED state with reason MODULE_SYNC_FAILED' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState('ch-1') - assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED - assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_SYNC_FAILED - }) + def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState('ch-1') + assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED + assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_SYNC_FAILED and: 'CM-handle has no modules' assert objectUnderTest.getYangResourcesModuleReferences('ch-1').empty @@ -141,9 +135,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'the CM-handle goes to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState('ch-3').cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState('ch-3').cmHandleState and: 'the CM-handle has expected moduleSetTag' assert objectUnderTest.getNcmpServiceCmHandle('ch-3').moduleSetTag == 'B' @@ -178,9 +170,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { assert dmiPluginRegistrationResponse.createdCmHandles.sort { it.cmHandle } == [ CmHandleRegistrationResponse.createSuccessResponse('ch-3'), CmHandleRegistrationResponse.createSuccessResponse('ch-4'), - CmHandleRegistrationResponse.createFailureResponse('ch-5', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED), + CmHandleRegistrationResponse.createFailureResponse('ch-5', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST), CmHandleRegistrationResponse.createSuccessResponse('ch-6'), - CmHandleRegistrationResponse.createFailureResponse('ch-7', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED), + CmHandleRegistrationResponse.createFailureResponse('ch-7', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST), ] cleanup: 'deregister CM handles' @@ -200,9 +192,7 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handles go to LOCKED state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED - }) + assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED when: 'DMI is available for retry' dmiDispatcher1.moduleNamesPerCmHandleId = ['ch-1': ['M1', 'M2'], 'ch-2': ['M1', 'M2']] @@ -212,11 +202,9 @@ class CmHandleCreateSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } then: 'Both CM-handles go to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - ['ch-1', 'ch-2'].each { cmHandleId -> - assert objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState == CmHandleState.READY - } - }) + ['ch-1', 'ch-2'].each { cmHandleId -> + assert objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState == CmHandleState.READY + } and: 'Both CM-handles have expected modules' ['ch-1', 'ch-2'].each { cmHandleId -> diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy index f2593ce587..22bbaa81df 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpdateSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -76,8 +76,8 @@ class CmHandleUpdateSpec extends CpsIntegrationSpecBase { def dmiPluginRegistrationResponse = objectUnderTest.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: DMI1_URL, updatedCmHandles: [cmHandleToUpdate])) - then: 'registration gives failure response, due to alternate ID being already associated' - assert dmiPluginRegistrationResponse.updatedCmHandles == [CmHandleRegistrationResponse.createFailureResponse('ch-1', NcmpResponseStatus.ALTERNATE_ID_ALREADY_ASSOCIATED)] + then: 'registration gives failure response, due to cm-handle already existing' + assert dmiPluginRegistrationResponse.updatedCmHandles == [CmHandleRegistrationResponse.createFailureResponse('ch-1', NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST)] and: 'the CM-handle still has the old alternate ID' assert objectUnderTest.getNcmpServiceCmHandle('ch-1').alternateId == 'original' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy index 28714fd123..43540a9675 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy @@ -27,7 +27,6 @@ import org.onap.cps.ncmp.api.inventory.models.DmiPluginRegistration import org.onap.cps.ncmp.api.inventory.models.LockReasonCategory import org.onap.cps.ncmp.api.inventory.models.UpgradedCmHandles import org.onap.cps.ncmp.impl.NetworkCmProxyInventoryFacadeImpl -import spock.util.concurrent.PollingConditions class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { @@ -67,9 +66,7 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } then: 'CM-handle goes to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState and: 'the CM-handle has expected moduleSetTag' assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag @@ -79,7 +76,6 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { cleanup: 'deregister CM-handle and remove all associated module resources' deregisterCmHandle(DMI1_URL, cmHandleId) - cpsModuleService.deleteAllUnusedYangModuleData() where: 'following module set tags are used' initialModuleSetTag | updatedModuleSetTag @@ -112,9 +108,7 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } and: 'CM-handle goes to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState - }) + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState and: 'the CM-handle has expected moduleSetTag' assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag @@ -171,11 +165,9 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } then: 'CM-handle goes to LOCKED state with reason MODULE_UPGRADE_FAILED' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId) - assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED - assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED - }) + def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId) + assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED + assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED and: 'the CM-handle has same moduleSetTag as before' assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == 'oldTag' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy index 43db9b208e..8bbe0ca194 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy @@ -20,17 +20,18 @@ package org.onap.cps.integration.functional.ncmp +import com.hazelcast.map.IMap import io.micrometer.core.instrument.MeterRegistry -import spock.lang.Ignore - -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.springframework.beans.factory.annotation.Autowired import org.springframework.util.StopWatch +import spock.lang.Ignore import spock.util.concurrent.PollingConditions +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { ModuleSyncWatchdog objectUnderTest @@ -38,11 +39,15 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { @Autowired MeterRegistry meterRegistry + @Autowired + IMap<String, Integer> cmHandlesByState + def executorService = Executors.newFixedThreadPool(2) def PARALLEL_SYNC_SAMPLE_SIZE = 100 def setup() { objectUnderTest = moduleSyncWatchdog + clearCmHandleStateGauge() } def cleanup() { @@ -64,18 +69,16 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } - @Ignore - /** this test has intermittent failures, due to timeouts. + /** this test has intermittent failures, due to race conditions * Ignored but left here as it might be valuable to further optimization investigations. **/ - + @Ignore def 'CPS-2478 Highlight (and improve) module sync inefficiencies.'() { given: 'register 250 cm handles with module set tag cps-2478-A' def numberOfTags = 2 def cmHandlesPerTag = 250 def totalCmHandles = numberOfTags * cmHandlesPerTag def offset = 1 - def minimumBatches = totalCmHandles / 100 registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset) and: 'register anther 250 cm handles with module set tag cps-2478-B' offset += cmHandlesPerTag @@ -85,23 +88,21 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { when: 'sync all advised cm handles' objectUnderTest.moduleSyncAdvisedCmHandles() Thread.sleep(100) - then: 'retry until both schema sets are stored in db (1 schema set for each module set tag)' - def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer() - new PollingConditions().within(10, () -> { - objectUnderTest.moduleSyncAdvisedCmHandles() - Thread.sleep(100) - assert dbSchemaSetStorageTimer.count() == 2 - }) - then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)' - def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() + then: 'Keep processing until there are no more LOCKED or ADVISED cm handles' new PollingConditions().within(10, () -> { - assert dbStateUpdateTimer.count() >= minimumBatches + def advised = cmHandlesByState.get('advisedCmHandlesCount') + def locked = cmHandlesByState.get('lockedCmHandlesCount') + if ( locked > 0 | advised > 0 ) { + println "CPS-2576 Need to retry ${locked} LOCKED / ${advised} ADVISED cm Handles" + objectUnderTest.moduleSyncAdvisedCmHandles() + Thread.sleep(100) + } + assert cmHandlesByState.get('lockedCmHandlesCount') + cmHandlesByState.get('advisedCmHandlesCount') == 0 }) - and: 'one call to DMI per module set tag to get module references (may be more due to parallel processing of batches)' - def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer() - assert dmiModuleRetrievalTimer.count() >= numberOfTags && dmiModuleRetrievalTimer.count() <= minimumBatches - and: 'log the relevant instrumentation' + def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer() + def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.createFromNewAndExistingModules').timer() + def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() logInstrumentation(dmiModuleRetrievalTimer, 'get modules from DMI ') logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets ') logInstrumentation(dbStateUpdateTimer, 'batch state updates ') @@ -110,7 +111,6 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { def stopWatch = new StopWatch() stopWatch.start() deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1) - cpsModuleService.deleteAllUnusedYangModuleData() stopWatch.stop() println "*** CPS-2478, Deletion of $totalCmHandles cm handles took ${stopWatch.getTotalTimeMillis()} milliseconds" } @@ -131,6 +131,28 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) } + /** this test has intermittent failures, due to race conditions + * Ignored but left here as it might be valuable to further optimization investigations. + **/ + @Ignore + def 'Schema sets with overlapping modules processed at the same time (DB constraint violation).'() { + given: 'register one batch (100) cm handles of tag A (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagA', 100, 1, ModuleNameStrategy.OVERLAPPING) + and: 'register another batch cm handles of tag B (with overlapping module names)' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'tagB', 100, 101, ModuleNameStrategy.OVERLAPPING) + and: 'populate the work queue with both batches' + objectUnderTest.populateWorkQueueIfNeeded() + when: 'advised cm handles are processed on 2 threads (exactly one batch for each)' + objectUnderTest.moduleSyncAdvisedCmHandles() + executorService.execute(moduleSyncAdvisedCmHandles) + then: 'all cm handles have been processed' + assert getNumberOfProcessedCmHandles() == 200 + then: 'at least 1 cm handle is in state LOCKED' + assert cmHandlesByState.get('lockedCmHandlesCount') >= 1 + cleanup: 'remove all test cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, 200, 1) + } + def 'Populate module sync work queue on two parallel threads with a slight difference in start time.'() { // This test proved that the issue in CPS-2403 did not arise if the the queue was populated and given time to be distributed given: 'the queue is empty at the start' @@ -169,4 +191,21 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { } } + def moduleSyncAdvisedCmHandles = () -> { + try { + objectUnderTest.moduleSyncAdvisedCmHandles() + } catch (InterruptedException e) { + e.printStackTrace() + } + } + + def clearCmHandleStateGauge() { + cmHandlesByState.keySet().each { cmHandlesByState.put(it, 0)} + } + + def getNumberOfProcessedCmHandles() { + return cmHandlesByState.get('readyCmHandlesCount') + cmHandlesByState.get('lockedCmHandlesCount') + } + + } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy index 7ce3cf5e17..77349fe0a5 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/RestApiSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.springframework.http.MediaType -import spock.util.concurrent.PollingConditions class RestApiSpec extends CpsIntegrationSpecBase { @@ -47,13 +46,11 @@ class RestApiSpec extends CpsIntegrationSpecBase { and: 'the module sync watchdog is triggered' moduleSyncWatchdog.moduleSyncAdvisedCmHandles() then: 'CM-handles go to READY state' - new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - (1..3).each { - mvc.perform(get('/ncmp/v1/ch/ch-'+it)) - .andExpect(status().isOk()) - .andExpect(jsonPath('$.state.cmHandleState').value('READY')) - } - }) + (1..3).each { + mvc.perform(get('/ncmp/v1/ch/ch-'+it)) + .andExpect(status().isOk()) + .andExpect(jsonPath('$.state.cmHandleState').value('READY')) + } } def 'Search for CM Handles by module using REST API.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy index a37bb6ad4d..d8553419ce 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy @@ -47,15 +47,15 @@ class CpsModuleServicePerfTest extends CpsPerfTestBase { def 'Store new schema set with many modules'() { when: 'a new schema set with 200 modules is stored' - def newYangResourcesNameToContentMap = [:] + def newYangResourceContentPerName = [:] (1..200).each { def year = 2000 + it def resourceName = "module${it}".toString() def moduleName = "stores${it}" def content = NEW_RESOURCE_CONTENT.replace('2020',String.valueOf(year)).replace('stores',moduleName) - newYangResourcesNameToContentMap.put(resourceName, content) + newYangResourceContentPerName.put(resourceName, content) } - objectUnderTest.createSchemaSet(CPS_PERFORMANCE_TEST_DATASPACE, 'perfSchemaSet', newYangResourcesNameToContentMap) + objectUnderTest.createSchemaSet(CPS_PERFORMANCE_TEST_DATASPACE, 'perfSchemaSet', newYangResourceContentPerName) then: 'the schema set is persisted correctly' def result = cpsModuleService.getSchemaSet(CPS_PERFORMANCE_TEST_DATASPACE, 'perfSchemaSet') result.moduleReferences.size() == 200 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy index 613f760b0c..e52d3f819c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -100,7 +100,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, (i..i+100).collect {SCHEMA_SET_PREFIX + it}) } cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, [SCHEMA_SET_PREFIX + '0']) - cpsModuleService.deleteAllUnusedYangModuleData() + cpsModuleService.deleteAllUnusedYangModuleData(CPS_PERFORMANCE_TEST_DATASPACE) } // This makes a Yang module of approximately target length in bytes by padding the description field with many '*' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index 364127f388..acc95cab8d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -103,5 +103,4 @@ class QueryPerfTest extends CpsPerfTestBase { 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.11 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.34 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } - } diff --git a/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java b/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java index ff4aec4175..60c1637c5a 100644 --- a/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java +++ b/integration-test/src/test/java/org/onap/cps/integration/KafkaTestContainer.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation. + * Copyright (C) 2024-2025 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.StringDeserializer; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.utility.DockerImageName; +import org.testcontainers.kafka.ConfluentKafkaContainer; /** * The Apache Kafka test container wrapper. @@ -35,14 +34,14 @@ import org.testcontainers.utility.DockerImageName; * Avoid unnecessary resource and time consumption. */ @Slf4j -public class KafkaTestContainer extends KafkaContainer { +public class KafkaTestContainer extends ConfluentKafkaContainer { - private static final String IMAGE_NAME_AND_VERSION = "registry.nordix.org/onaptest/confluentinc/cp-kafka:6.2.1"; + private static final String IMAGE_NAME_AND_VERSION = "confluentinc/cp-kafka:7.8.0"; private static volatile KafkaTestContainer kafkaTestContainer; private KafkaTestContainer() { - super(DockerImageName.parse(IMAGE_NAME_AND_VERSION).asCompatibleSubstituteFor("confluentinc/cp-kafka")); + super(IMAGE_NAME_AND_VERSION); } /** diff --git a/integration-test/src/test/resources/application.yml b/integration-test/src/test/resources/application.yml index 30598dfb90..e213a70a59 100644 --- a/integration-test/src/test/resources/application.yml +++ b/integration-test/src/test/resources/application.yml @@ -102,6 +102,7 @@ app: cm-subscription-dmi-out: ${CM_SUBSCRIPTION_DMI_OUT_TOPIC:dmi-ncmp-cm-avc-subscription} cm-subscription-ncmp-out: ${CM_SUBSCRIPTION_NCMP_OUT_TOPIC:subscription-response} cm-events-topic: ${NCMP_CM_EVENTS_TOPIC:cm-events} + inventory-events-topic: ncmp-inventory-events lcm: events: topic: ${LCM_EVENTS_TOPIC:ncmp-events} @@ -189,10 +190,6 @@ ncmp: trust-level: dmi-availability-watchdog-ms: 30000 - modules-sync-watchdog: - async-executor: - parallelism-level: 2 - model-loader: maximum-attempt-count: 20 diff --git a/jacoco-report/pom.xml b/jacoco-report/pom.xml index be1d42788d..3cd2e5d62e 100644 --- a/jacoco-report/pom.xml +++ b/jacoco-report/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/k6-tests/ncmp/common/passthrough-crud.js b/k6-tests/ncmp/common/passthrough-crud.js index 251ddf8890..a3d48fd590 100644 --- a/k6-tests/ncmp/common/passthrough-crud.js +++ b/k6-tests/ncmp/common/passthrough-crud.js @@ -29,7 +29,7 @@ import { export function passthroughRead(useAlternateId) { const cmHandleReference = getRandomCmHandleReference(useAlternateId); - const resourceIdentifier = 'NRCellDU/attributes/cellLocalId'; + const resourceIdentifier = 'ManagedElement=NRNode1/GNBDUFunction=1'; const datastoreName = 'ncmp-datastore:passthrough-operational'; const includeDescendants = true; const url = generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants); @@ -38,7 +38,7 @@ export function passthroughRead(useAlternateId) { export function passthroughWrite(useAlternateId) { const cmHandleReference = getRandomCmHandleReference(useAlternateId); - const resourceIdentifier = 'NRCellDU/attributes/cellLocalId'; + const resourceIdentifier = 'ManagedElement=NRNode1/GNBDUFunction=1'; const datastoreName = 'ncmp-datastore:passthrough-running'; const includeDescendants = false; const url = generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants); @@ -54,7 +54,7 @@ export function legacyBatchRead(cmHandleIds) { const payload = JSON.stringify({ "operations": [ { - "resourceIdentifier": "NRCellDU/attributes/cellLocalId", + "resourceIdentifier": "ManagedElement=NRNode1/GNBDUFunction=1", "targetIds": cmHandleIds, "datastore": "ncmp-datastore:passthrough-operational", "options": "(fields=NRCellDU/attributes/cellLocalId)", diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index 66d2dfe448..6bedb1f949 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation + * Copyright (C) 2024-2025 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl; export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl; export const CONTAINER_UP_TIME_IN_SECONDS = testConfig.hosts.containerUpTimeInSeconds; export const LEGACY_BATCH_TOPIC_NAME = 'legacy_batch_topic'; -export const TOTAL_CM_HANDLES = 20000; +export const TOTAL_CM_HANDLES = 50000; export const REGISTRATION_BATCH_SIZE = 100; export const READ_DATA_FOR_CM_HANDLE_DELAY_MS = 300; // must have same value as in docker-compose.yml export const WRITE_DATA_FOR_CM_HANDLE_DELAY_MS = 670; // must have same value as in docker-compose.yml diff --git a/policy-executor-stub/pom.xml b/policy-executor-stub/pom.xml index bdd21e191f..20de885af8 100644 --- a/policy-executor-stub/pom.xml +++ b/policy-executor-stub/pom.xml @@ -6,7 +6,7 @@ <parent> <groupId>org.onap.cps</groupId> <artifactId>cps-parent</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <relativePath>../cps-parent/pom.xml</relativePath> </parent> @@ -27,7 +27,7 @@ <modelVersion>4.0.0</modelVersion>
<groupId>org.onap.cps</groupId>
<artifactId>cps-aggregator</artifactId>
- <version>3.6.0-SNAPSHOT</version>
+ <version>3.6.1-SNAPSHOT</version>
<packaging>pom</packaging>
<name>cps</name>
diff --git a/releases/3.6.0-container.yaml b/releases/3.6.0-container.yaml new file mode 100644 index 0000000000..f97a5184fc --- /dev/null +++ b/releases/3.6.0-container.yaml @@ -0,0 +1,8 @@ +distribution_type: container +container_release_tag: 3.6.0 +project: cps +log_dir: cps-maven-docker-stage-master/949/ +ref: 851ad00169168fab91d81f1fd23d7f84f7a1005d +containers: + - name: 'cps-and-ncmp' + version: '3.6.0-20250129T162415Z' diff --git a/releases/3.6.0.yaml b/releases/3.6.0.yaml new file mode 100644 index 0000000000..7476bd47a4 --- /dev/null +++ b/releases/3.6.0.yaml @@ -0,0 +1,4 @@ +distribution_type: maven +log_dir: cps-maven-stage-master/957/ +project: cps +version: 3.6.0 diff --git a/spotbugs/pom.xml b/spotbugs/pom.xml index 8c685e95bc..a6e47860c9 100644 --- a/spotbugs/pom.xml +++ b/spotbugs/pom.xml @@ -25,7 +25,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.onap.cps</groupId> <artifactId>spotbugs</artifactId> - <version>3.6.0-SNAPSHOT</version> + <version>3.6.1-SNAPSHOT</version> <properties> <onap.nexus.url>https://nexus.onap.org</onap.nexus.url> diff --git a/version.properties b/version.properties index 81372a7e58..6cf6639717 100644 --- a/version.properties +++ b/version.properties @@ -22,7 +22,7 @@ major=3 minor=6 -patch=0 +patch=1 base_version=${major}.${minor}.${patch} |