summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-x.gitignore4
-rwxr-xr-xcps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java27
-rw-r--r--cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy16
-rw-r--r--cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java8
-rw-r--r--cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java26
-rw-r--r--cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy31
-rw-r--r--cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy45
-rw-r--r--cps-rest/src/test/resources/yang-files-set-total-1083-bytes.zipbin0 -> 858 bytes
-rwxr-xr-xcsit/plans/cps/setup.sh6
-rw-r--r--docker-compose/docker-compose.yml49
-rw-r--r--docker-compose/prometheus.yml10
-rwxr-xr-xgenerate-metrics-report.sh109
12 files changed, 263 insertions, 68 deletions
diff --git a/.gitignore b/.gitignore
index a377967df0..4f80c2b8a3 100755
--- a/.gitignore
+++ b/.gitignore
@@ -31,4 +31,6 @@ tmp/
/__pycache__/*
/docs/docs/
-/docs/.vscode/ \ No newline at end of file
+/docs/.vscode/
+
+/metrics-reports/
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
index 59b960af65..e71b72ab86 100755
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
@@ -327,36 +327,33 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService
final List<String> tobeRemovedCmHandles) {
final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses =
new ArrayList<>(tobeRemovedCmHandles.size());
- final Map<String, YangModelCmHandle> cmHandleIdToYangModelCmHandleMap = tobeRemovedCmHandles.stream()
- .collect(Collectors.toMap(cmHandleId -> cmHandleId, inventoryPersistence::getYangModelCmHandle));
+ final Collection<YangModelCmHandle> yangModelCmHandles =
+ inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles);
- final Collection<YangModelCmHandle> yangModelCmHandles = cmHandleIdToYangModelCmHandleMap.values();
updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
+ final Set<String> notDeletedCmHandles = new HashSet<>();
for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
try {
batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
tobeRemovedCmHandleBatch.forEach(cmHandleId ->
cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
- } catch (final Exception batchException) {
+ } catch (final RuntimeException batchException) {
log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
for (final String cmHandleId : tobeRemovedCmHandleBatch) {
final CmHandleRegistrationResponse cmHandleRegistrationResponse =
deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
+ if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) {
+ notDeletedCmHandles.add(cmHandleId);
+ }
}
}
}
- final Collection<YangModelCmHandle> deletedYangModelCmHandles =
- cmHandleRegistrationResponses.stream()
- .filter(cmHandleRegistrationResponse ->
- cmHandleRegistrationResponse.getStatus().equals(CmHandleRegistrationResponse.Status.SUCCESS))
- .map(CmHandleRegistrationResponse::getCmHandle)
- .map(cmHandleIdToYangModelCmHandleMap::get)
- .collect(Collectors.toList());
- updateCmHandleStateBatch(deletedYangModelCmHandles, CmHandleState.DELETED);
+ yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId()));
+ updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED);
return cmHandleRegistrationResponses;
}
@@ -383,9 +380,9 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService
private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles,
final CmHandleState cmHandleState) {
- final Map<YangModelCmHandle, CmHandleState> cmHandleIdsToBeRemoved = new HashMap<>();
- yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleIdsToBeRemoved.put(yangModelCmHandle, cmHandleState));
- lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleIdsToBeRemoved);
+ final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size());
+ yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState));
+ lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
}
private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy
index e8c53738ba..cf3454991a 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy
@@ -74,8 +74,8 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification {
dmiRegistration.setCreatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-1', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])])
dmiRegistration.setUpdatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-2', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])])
dmiRegistration.setRemovedCmHandles(['cmhandle-2'])
- and: 'any cm handle is persisted'
- mockInventoryPersistence.getYangModelCmHandle(_) >> new YangModelCmHandle()
+ and: 'cm handles are persisted'
+ mockInventoryPersistence.getYangModelCmHandles(['cmhandle-2']) >> [new YangModelCmHandle()]
when: 'registration is processed'
objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration)
then: 'cm-handles are removed first'
@@ -245,7 +245,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification {
def 'Remove CmHandle Successfully: #scenario'() {
given: 'a registration'
- mockInventoryPersistence.getYangModelCmHandle(_) >> new YangModelCmHandle()
+ addPersistedYangModelCmHandles(['cmhandle'])
def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
removedCmHandles: ['cmhandle'])
and: '#scenario'
@@ -319,7 +319,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification {
def 'Remove CmHandle Error Handling: Schema Set Deletion failed'() {
given: 'a registration'
- mockInventoryPersistence.getYangModelCmHandle('cmhandle') >> new YangModelCmHandle()
+ addPersistedYangModelCmHandles(['cmhandle'])
def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
removedCmHandles: ['cmhandle'])
and: 'schema set deletion failed with unknown error'
@@ -344,7 +344,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification {
def 'Remove CmHandle Error Handling: #scenario'() {
given: 'a registration'
- mockInventoryPersistence.getYangModelCmHandle('cmhandle') >> new YangModelCmHandle()
+ addPersistedYangModelCmHandles(['cmhandle'])
def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
removedCmHandles: ['cmhandle'])
and: 'cm-handle deletion fails on batch'
@@ -378,9 +378,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification {
}
def addPersistedYangModelCmHandles(ids) {
- ids.each {
- def yangModelCmHandle = new YangModelCmHandle(id:it)
- mockInventoryPersistence.getYangModelCmHandle(it) >> yangModelCmHandle
- }
+ def yangModelCmHandles = ids.collect { new YangModelCmHandle(id:it) }
+ mockInventoryPersistence.getYangModelCmHandles(ids) >> yangModelCmHandles
}
}
diff --git a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java
index 534077caa6..3e01f6ee81 100644
--- a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java
+++ b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java
@@ -2,6 +2,7 @@
* ============LICENSE_START=======================================================
* Copyright (C) 2020 Pantheon.tech
* Modifications Copyright (C) 2021 Bell Canada.
+ * Modifications Copyright (C) 2023 Nordix Foundation.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -127,16 +128,17 @@ public class MultipartFileUtil {
}
private static String extractYangResourceContent(final ZipInputStream zipInputStream,
- final ZipFileSizeValidator zipFileSizeValidator) throws IOException {
+ final ZipFileSizeValidator zipFileSizeValidator)
+ throws IOException {
try (final var byteArrayOutputStream = new ByteArrayOutputStream()) {
var totalSizeEntry = 0;
int numberOfBytesRead;
final var buffer = new byte[READ_BUFFER_SIZE];
- zipFileSizeValidator.incrementTotalEntryInArchive();
+ zipFileSizeValidator.incrementTotalYangFileEntryCountInArchive();
while ((numberOfBytesRead = zipInputStream.read(buffer, 0, READ_BUFFER_SIZE)) > 0) {
byteArrayOutputStream.write(buffer, 0, numberOfBytesRead);
totalSizeEntry += numberOfBytesRead;
- zipFileSizeValidator.updateTotalSizeArchive(numberOfBytesRead);
+ zipFileSizeValidator.updateTotalUncompressedSizeOfYangFilesInArchive(numberOfBytesRead);
zipFileSizeValidator.validateCompresssionRatio(totalSizeEntry);
}
return byteArrayOutputStream.toString(StandardCharsets.UTF_8);
diff --git a/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java b/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java
index d148fb70d4..2e303d1d45 100644
--- a/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java
+++ b/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java
@@ -1,6 +1,7 @@
/*
* ============LICENSE_START=======================================================
* Copyright (C) 2021 Bell Canada.
+ * Modifications Copyright (C) 2023 Nordix Foundation.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,31 +29,33 @@ import org.onap.cps.spi.exceptions.ModelValidationException;
public class ZipFileSizeValidator {
private static final int THRESHOLD_ENTRIES = 10000;
- private static final int THRESHOLD_SIZE = 100000000;
+ private static int THRESHOLD_SIZE = 100000000;
private static final double THRESHOLD_RATIO = 40;
private static final String INVALID_ZIP = "Invalid ZIP archive content.";
- private int totalSizeArchive = 0;
- private int totalEntryInArchive = 0;
+ private int totalUncompressedSizeOfYangFilesInArchive = 0;
+ private int totalYangFileEntriesInArchive = 0;
private long compressedSize = 0;
/**
* Increment the totalEntryInArchive by 1.
*/
- public void incrementTotalEntryInArchive() {
- totalEntryInArchive++;
+ public void incrementTotalYangFileEntryCountInArchive() {
+ totalYangFileEntriesInArchive++;
}
/**
* Update the totalSizeArchive by numberOfBytesRead.
+ *
* @param numberOfBytesRead the number of bytes of each entry
*/
- public void updateTotalSizeArchive(final int numberOfBytesRead) {
- totalSizeArchive += numberOfBytesRead;
+ public void updateTotalUncompressedSizeOfYangFilesInArchive(final int numberOfBytesRead) {
+ totalUncompressedSizeOfYangFilesInArchive += numberOfBytesRead;
}
/**
* Validate the total Compression size of the zip.
+ *
* @param totalEntrySize the size of the unzipped entry.
*/
public void validateCompresssionRatio(final int totalEntrySize) {
@@ -68,13 +71,14 @@ public class ZipFileSizeValidator {
* Validate the total Size and number of entries in the zip.
*/
public void validateSizeAndEntries() {
- if (totalSizeArchive > THRESHOLD_SIZE) {
+ if (totalUncompressedSizeOfYangFilesInArchive > THRESHOLD_SIZE) {
throw new ModelValidationException(INVALID_ZIP,
- String.format("The uncompressed data size exceeds the CPS limit %s bytes.", THRESHOLD_SIZE));
+ String.format("The total size of uncompressed yang files exceeds the CPS limit of %s bytes.",
+ THRESHOLD_SIZE));
}
- if (totalEntryInArchive > THRESHOLD_ENTRIES) {
+ if (totalYangFileEntriesInArchive > THRESHOLD_ENTRIES) {
throw new ModelValidationException(INVALID_ZIP,
- String.format("The number of entries in the archive exceeds the CPS limit %s.",
+ String.format("The number of yang file entries in the archive exceeds the CPS limit %s.",
THRESHOLD_ENTRIES));
}
}
diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy
index 3f4729ec79..67ee50e893 100644
--- a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy
+++ b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy
@@ -1,6 +1,7 @@
/*
* ============LICENSE_START=======================================================
* Copyright (C) 2020 Pantheon.tech
+ * Modifications Copyright (C) 2023 Nordix Foundation.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -51,7 +52,7 @@ class MultipartFileUtilSpec extends Specification {
def 'Extract yang resources from zip archive.'() {
given: 'uploaded zip archive containing 2 yang files and 1 not yang (json) file'
def multipartFile = new MockMultipartFile("file", "TEST.ZIP", "application/zip",
- getClass().getResource("/yang-files-set.zip").getBytes())
+ getClass().getResource("/yang-files-set.zip").getBytes())
when: 'resources are extracted from zip file'
def result = MultipartFileUtil.extractYangResourcesMap(multipartFile)
then: 'information from yang files is extracted, not yang file (json) is ignored'
@@ -60,6 +61,32 @@ class MultipartFileUtilSpec extends Specification {
assert result["component.yang"] == "fake component content 1\n"
}
+ def 'Yang file limits in zip archive: #scenario for the bug reported in CPS-1477'() {
+ given: 'a yang file size (uncompressed) limit of #threshold bytes'
+ ZipFileSizeValidator.THRESHOLD_SIZE = threshold
+ and: 'an archive with a yang file of 1083 bytes'
+ def multipartFile = multipartZipFileFromResource('/yang-files-set-total-1083-bytes.zip')
+ when: 'attempt to extract yang files'
+ def thrownException = null
+ try {
+ MultipartFileUtil.extractYangResourcesMap(multipartFile)
+ } catch (Exception e) {
+ thrownException = e
+ }
+ then: 'ModelValidationException indicating size limit is only thrown when threshold exceeded'
+ if (thresholdExceeded) {
+ assert thrownException instanceof ModelValidationException
+ assert thrownException.details.contains('limit of ' + threshold + ' bytes')
+ } else {
+ assert thrownException == null
+ }
+ where:
+ scenario | threshold || thresholdExceeded
+ 'exceed limit' | 1082 || true
+ 'equals to limit' | 1083 || false
+ 'within limit' | 1084 || false
+ }
+
def 'Extract resources from zip archive having #caseDescriptor.'() {
when: 'attempt to extract resources from zip file is performed'
MultipartFileUtil.extractYangResourcesMap(multipartFile)
@@ -91,7 +118,7 @@ class MultipartFileUtilSpec extends Specification {
def multipartZipFileFromResource(resourcePath) {
return new MockMultipartFile("file", "TEST.ZIP", "application/zip",
- getClass().getResource(resourcePath).getBytes())
+ getClass().getResource(resourcePath).getBytes())
}
def multipartFileForIOException(extension) {
diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy
index 16fbf9885a..60ecb2e3bb 100644
--- a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy
+++ b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy
@@ -1,6 +1,7 @@
/*
* ============LICENSE_START=======================================================
* Copyright (C) 2021 Bell Canada.
+ * Modifications Copyright (C) 2023 Nordix Foundation.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,25 +33,25 @@ class ZipFileSizeValidatorSpec extends Specification {
def compressedFileSize = 100
def setup() {
- objectUnderTest.setTotalEntryInArchive(0)
- objectUnderTest.setTotalSizeArchive(0)
+ objectUnderTest.setTotalYangFileEntriesInArchive(0)
+ objectUnderTest.setTotalUncompressedSizeOfYangFilesInArchive(0)
objectUnderTest.setCompressedSize(compressedFileSize)
}
- def 'Increment the total entries in Archive.'() {
- when: 'the totalEntriesInArchive value is incremented'
- objectUnderTest.incrementTotalEntryInArchive()
- then: 'the totalEntriesInArchive is incremented by 1'
- assert objectUnderTest.totalEntryInArchive == old(objectUnderTest.totalEntryInArchive) + 1
+ def 'Increment the total yang file entry count in Archive.'() {
+ when: 'the totalYangFileEntryInArchive value is incremented'
+ objectUnderTest.incrementTotalYangFileEntryCountInArchive()
+ then: 'the totalYangFileEntryInArchive is incremented by 1'
+ assert objectUnderTest.totalYangFileEntriesInArchive == old(objectUnderTest.totalYangFileEntriesInArchive) + 1
}
- def 'Update the total size of Archive.'() {
+ def 'Update the total uncompressed size of yang files in Archive.'() {
given: 'the size of an entry of archive'
def entrySize = 100
- when: 'the totalSizeArchive is to be updated with the latest entry Size'
- objectUnderTest.updateTotalSizeArchive(entrySize)
- then: 'the totalSizeArchive is updated as expected'
- assert objectUnderTest.totalSizeArchive == old(objectUnderTest.totalSizeArchive) + entrySize
+ when: 'the totalUncompressedSizeOfYangFilesInArchive is to be updated with the latest entry Size'
+ objectUnderTest.updateTotalUncompressedSizeOfYangFilesInArchive(entrySize)
+ then: 'the totalUncompressedSizeOfYangFilesInArchive is updated as expected'
+ assert objectUnderTest.totalUncompressedSizeOfYangFilesInArchive == old(objectUnderTest.totalUncompressedSizeOfYangFilesInArchive) + entrySize
}
def 'Validate the zip archive for compression ratio less that threshold compression ratio.'() {
@@ -73,29 +74,29 @@ class ZipFileSizeValidatorSpec extends Specification {
def 'Validate the zip archive for thresholdSize and thresholdEntries #caseDescriptor.'() {
given:
- objectUnderTest.setTotalEntryInArchive(totalEntriesInArchive)
- objectUnderTest.setTotalSizeArchive(totalSizeArchive)
+ objectUnderTest.setTotalYangFileEntriesInArchive(totalYangEntriesInArchive)
+ objectUnderTest.setTotalUncompressedSizeOfYangFilesInArchive(totalUncompressedSizeofYangArchive)
when: 'the validation is performed against the threshold size and threshold Entries count'
objectUnderTest.validateSizeAndEntries()
then: 'validation passes and no exception is thrown'
noExceptionThrown()
where: 'following cases are tested'
- caseDescriptor | totalSizeArchive | totalEntriesInArchive
- 'less than threshold value' | thresholdSize - 1 | thresholdEntries - 1
- 'at threshold value' | thresholdSize | thresholdEntries
+ caseDescriptor | totalUncompressedSizeofYangArchive | totalYangEntriesInArchive
+ 'less than threshold value' | thresholdSize - 1 | thresholdEntries - 1
+ 'at threshold value' | thresholdSize | thresholdEntries
}
def 'Validate the zip archive for thresholdSize and thresholdEntries with #caseDescriptor.'() {
given:
- objectUnderTest.setTotalEntryInArchive(totalEntriesInArchive)
- objectUnderTest.setTotalSizeArchive(totalSizeArchive)
+ objectUnderTest.setTotalYangFileEntriesInArchive(totalYangEntriesInArchive)
+ objectUnderTest.setTotalUncompressedSizeOfYangFilesInArchive(totalUncompressedSizeofYangArchive)
when: 'the validation is performed against the threshold size and threshold Entries count'
objectUnderTest.validateSizeAndEntries()
then: 'validation fails and exception is thrown'
thrown ModelValidationException
where: 'following cases are tested'
- caseDescriptor | totalSizeArchive | totalEntriesInArchive
- 'totalEntriesInArchive exceeds threshold value' | thresholdSize | thresholdEntries + 1
- 'totalSizeArchive exceeds threshold value' | thresholdSize + 1 | thresholdEntries
+ caseDescriptor | totalUncompressedSizeofYangArchive | totalYangEntriesInArchive
+ 'totalEntriesInArchive exceeds threshold value' | thresholdSize | thresholdEntries + 1
+ 'totalSizeArchive exceeds threshold value' | thresholdSize + 1 | thresholdEntries
}
}
diff --git a/cps-rest/src/test/resources/yang-files-set-total-1083-bytes.zip b/cps-rest/src/test/resources/yang-files-set-total-1083-bytes.zip
new file mode 100644
index 0000000000..9908055e31
--- /dev/null
+++ b/cps-rest/src/test/resources/yang-files-set-total-1083-bytes.zip
Binary files differ
diff --git a/csit/plans/cps/setup.sh b/csit/plans/cps/setup.sh
index c715da142f..e7ad67e2b6 100755
--- a/csit/plans/cps/setup.sh
+++ b/csit/plans/cps/setup.sh
@@ -63,8 +63,8 @@ curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compos
chmod +x docker-compose
docker-compose version
-# start CPS/NCMP, DMI, and PostgreSQL containers with docker compose
-docker-compose up -d
+# start CPS/NCMP, DMI Plugin, and PostgreSQL containers with docker compose
+docker-compose --profile dmi-service up -d
###################### setup sdnc #######################################
source $WORKSPACE/plans/cps/sdnc/sdnc_setup.sh
@@ -129,4 +129,4 @@ check_health $DMI_HOST:$DMI_MANAGEMENT_PORT 'dmi-plugin'
###################### ROBOT Configurations ##########################
# Pass variables required for Robot test suites in ROBOT_VARIABLES
-ROBOT_VARIABLES="-v CPS_CORE_HOST:$CPS_CORE_HOST -v CPS_CORE_PORT:$CPS_CORE_PORT -v DMI_HOST:$LOCAL_IP -v DMI_PORT:$DMI_PORT -v CPS_CORE_MANAGEMENT_PORT:$CPS_CORE_MANAGEMENT_PORT -v DATADIR:$WORKSPACE/data --exitonfailure" \ No newline at end of file
+ROBOT_VARIABLES="-v CPS_CORE_HOST:$CPS_CORE_HOST -v CPS_CORE_PORT:$CPS_CORE_PORT -v DMI_HOST:$LOCAL_IP -v DMI_PORT:$DMI_PORT -v CPS_CORE_MANAGEMENT_PORT:$CPS_CORE_MANAGEMENT_PORT -v DATADIR:$WORKSPACE/data --exitonfailure"
diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml
index dd7749ac58..08929ab222 100644
--- a/docker-compose/docker-compose.yml
+++ b/docker-compose/docker-compose.yml
@@ -18,7 +18,8 @@
services:
- ### docker-compose up -d -> run ALL services ###
+ ### docker-compose --profile dmi-service up -d -> run CPS services incl. dmi-plugin ###
+ ### docker-compose --profile dmi-stub --profile monitoring up -d -> run CPS with stubbed dmi-plugin (for registration performance testing)
### to disable notifications make notification.enabled to false & comment out kafka/zookeeper services ###
dbpostgresql:
@@ -83,7 +84,6 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- ### Comment out this section if dmi plugin is not required ###
ncmp-dmi-plugin:
container_name: ncmp-dmi-plugin
image: ${DOCKER_REPO:-nexus3.onap.org:10003}/onap/ncmp-dmi-plugin:${DMI_VERSION:-1.3.0-SNAPSHOT-latest}
@@ -108,6 +108,20 @@ services:
notification.data-updated.enabled: 'true'
NOTIFICATION_DATASPACE_FILTER_PATTERNS: '.*'
restart: unless-stopped
+ profiles:
+ - dmi-service
+
+ ncmp-dmi-plugin-stub:
+ container_name: ncmp-dmi-plugin-stub
+ image: wiremock/wiremock:2.33.2
+ ports:
+ - ${DMI_PORT:-8783}:8080
+ volumes:
+ - ../dmi-plugin-perf-stub/mappings:/home/wiremock/mappings
+ - ../dmi-plugin-perf-stub/files:/home/wiremock/__files
+ restart: unless-stopped
+ profiles:
+ - dmi-stub
init-db:
build: ${CPS_HOME:-.}/docker-compose/initfile
@@ -119,3 +133,34 @@ services:
depends_on:
cps-and-ncmp:
condition: service_started
+
+ prometheus:
+ container_name: prometheus-container
+ image: prom/prometheus:latest
+ ports:
+ - 9090:9090
+ restart: always
+ volumes:
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
+ profiles:
+ - monitoring
+
+ grafana:
+ image: grafana/grafana-oss:latest
+ user: ""
+ container_name: grafana-container
+ depends_on:
+ prometheus:
+ condition: service_started
+ ports:
+ - 3000:3000
+ volumes:
+ - grafana:/var/lib/grafana
+ environment:
+ - GF_SECURITY_ADMIN_PASSWORD=admin
+ - GF_SERVER_DOMAIN:localhost
+ profiles:
+ - monitoring
+
+volumes:
+ grafana:
diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml
new file mode 100644
index 0000000000..30f7c837f1
--- /dev/null
+++ b/docker-compose/prometheus.yml
@@ -0,0 +1,10 @@
+global:
+ scrape_interval: 5s
+ evaluation_interval: 5s
+
+scrape_configs:
+ - job_name: 'cps-and-ncm'
+ metrics_path: '/manage/prometheus'
+ scrape_interval: 5s
+ static_configs:
+ - targets: ['cps-and-ncmp:8081']
diff --git a/generate-metrics-report.sh b/generate-metrics-report.sh
new file mode 100755
index 0000000000..f1eac14719
--- /dev/null
+++ b/generate-metrics-report.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Copyright 2023 Nordix Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o errexit # Exit on most errors
+set -o nounset # Disallow expansion of unset variables
+set -o pipefail # Use last non-zero exit code in a pipeline
+#set -o xtrace # Uncomment for debugging
+
+function script_usage() {
+ cat <<EOF
+Usage:
+ -h|--help Displays this help
+ -u|--metrics-url=URL URL to Prometheus metrics
+ Default: http://localhost:8887/manage/prometheus
+ -o|--output=FILE Path to output file
+ Default: metrics-reports/metrics-[timestamp].tsv
+EOF
+}
+
+function parse_params() {
+ # Set parameter defaults
+ METRICS_URL="http://localhost:8887/manage/prometheus"
+ OUTFILE="metrics-reports/metrics-$(date --iso-8601=seconds).tsv"
+ TEMP_DIR="/tmp/cps-metrics"
+
+ # Parse parameters
+ local param
+ while [[ $# -gt 0 ]]; do
+ param="$1"
+ shift
+ case $param in
+ -h | --help)
+ script_usage
+ exit 0
+ ;;
+ -u | --metrics-url)
+ METRICS_URL=$1
+ shift
+ ;;
+ -o | --output)
+ OUTFILE=$1
+ shift
+ ;;
+ *)
+ echo "Invalid parameter was provided: $param" >&2
+ script_usage
+ exit 1
+ ;;
+ esac
+ done
+}
+
+function generate_report() {
+ # Create needed directories.
+ mkdir -p $TEMP_DIR "$(dirname "$OUTFILE")"
+
+ # Scrape raw metrics (suppress progress meter).
+ curl --fail --silent --show-error --output $TEMP_DIR/metrics-raw.txt "$METRICS_URL"
+
+ # Remove comments, sort by name, and separate by tabs.
+ grep --invert-match "^#" $TEMP_DIR/metrics-raw.txt | sort | sed 's/,[}]/}\t/' >$TEMP_DIR/metrics-all.txt
+
+ # Extract useful metrics.
+ grep -E "^cps_|^spring_data_" $TEMP_DIR/metrics-all.txt >$TEMP_DIR/metrics-cps.txt
+
+ # Extract into columns.
+ grep "_count" $TEMP_DIR/metrics-cps.txt | sed 's/_count//' | cut -f 1 >$TEMP_DIR/column1.txt
+ grep "_count" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column2.txt
+ grep "_sum" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column3.txt
+ grep "_max" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column4.txt
+
+ # Combine columns into report.
+ paste $TEMP_DIR/column{1,2,3,4}.txt >$TEMP_DIR/report.txt
+
+ # Sort by Sum (column 3), descending.
+ sort --general-numeric-sort --reverse --field-separator=$'\t' --key=3 $TEMP_DIR/report.txt >$TEMP_DIR/report-sorted.txt
+
+ # Compile final report, with column headers.
+ echo -e "Method\tCount\tSum\tMax" >"$OUTFILE"
+ cat $TEMP_DIR/report-sorted.txt >>"$OUTFILE"
+
+ # Output path to generated file
+ echo "$OUTFILE"
+}
+
+function cleanup() {
+ rm -f $TEMP_DIR/* && rmdir $TEMP_DIR 2>/dev/null
+}
+# Set up the cleanup function to be triggered upon script exit
+trap cleanup EXIT
+
+# Main script logic
+parse_params "$@"
+generate_report
+exit 0