diff options
12 files changed, 263 insertions, 68 deletions
diff --git a/.gitignore b/.gitignore index a377967df..4f80c2b8a 100755 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,6 @@ tmp/ /__pycache__/* /docs/docs/ -/docs/.vscode/
\ No newline at end of file +/docs/.vscode/ + +/metrics-reports/ diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java index 59b960af6..e71b72ab8 100755 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java @@ -327,36 +327,33 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService final List<String> tobeRemovedCmHandles) { final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>(tobeRemovedCmHandles.size()); - final Map<String, YangModelCmHandle> cmHandleIdToYangModelCmHandleMap = tobeRemovedCmHandles.stream() - .collect(Collectors.toMap(cmHandleId -> cmHandleId, inventoryPersistence::getYangModelCmHandle)); + final Collection<YangModelCmHandle> yangModelCmHandles = + inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles); - final Collection<YangModelCmHandle> yangModelCmHandles = cmHandleIdToYangModelCmHandleMap.values(); updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING); + final Set<String> notDeletedCmHandles = new HashSet<>(); for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) { try { batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch); tobeRemovedCmHandleBatch.forEach(cmHandleId -> cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId))); - } catch (final Exception batchException) { + } catch (final RuntimeException batchException) { log.error("Unable to de-register cm-handle batch, retrying on each cm handle"); for (final String cmHandleId : tobeRemovedCmHandleBatch) { final CmHandleRegistrationResponse cmHandleRegistrationResponse = deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId); cmHandleRegistrationResponses.add(cmHandleRegistrationResponse); + if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) { + notDeletedCmHandles.add(cmHandleId); + } } } } - final Collection<YangModelCmHandle> deletedYangModelCmHandles = - cmHandleRegistrationResponses.stream() - .filter(cmHandleRegistrationResponse -> - cmHandleRegistrationResponse.getStatus().equals(CmHandleRegistrationResponse.Status.SUCCESS)) - .map(CmHandleRegistrationResponse::getCmHandle) - .map(cmHandleIdToYangModelCmHandleMap::get) - .collect(Collectors.toList()); - updateCmHandleStateBatch(deletedYangModelCmHandles, CmHandleState.DELETED); + yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId())); + updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED); return cmHandleRegistrationResponses; } @@ -383,9 +380,9 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles, final CmHandleState cmHandleState) { - final Map<YangModelCmHandle, CmHandleState> cmHandleIdsToBeRemoved = new HashMap<>(); - yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleIdsToBeRemoved.put(yangModelCmHandle, cmHandleState)); - lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleIdsToBeRemoved); + final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size()); + yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState)); + lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle); } private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy index e8c53738b..cf3454991 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy @@ -74,8 +74,8 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { dmiRegistration.setCreatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-1', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])]) dmiRegistration.setUpdatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-2', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])]) dmiRegistration.setRemovedCmHandles(['cmhandle-2']) - and: 'any cm handle is persisted' - mockInventoryPersistence.getYangModelCmHandle(_) >> new YangModelCmHandle() + and: 'cm handles are persisted' + mockInventoryPersistence.getYangModelCmHandles(['cmhandle-2']) >> [new YangModelCmHandle()] when: 'registration is processed' objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration) then: 'cm-handles are removed first' @@ -245,7 +245,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { def 'Remove CmHandle Successfully: #scenario'() { given: 'a registration' - mockInventoryPersistence.getYangModelCmHandle(_) >> new YangModelCmHandle() + addPersistedYangModelCmHandles(['cmhandle']) def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: '#scenario' @@ -319,7 +319,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { def 'Remove CmHandle Error Handling: Schema Set Deletion failed'() { given: 'a registration' - mockInventoryPersistence.getYangModelCmHandle('cmhandle') >> new YangModelCmHandle() + addPersistedYangModelCmHandles(['cmhandle']) def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: 'schema set deletion failed with unknown error' @@ -344,7 +344,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { def 'Remove CmHandle Error Handling: #scenario'() { given: 'a registration' - mockInventoryPersistence.getYangModelCmHandle('cmhandle') >> new YangModelCmHandle() + addPersistedYangModelCmHandles(['cmhandle']) def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: 'cm-handle deletion fails on batch' @@ -378,9 +378,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { } def addPersistedYangModelCmHandles(ids) { - ids.each { - def yangModelCmHandle = new YangModelCmHandle(id:it) - mockInventoryPersistence.getYangModelCmHandle(it) >> yangModelCmHandle - } + def yangModelCmHandles = ids.collect { new YangModelCmHandle(id:it) } + mockInventoryPersistence.getYangModelCmHandles(ids) >> yangModelCmHandles } } diff --git a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java index 534077caa..3e01f6ee8 100644 --- a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java @@ -2,6 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2020 Pantheon.tech * Modifications Copyright (C) 2021 Bell Canada. + * Modifications Copyright (C) 2023 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -127,16 +128,17 @@ public class MultipartFileUtil { } private static String extractYangResourceContent(final ZipInputStream zipInputStream, - final ZipFileSizeValidator zipFileSizeValidator) throws IOException { + final ZipFileSizeValidator zipFileSizeValidator) + throws IOException { try (final var byteArrayOutputStream = new ByteArrayOutputStream()) { var totalSizeEntry = 0; int numberOfBytesRead; final var buffer = new byte[READ_BUFFER_SIZE]; - zipFileSizeValidator.incrementTotalEntryInArchive(); + zipFileSizeValidator.incrementTotalYangFileEntryCountInArchive(); while ((numberOfBytesRead = zipInputStream.read(buffer, 0, READ_BUFFER_SIZE)) > 0) { byteArrayOutputStream.write(buffer, 0, numberOfBytesRead); totalSizeEntry += numberOfBytesRead; - zipFileSizeValidator.updateTotalSizeArchive(numberOfBytesRead); + zipFileSizeValidator.updateTotalUncompressedSizeOfYangFilesInArchive(numberOfBytesRead); zipFileSizeValidator.validateCompresssionRatio(totalSizeEntry); } return byteArrayOutputStream.toString(StandardCharsets.UTF_8); diff --git a/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java b/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java index d148fb70d..2e303d1d4 100644 --- a/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/utils/ZipFileSizeValidator.java @@ -1,6 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2021 Bell Canada. + * Modifications Copyright (C) 2023 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,31 +29,33 @@ import org.onap.cps.spi.exceptions.ModelValidationException; public class ZipFileSizeValidator { private static final int THRESHOLD_ENTRIES = 10000; - private static final int THRESHOLD_SIZE = 100000000; + private static int THRESHOLD_SIZE = 100000000; private static final double THRESHOLD_RATIO = 40; private static final String INVALID_ZIP = "Invalid ZIP archive content."; - private int totalSizeArchive = 0; - private int totalEntryInArchive = 0; + private int totalUncompressedSizeOfYangFilesInArchive = 0; + private int totalYangFileEntriesInArchive = 0; private long compressedSize = 0; /** * Increment the totalEntryInArchive by 1. */ - public void incrementTotalEntryInArchive() { - totalEntryInArchive++; + public void incrementTotalYangFileEntryCountInArchive() { + totalYangFileEntriesInArchive++; } /** * Update the totalSizeArchive by numberOfBytesRead. + * * @param numberOfBytesRead the number of bytes of each entry */ - public void updateTotalSizeArchive(final int numberOfBytesRead) { - totalSizeArchive += numberOfBytesRead; + public void updateTotalUncompressedSizeOfYangFilesInArchive(final int numberOfBytesRead) { + totalUncompressedSizeOfYangFilesInArchive += numberOfBytesRead; } /** * Validate the total Compression size of the zip. + * * @param totalEntrySize the size of the unzipped entry. */ public void validateCompresssionRatio(final int totalEntrySize) { @@ -68,13 +71,14 @@ public class ZipFileSizeValidator { * Validate the total Size and number of entries in the zip. */ public void validateSizeAndEntries() { - if (totalSizeArchive > THRESHOLD_SIZE) { + if (totalUncompressedSizeOfYangFilesInArchive > THRESHOLD_SIZE) { throw new ModelValidationException(INVALID_ZIP, - String.format("The uncompressed data size exceeds the CPS limit %s bytes.", THRESHOLD_SIZE)); + String.format("The total size of uncompressed yang files exceeds the CPS limit of %s bytes.", + THRESHOLD_SIZE)); } - if (totalEntryInArchive > THRESHOLD_ENTRIES) { + if (totalYangFileEntriesInArchive > THRESHOLD_ENTRIES) { throw new ModelValidationException(INVALID_ZIP, - String.format("The number of entries in the archive exceeds the CPS limit %s.", + String.format("The number of yang file entries in the archive exceeds the CPS limit %s.", THRESHOLD_ENTRIES)); } } diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy index 3f4729ec7..67ee50e89 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy @@ -1,6 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2020 Pantheon.tech + * Modifications Copyright (C) 2023 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +52,7 @@ class MultipartFileUtilSpec extends Specification { def 'Extract yang resources from zip archive.'() { given: 'uploaded zip archive containing 2 yang files and 1 not yang (json) file' def multipartFile = new MockMultipartFile("file", "TEST.ZIP", "application/zip", - getClass().getResource("/yang-files-set.zip").getBytes()) + getClass().getResource("/yang-files-set.zip").getBytes()) when: 'resources are extracted from zip file' def result = MultipartFileUtil.extractYangResourcesMap(multipartFile) then: 'information from yang files is extracted, not yang file (json) is ignored' @@ -60,6 +61,32 @@ class MultipartFileUtilSpec extends Specification { assert result["component.yang"] == "fake component content 1\n" } + def 'Yang file limits in zip archive: #scenario for the bug reported in CPS-1477'() { + given: 'a yang file size (uncompressed) limit of #threshold bytes' + ZipFileSizeValidator.THRESHOLD_SIZE = threshold + and: 'an archive with a yang file of 1083 bytes' + def multipartFile = multipartZipFileFromResource('/yang-files-set-total-1083-bytes.zip') + when: 'attempt to extract yang files' + def thrownException = null + try { + MultipartFileUtil.extractYangResourcesMap(multipartFile) + } catch (Exception e) { + thrownException = e + } + then: 'ModelValidationException indicating size limit is only thrown when threshold exceeded' + if (thresholdExceeded) { + assert thrownException instanceof ModelValidationException + assert thrownException.details.contains('limit of ' + threshold + ' bytes') + } else { + assert thrownException == null + } + where: + scenario | threshold || thresholdExceeded + 'exceed limit' | 1082 || true + 'equals to limit' | 1083 || false + 'within limit' | 1084 || false + } + def 'Extract resources from zip archive having #caseDescriptor.'() { when: 'attempt to extract resources from zip file is performed' MultipartFileUtil.extractYangResourcesMap(multipartFile) @@ -91,7 +118,7 @@ class MultipartFileUtilSpec extends Specification { def multipartZipFileFromResource(resourcePath) { return new MockMultipartFile("file", "TEST.ZIP", "application/zip", - getClass().getResource(resourcePath).getBytes()) + getClass().getResource(resourcePath).getBytes()) } def multipartFileForIOException(extension) { diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy index 16fbf9885..60ecb2e3b 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/ZipFileSizeValidatorSpec.groovy @@ -1,6 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2021 Bell Canada. + * Modifications Copyright (C) 2023 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,25 +33,25 @@ class ZipFileSizeValidatorSpec extends Specification { def compressedFileSize = 100 def setup() { - objectUnderTest.setTotalEntryInArchive(0) - objectUnderTest.setTotalSizeArchive(0) + objectUnderTest.setTotalYangFileEntriesInArchive(0) + objectUnderTest.setTotalUncompressedSizeOfYangFilesInArchive(0) objectUnderTest.setCompressedSize(compressedFileSize) } - def 'Increment the total entries in Archive.'() { - when: 'the totalEntriesInArchive value is incremented' - objectUnderTest.incrementTotalEntryInArchive() - then: 'the totalEntriesInArchive is incremented by 1' - assert objectUnderTest.totalEntryInArchive == old(objectUnderTest.totalEntryInArchive) + 1 + def 'Increment the total yang file entry count in Archive.'() { + when: 'the totalYangFileEntryInArchive value is incremented' + objectUnderTest.incrementTotalYangFileEntryCountInArchive() + then: 'the totalYangFileEntryInArchive is incremented by 1' + assert objectUnderTest.totalYangFileEntriesInArchive == old(objectUnderTest.totalYangFileEntriesInArchive) + 1 } - def 'Update the total size of Archive.'() { + def 'Update the total uncompressed size of yang files in Archive.'() { given: 'the size of an entry of archive' def entrySize = 100 - when: 'the totalSizeArchive is to be updated with the latest entry Size' - objectUnderTest.updateTotalSizeArchive(entrySize) - then: 'the totalSizeArchive is updated as expected' - assert objectUnderTest.totalSizeArchive == old(objectUnderTest.totalSizeArchive) + entrySize + when: 'the totalUncompressedSizeOfYangFilesInArchive is to be updated with the latest entry Size' + objectUnderTest.updateTotalUncompressedSizeOfYangFilesInArchive(entrySize) + then: 'the totalUncompressedSizeOfYangFilesInArchive is updated as expected' + assert objectUnderTest.totalUncompressedSizeOfYangFilesInArchive == old(objectUnderTest.totalUncompressedSizeOfYangFilesInArchive) + entrySize } def 'Validate the zip archive for compression ratio less that threshold compression ratio.'() { @@ -73,29 +74,29 @@ class ZipFileSizeValidatorSpec extends Specification { def 'Validate the zip archive for thresholdSize and thresholdEntries #caseDescriptor.'() { given: - objectUnderTest.setTotalEntryInArchive(totalEntriesInArchive) - objectUnderTest.setTotalSizeArchive(totalSizeArchive) + objectUnderTest.setTotalYangFileEntriesInArchive(totalYangEntriesInArchive) + objectUnderTest.setTotalUncompressedSizeOfYangFilesInArchive(totalUncompressedSizeofYangArchive) when: 'the validation is performed against the threshold size and threshold Entries count' objectUnderTest.validateSizeAndEntries() then: 'validation passes and no exception is thrown' noExceptionThrown() where: 'following cases are tested' - caseDescriptor | totalSizeArchive | totalEntriesInArchive - 'less than threshold value' | thresholdSize - 1 | thresholdEntries - 1 - 'at threshold value' | thresholdSize | thresholdEntries + caseDescriptor | totalUncompressedSizeofYangArchive | totalYangEntriesInArchive + 'less than threshold value' | thresholdSize - 1 | thresholdEntries - 1 + 'at threshold value' | thresholdSize | thresholdEntries } def 'Validate the zip archive for thresholdSize and thresholdEntries with #caseDescriptor.'() { given: - objectUnderTest.setTotalEntryInArchive(totalEntriesInArchive) - objectUnderTest.setTotalSizeArchive(totalSizeArchive) + objectUnderTest.setTotalYangFileEntriesInArchive(totalYangEntriesInArchive) + objectUnderTest.setTotalUncompressedSizeOfYangFilesInArchive(totalUncompressedSizeofYangArchive) when: 'the validation is performed against the threshold size and threshold Entries count' objectUnderTest.validateSizeAndEntries() then: 'validation fails and exception is thrown' thrown ModelValidationException where: 'following cases are tested' - caseDescriptor | totalSizeArchive | totalEntriesInArchive - 'totalEntriesInArchive exceeds threshold value' | thresholdSize | thresholdEntries + 1 - 'totalSizeArchive exceeds threshold value' | thresholdSize + 1 | thresholdEntries + caseDescriptor | totalUncompressedSizeofYangArchive | totalYangEntriesInArchive + 'totalEntriesInArchive exceeds threshold value' | thresholdSize | thresholdEntries + 1 + 'totalSizeArchive exceeds threshold value' | thresholdSize + 1 | thresholdEntries } } diff --git a/cps-rest/src/test/resources/yang-files-set-total-1083-bytes.zip b/cps-rest/src/test/resources/yang-files-set-total-1083-bytes.zip Binary files differnew file mode 100644 index 000000000..9908055e3 --- /dev/null +++ b/cps-rest/src/test/resources/yang-files-set-total-1083-bytes.zip diff --git a/csit/plans/cps/setup.sh b/csit/plans/cps/setup.sh index c715da142..e7ad67e2b 100755 --- a/csit/plans/cps/setup.sh +++ b/csit/plans/cps/setup.sh @@ -63,8 +63,8 @@ curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compos chmod +x docker-compose docker-compose version -# start CPS/NCMP, DMI, and PostgreSQL containers with docker compose -docker-compose up -d +# start CPS/NCMP, DMI Plugin, and PostgreSQL containers with docker compose +docker-compose --profile dmi-service up -d ###################### setup sdnc ####################################### source $WORKSPACE/plans/cps/sdnc/sdnc_setup.sh @@ -129,4 +129,4 @@ check_health $DMI_HOST:$DMI_MANAGEMENT_PORT 'dmi-plugin' ###################### ROBOT Configurations ########################## # Pass variables required for Robot test suites in ROBOT_VARIABLES -ROBOT_VARIABLES="-v CPS_CORE_HOST:$CPS_CORE_HOST -v CPS_CORE_PORT:$CPS_CORE_PORT -v DMI_HOST:$LOCAL_IP -v DMI_PORT:$DMI_PORT -v CPS_CORE_MANAGEMENT_PORT:$CPS_CORE_MANAGEMENT_PORT -v DATADIR:$WORKSPACE/data --exitonfailure"
\ No newline at end of file +ROBOT_VARIABLES="-v CPS_CORE_HOST:$CPS_CORE_HOST -v CPS_CORE_PORT:$CPS_CORE_PORT -v DMI_HOST:$LOCAL_IP -v DMI_PORT:$DMI_PORT -v CPS_CORE_MANAGEMENT_PORT:$CPS_CORE_MANAGEMENT_PORT -v DATADIR:$WORKSPACE/data --exitonfailure" diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index dd7749ac5..08929ab22 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -18,7 +18,8 @@ services: - ### docker-compose up -d -> run ALL services ### + ### docker-compose --profile dmi-service up -d -> run CPS services incl. dmi-plugin ### + ### docker-compose --profile dmi-stub --profile monitoring up -d -> run CPS with stubbed dmi-plugin (for registration performance testing) ### to disable notifications make notification.enabled to false & comment out kafka/zookeeper services ### dbpostgresql: @@ -83,7 +84,6 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - ### Comment out this section if dmi plugin is not required ### ncmp-dmi-plugin: container_name: ncmp-dmi-plugin image: ${DOCKER_REPO:-nexus3.onap.org:10003}/onap/ncmp-dmi-plugin:${DMI_VERSION:-1.3.0-SNAPSHOT-latest} @@ -108,6 +108,20 @@ services: notification.data-updated.enabled: 'true' NOTIFICATION_DATASPACE_FILTER_PATTERNS: '.*' restart: unless-stopped + profiles: + - dmi-service + + ncmp-dmi-plugin-stub: + container_name: ncmp-dmi-plugin-stub + image: wiremock/wiremock:2.33.2 + ports: + - ${DMI_PORT:-8783}:8080 + volumes: + - ../dmi-plugin-perf-stub/mappings:/home/wiremock/mappings + - ../dmi-plugin-perf-stub/files:/home/wiremock/__files + restart: unless-stopped + profiles: + - dmi-stub init-db: build: ${CPS_HOME:-.}/docker-compose/initfile @@ -119,3 +133,34 @@ services: depends_on: cps-and-ncmp: condition: service_started + + prometheus: + container_name: prometheus-container + image: prom/prometheus:latest + ports: + - 9090:9090 + restart: always + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + profiles: + - monitoring + + grafana: + image: grafana/grafana-oss:latest + user: "" + container_name: grafana-container + depends_on: + prometheus: + condition: service_started + ports: + - 3000:3000 + volumes: + - grafana:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_SERVER_DOMAIN:localhost + profiles: + - monitoring + +volumes: + grafana: diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml new file mode 100644 index 000000000..30f7c837f --- /dev/null +++ b/docker-compose/prometheus.yml @@ -0,0 +1,10 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: 'cps-and-ncm' + metrics_path: '/manage/prometheus' + scrape_interval: 5s + static_configs: + - targets: ['cps-and-ncmp:8081'] diff --git a/generate-metrics-report.sh b/generate-metrics-report.sh new file mode 100755 index 000000000..f1eac1471 --- /dev/null +++ b/generate-metrics-report.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# +# Copyright 2023 Nordix Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o errexit # Exit on most errors +set -o nounset # Disallow expansion of unset variables +set -o pipefail # Use last non-zero exit code in a pipeline +#set -o xtrace # Uncomment for debugging + +function script_usage() { + cat <<EOF +Usage: + -h|--help Displays this help + -u|--metrics-url=URL URL to Prometheus metrics + Default: http://localhost:8887/manage/prometheus + -o|--output=FILE Path to output file + Default: metrics-reports/metrics-[timestamp].tsv +EOF +} + +function parse_params() { + # Set parameter defaults + METRICS_URL="http://localhost:8887/manage/prometheus" + OUTFILE="metrics-reports/metrics-$(date --iso-8601=seconds).tsv" + TEMP_DIR="/tmp/cps-metrics" + + # Parse parameters + local param + while [[ $# -gt 0 ]]; do + param="$1" + shift + case $param in + -h | --help) + script_usage + exit 0 + ;; + -u | --metrics-url) + METRICS_URL=$1 + shift + ;; + -o | --output) + OUTFILE=$1 + shift + ;; + *) + echo "Invalid parameter was provided: $param" >&2 + script_usage + exit 1 + ;; + esac + done +} + +function generate_report() { + # Create needed directories. + mkdir -p $TEMP_DIR "$(dirname "$OUTFILE")" + + # Scrape raw metrics (suppress progress meter). + curl --fail --silent --show-error --output $TEMP_DIR/metrics-raw.txt "$METRICS_URL" + + # Remove comments, sort by name, and separate by tabs. + grep --invert-match "^#" $TEMP_DIR/metrics-raw.txt | sort | sed 's/,[}]/}\t/' >$TEMP_DIR/metrics-all.txt + + # Extract useful metrics. + grep -E "^cps_|^spring_data_" $TEMP_DIR/metrics-all.txt >$TEMP_DIR/metrics-cps.txt + + # Extract into columns. + grep "_count" $TEMP_DIR/metrics-cps.txt | sed 's/_count//' | cut -f 1 >$TEMP_DIR/column1.txt + grep "_count" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column2.txt + grep "_sum" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column3.txt + grep "_max" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column4.txt + + # Combine columns into report. + paste $TEMP_DIR/column{1,2,3,4}.txt >$TEMP_DIR/report.txt + + # Sort by Sum (column 3), descending. + sort --general-numeric-sort --reverse --field-separator=$'\t' --key=3 $TEMP_DIR/report.txt >$TEMP_DIR/report-sorted.txt + + # Compile final report, with column headers. + echo -e "Method\tCount\tSum\tMax" >"$OUTFILE" + cat $TEMP_DIR/report-sorted.txt >>"$OUTFILE" + + # Output path to generated file + echo "$OUTFILE" +} + +function cleanup() { + rm -f $TEMP_DIR/* && rmdir $TEMP_DIR 2>/dev/null +} +# Set up the cleanup function to be triggered upon script exit +trap cleanup EXIT + +# Main script logic +parse_params "$@" +generate_report +exit 0 |