diff options
4 files changed, 131 insertions, 25 deletions
diff --git a/.gitignore b/.gitignore index a377967df..4f80c2b8a 100755 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,6 @@ tmp/ /__pycache__/* /docs/docs/ -/docs/.vscode/
\ No newline at end of file +/docs/.vscode/ + +/metrics-reports/ diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java index 59b960af6..e71b72ab8 100755 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java @@ -327,36 +327,33 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService final List<String> tobeRemovedCmHandles) { final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>(tobeRemovedCmHandles.size()); - final Map<String, YangModelCmHandle> cmHandleIdToYangModelCmHandleMap = tobeRemovedCmHandles.stream() - .collect(Collectors.toMap(cmHandleId -> cmHandleId, inventoryPersistence::getYangModelCmHandle)); + final Collection<YangModelCmHandle> yangModelCmHandles = + inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles); - final Collection<YangModelCmHandle> yangModelCmHandles = cmHandleIdToYangModelCmHandleMap.values(); updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING); + final Set<String> notDeletedCmHandles = new HashSet<>(); for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) { try { batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch); tobeRemovedCmHandleBatch.forEach(cmHandleId -> cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId))); - } catch (final Exception batchException) { + } catch (final RuntimeException batchException) { log.error("Unable to de-register cm-handle batch, retrying on each cm handle"); for (final String cmHandleId : tobeRemovedCmHandleBatch) { final CmHandleRegistrationResponse cmHandleRegistrationResponse = deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId); cmHandleRegistrationResponses.add(cmHandleRegistrationResponse); + if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) { + notDeletedCmHandles.add(cmHandleId); + } } } } - final Collection<YangModelCmHandle> deletedYangModelCmHandles = - cmHandleRegistrationResponses.stream() - .filter(cmHandleRegistrationResponse -> - cmHandleRegistrationResponse.getStatus().equals(CmHandleRegistrationResponse.Status.SUCCESS)) - .map(CmHandleRegistrationResponse::getCmHandle) - .map(cmHandleIdToYangModelCmHandleMap::get) - .collect(Collectors.toList()); - updateCmHandleStateBatch(deletedYangModelCmHandles, CmHandleState.DELETED); + yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId())); + updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED); return cmHandleRegistrationResponses; } @@ -383,9 +380,9 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles, final CmHandleState cmHandleState) { - final Map<YangModelCmHandle, CmHandleState> cmHandleIdsToBeRemoved = new HashMap<>(); - yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleIdsToBeRemoved.put(yangModelCmHandle, cmHandleState)); - lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleIdsToBeRemoved); + final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size()); + yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState)); + lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle); } private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy index e8c53738b..cf3454991 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy @@ -74,8 +74,8 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { dmiRegistration.setCreatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-1', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])]) dmiRegistration.setUpdatedCmHandles([new NcmpServiceCmHandle(cmHandleId: 'cmhandle-2', publicProperties: ['publicProp1': 'value'], dmiProperties: [:])]) dmiRegistration.setRemovedCmHandles(['cmhandle-2']) - and: 'any cm handle is persisted' - mockInventoryPersistence.getYangModelCmHandle(_) >> new YangModelCmHandle() + and: 'cm handles are persisted' + mockInventoryPersistence.getYangModelCmHandles(['cmhandle-2']) >> [new YangModelCmHandle()] when: 'registration is processed' objectUnderTest.updateDmiRegistrationAndSyncModule(dmiRegistration) then: 'cm-handles are removed first' @@ -245,7 +245,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { def 'Remove CmHandle Successfully: #scenario'() { given: 'a registration' - mockInventoryPersistence.getYangModelCmHandle(_) >> new YangModelCmHandle() + addPersistedYangModelCmHandles(['cmhandle']) def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: '#scenario' @@ -319,7 +319,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { def 'Remove CmHandle Error Handling: Schema Set Deletion failed'() { given: 'a registration' - mockInventoryPersistence.getYangModelCmHandle('cmhandle') >> new YangModelCmHandle() + addPersistedYangModelCmHandles(['cmhandle']) def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: 'schema set deletion failed with unknown error' @@ -344,7 +344,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { def 'Remove CmHandle Error Handling: #scenario'() { given: 'a registration' - mockInventoryPersistence.getYangModelCmHandle('cmhandle') >> new YangModelCmHandle() + addPersistedYangModelCmHandles(['cmhandle']) def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: 'cm-handle deletion fails on batch' @@ -378,9 +378,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { } def addPersistedYangModelCmHandles(ids) { - ids.each { - def yangModelCmHandle = new YangModelCmHandle(id:it) - mockInventoryPersistence.getYangModelCmHandle(it) >> yangModelCmHandle - } + def yangModelCmHandles = ids.collect { new YangModelCmHandle(id:it) } + mockInventoryPersistence.getYangModelCmHandles(ids) >> yangModelCmHandles } } diff --git a/generate-metrics-report.sh b/generate-metrics-report.sh new file mode 100755 index 000000000..f1eac1471 --- /dev/null +++ b/generate-metrics-report.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# +# Copyright 2023 Nordix Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o errexit # Exit on most errors +set -o nounset # Disallow expansion of unset variables +set -o pipefail # Use last non-zero exit code in a pipeline +#set -o xtrace # Uncomment for debugging + +function script_usage() { + cat <<EOF +Usage: + -h|--help Displays this help + -u|--metrics-url=URL URL to Prometheus metrics + Default: http://localhost:8887/manage/prometheus + -o|--output=FILE Path to output file + Default: metrics-reports/metrics-[timestamp].tsv +EOF +} + +function parse_params() { + # Set parameter defaults + METRICS_URL="http://localhost:8887/manage/prometheus" + OUTFILE="metrics-reports/metrics-$(date --iso-8601=seconds).tsv" + TEMP_DIR="/tmp/cps-metrics" + + # Parse parameters + local param + while [[ $# -gt 0 ]]; do + param="$1" + shift + case $param in + -h | --help) + script_usage + exit 0 + ;; + -u | --metrics-url) + METRICS_URL=$1 + shift + ;; + -o | --output) + OUTFILE=$1 + shift + ;; + *) + echo "Invalid parameter was provided: $param" >&2 + script_usage + exit 1 + ;; + esac + done +} + +function generate_report() { + # Create needed directories. + mkdir -p $TEMP_DIR "$(dirname "$OUTFILE")" + + # Scrape raw metrics (suppress progress meter). + curl --fail --silent --show-error --output $TEMP_DIR/metrics-raw.txt "$METRICS_URL" + + # Remove comments, sort by name, and separate by tabs. + grep --invert-match "^#" $TEMP_DIR/metrics-raw.txt | sort | sed 's/,[}]/}\t/' >$TEMP_DIR/metrics-all.txt + + # Extract useful metrics. + grep -E "^cps_|^spring_data_" $TEMP_DIR/metrics-all.txt >$TEMP_DIR/metrics-cps.txt + + # Extract into columns. + grep "_count" $TEMP_DIR/metrics-cps.txt | sed 's/_count//' | cut -f 1 >$TEMP_DIR/column1.txt + grep "_count" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column2.txt + grep "_sum" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column3.txt + grep "_max" $TEMP_DIR/metrics-cps.txt | cut -f 2 >$TEMP_DIR/column4.txt + + # Combine columns into report. + paste $TEMP_DIR/column{1,2,3,4}.txt >$TEMP_DIR/report.txt + + # Sort by Sum (column 3), descending. + sort --general-numeric-sort --reverse --field-separator=$'\t' --key=3 $TEMP_DIR/report.txt >$TEMP_DIR/report-sorted.txt + + # Compile final report, with column headers. + echo -e "Method\tCount\tSum\tMax" >"$OUTFILE" + cat $TEMP_DIR/report-sorted.txt >>"$OUTFILE" + + # Output path to generated file + echo "$OUTFILE" +} + +function cleanup() { + rm -f $TEMP_DIR/* && rmdir $TEMP_DIR 2>/dev/null +} +# Set up the cleanup function to be triggered upon script exit +trap cleanup EXIT + +# Main script logic +parse_params "$@" +generate_report +exit 0 |