diff options
5 files changed, 113 insertions, 34 deletions
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java index 50e671d247..19547bbccf 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2023 Nordix Foundation + * Copyright (C) 2021-2024 Nordix Foundation * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2022-2023 TechMahindra Ltd. @@ -38,6 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -526,7 +527,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService private void updateFragmentEntityAndDescendantsWithDataNode(final FragmentEntity existingFragmentEntity, final DataNode newDataNode) { - existingFragmentEntity.setAttributes(jsonObjectMapper.asJsonString(newDataNode.getLeaves())); + copyAttributesFromNewDataNode(existingFragmentEntity, newDataNode); final Map<String, FragmentEntity> existingChildrenByXpath = existingFragmentEntity.getChildFragments().stream() .collect(Collectors.toMap(FragmentEntity::getXpath, childFragmentEntity -> childFragmentEntity)); @@ -668,7 +669,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService return convertToFragmentWithAllDescendants(parentEntity.getAnchor(), newListElement); } if (newListElement.getChildDataNodes().isEmpty()) { - copyAttributesFromNewListElement(existingListElementEntity, newListElement); + copyAttributesFromNewDataNode(existingListElementEntity, newListElement); existingListElementEntity.getChildFragments().clear(); } else { updateFragmentEntityAndDescendantsWithDataNode(existingListElementEntity, newListElement); @@ -681,12 +682,28 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService return !existingListElementsByXpath.containsKey(replacementDataNode.getXpath()); } - private void copyAttributesFromNewListElement(final FragmentEntity existingListElementEntity, - final DataNode newListElement) { - final FragmentEntity replacementFragmentEntity = - FragmentEntity.builder().attributes(jsonObjectMapper.asJsonString( - newListElement.getLeaves())).build(); - existingListElementEntity.setAttributes(replacementFragmentEntity.getAttributes()); + private void copyAttributesFromNewDataNode(final FragmentEntity existingFragmentEntity, + final DataNode newDataNode) { + final String oldOrderedLeavesAsJson = getOrderedLeavesAsJson(existingFragmentEntity.getAttributes()); + final String newOrderedLeavesAsJson = getOrderedLeavesAsJson(newDataNode.getLeaves()); + if (!oldOrderedLeavesAsJson.equals(newOrderedLeavesAsJson)) { + existingFragmentEntity.setAttributes(jsonObjectMapper.asJsonString(newDataNode.getLeaves())); + } + } + + private String getOrderedLeavesAsJson(final Map<String, Serializable> currentLeaves) { + final Map<String, Serializable> sortedLeaves = new TreeMap<>(String::compareTo); + sortedLeaves.putAll(currentLeaves); + return jsonObjectMapper.asJsonString(sortedLeaves); + } + + private String getOrderedLeavesAsJson(final String currentLeavesAsString) { + if (currentLeavesAsString == null) { + return "{}"; + } + final Map<String, Serializable> sortedLeaves = jsonObjectMapper.convertJsonString(currentLeavesAsString, + TreeMap.class); + return jsonObjectMapper.asJsonString(sortedLeaves); } private static Map<String, FragmentEntity> extractListElementFragmentEntitiesByXPath( diff --git a/docs/ncmp-data-operation.rst b/docs/ncmp-data-operation.rst index 617b3ed309..94d5ee9c0a 100644 --- a/docs/ncmp-data-operation.rst +++ b/docs/ncmp-data-operation.rst @@ -6,8 +6,8 @@ .. _cmHandleDataOperation: -CM Handles Data Operation Endpoints -################################### +Data Operations Endpoint +######################## .. toctree:: :maxdepth: 1 @@ -15,11 +15,11 @@ CM Handles Data Operation Endpoints Introduction ============ -For data operation CM Handles we have a Post endpoints: +For all data operations on cm handle(s), we have a post endpoint: - /ncmp/v1/data?topic={client-topic-name} forward request to it's dmi plugin service. -- Returns request id (UUID) with http status 202. +- When asynchronous (with topic) operations are executed, a request id (UUID) will be returned. Request Body ============ @@ -143,6 +143,6 @@ DMI Service 2 (POST) : `http://{dmi-host-name}:{dmi-port}/dmi/v1/data?topic=my-t } ] -Above examples are for illustration purpose only please refer link below for latest schema. +Above examples are for illustration purposes only. Please refer to link below for latest schema. :download:`Data operation event schema <schemas/data-operation-event-schema-1.0.0.json>`
\ No newline at end of file diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 7fabfc3bb3..b7f4c4fbee 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2021-2023 Nordix Foundation +.. Copyright (C) 2021-2024 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _release_notes: @@ -43,6 +43,7 @@ Bug Fixes Features -------- + - `CPS-2018 <https://jira.onap.org/browse/CPS-2018>`_ Improve performance of CPS update operations. Version: 3.4.1 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index 35f65551f8..b3030b1c6b 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2024 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -80,8 +80,8 @@ class UpdatePerfTest extends CpsPerfTestBase { where: scenario | totalNodes | startId | changeLeaves || timeLimit | memoryLimit 'Replace 0 nodes with 100' | 100 | 1 | false || 7 | 250 - 'Replace 100 using same data' | 100 | 1 | false || 5 | 250 - 'Replace 100 with new leaf values' | 100 | 1 | true || 5 | 250 + 'Replace 100 using same data' | 100 | 1 | false || 3 | 250 + 'Replace 100 with new leaf values' | 100 | 1 | true || 3 | 250 'Replace 100 with 100 new nodes' | 100 | 101 | false || 12 | 300 'Replace 50 existing and 50 new' | 100 | 151 | true || 8 | 250 'Replace 100 nodes with 0' | 0 | 1 | false || 5 | 250 @@ -106,8 +106,8 @@ class UpdatePerfTest extends CpsPerfTestBase { where: scenario | totalNodes | startId | changeLeaves || timeLimit | memoryLimit 'Replace list of 0 with 100' | 100 | 1 | false || 7 | 250 - 'Replace list of 100 using same data' | 100 | 1 | false || 5 | 250 - 'Replace list of 100 with new leaf values' | 100 | 1 | true || 5 | 250 + 'Replace list of 100 using same data' | 100 | 1 | false || 3 | 250 + 'Replace list of 100 with new leaf values' | 100 | 1 | true || 3 | 250 'Replace list with 100 new nodes' | 100 | 101 | false || 12 | 300 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 8 | 250 'Replace list of 100 nodes with 1' | 1 | 1 | false || 5 | 250 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index 2d38a0dfb6..a5669baf6a 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2024 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -25,21 +25,24 @@ import org.onap.cps.integration.performance.base.CpsPerfTestBase class WritePerfTest extends CpsPerfTestBase { + static final def WRITE_TEST_ANCHOR = 'writeTestAnchor' + def 'Writing openroadm data has linear time.'() { given: 'an empty anchor exists for openroadm' - cpsAnchorService.createAnchor(CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'writeAnchor') + cpsAnchorService.createAnchor(CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, WRITE_TEST_ANCHOR) and: 'a list of device nodes to add' def jsonData = generateOpenRoadData(totalNodes) when: 'device nodes are added' resourceMeter.start() - cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', jsonData, OffsetDateTime.now()) + cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, jsonData, OffsetDateTime.now()) resourceMeter.stop() - def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation takes less than #expectedDuration and memory used is within limit' - recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Writing ${totalNodes} devices", + expectedDuration, resourceMeter.getTotalTimeInSeconds(), + memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) cleanup: - cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) - cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') + cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, OffsetDateTime.now()) + cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalNodes || expectedDuration | memoryLimit 50 || 4 | 100 @@ -50,21 +53,22 @@ class WritePerfTest extends CpsPerfTestBase { def 'Writing bookstore data has exponential time.'() { given: 'an anchor containing a bookstore with a single category' - cpsAnchorService.createAnchor(CPS_PERFORMANCE_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET, 'writeAnchor') + cpsAnchorService.createAnchor(CPS_PERFORMANCE_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET, WRITE_TEST_ANCHOR) def parentNodeData = '{"bookstore": { "categories": [{ "code": 1, "name": "Test", "books" : [] }] }}' - cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', parentNodeData, OffsetDateTime.now()) + cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, parentNodeData, OffsetDateTime.now()) and: 'a list of books to add' def booksData = '{"books":[' + (1..totalBooks).collect {'{ "title": "' + it + '" }' }.join(',') + ']}' when: 'books are added' resourceMeter.start() - cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now()) + cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now()) resourceMeter.stop() - def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation takes less than #expectedDuration and memory used is within limit' - recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Writing ${totalBooks} books", + expectedDuration, resourceMeter.getTotalTimeInSeconds(), + memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) cleanup: - cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) - cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') + cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, OffsetDateTime.now()) + cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalBooks || expectedDuration | memoryLimit 800 || 1 | 50 @@ -73,4 +77,61 @@ class WritePerfTest extends CpsPerfTestBase { 6400 || 18 | 200 } + def 'Writing openroadm list data using saveListElements.'() { + given: 'an anchor and empty container node for openroadm' + cpsAnchorService.createAnchor(CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, WRITE_TEST_ANCHOR) + cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, + '{ "openroadm-devices": { "openroadm-device": []}}', now) + and: 'a list of device nodes to add' + def innerNode = readResourceDataFile('openroadm/innerNode.json') + def jsonListData = '{ "openroadm-device": [' + + (1..totalNodes).collect { innerNode.replace('NODE_ID_HERE', it.toString()) }.join(',') + + ']}' + when: 'device nodes are added' + resourceMeter.start() + cpsDataService.saveListElements(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', jsonListData, OffsetDateTime.now()) + resourceMeter.stop() + then: 'the operation takes less than #expectedDuration and memory used is within limit' + recordAndAssertResourceUsage("Saving list of ${totalNodes} devices", + expectedDuration, resourceMeter.getTotalTimeInSeconds(), + memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + cleanup: + cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, OffsetDateTime.now()) + cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) + where: + totalNodes || expectedDuration | memoryLimit + 50 || 4 | 200 + 100 || 7 | 200 + 200 || 14 | 250 + 400 || 28 | 250 + } + + def 'Writing openroadm list data using saveListElementsBatch.'() { + given: 'an anchor and empty container node for openroadm' + cpsAnchorService.createAnchor(CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, WRITE_TEST_ANCHOR) + cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, + '{ "openroadm-devices": { "openroadm-device": []}}', now) + and: 'a list of device nodes to add' + def innerNode = readResourceDataFile('openroadm/innerNode.json') + def multipleJsonData = (1..totalNodes).collect { + '{ "openroadm-device": [' + innerNode.replace('NODE_ID_HERE', it.toString()) + ']}' } + when: 'device nodes are added' + resourceMeter.start() + cpsDataService.saveListElementsBatch(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', multipleJsonData, OffsetDateTime.now()) + resourceMeter.stop() + then: 'the operation takes less than #expectedDuration and memory used is within limit' + recordAndAssertResourceUsage("Saving batch of ${totalNodes} lists", + expectedDuration, resourceMeter.getTotalTimeInSeconds(), + memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + cleanup: + cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, OffsetDateTime.now()) + cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) + where: + totalNodes || expectedDuration | memoryLimit + 50 || 16 | 500 + 100 || 32 | 500 + 200 || 64 | 1000 + 400 || 128 | 1250 + } + } |