From 7cdd659c994607285bc8f5093905938a478d43c6 Mon Sep 17 00:00:00 2001 From: ToineSiebelink Date: Thu, 21 Sep 2023 14:04:29 +0100 Subject: CM Data Subscriptions PoC/Performance test - New model introduced in test can be ported to production code - Groovy test around the new use-cases can be used as a guid for actau production code solution - Current worst use-case about 10 secodn son windows laptop. is acceptable as per Requiremenyt of 30 second - TODO: update test limits/expectations based on CI results Issue-ID: CPS-1881 Signed-off-by: ToineSiebelink Change-Id: I05f3adf7f9cc4d9a9c94a8435a392ed76f9fad66 --- .../integration/base/CpsIntegrationSpecBase.groovy | 28 +++++ .../CpsDataServiceIntegrationSpec.groovy | 1 - .../performance/base/NcmpPerfTestBase.groovy | 89 ++++++++++++++++ .../base/NcmpRegistryPerfTestBase.groovy | 54 ---------- .../ncmp/CmDataSubscriptionsPerfTest.groovy | 113 +++++++++++++++++++++ .../performance/ncmp/CmHandleQueryPerfTest.groovy | 4 +- .../cm-data-subscriptions@2023-09-21.yang | 49 +++++++++ 7 files changed, 281 insertions(+), 57 deletions(-) create mode 100644 integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy delete mode 100644 integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpRegistryPerfTestBase.groovy create mode 100644 integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy create mode 100644 integration-test/src/test/resources/data/cm-data-subscriptions/cm-data-subscriptions@2023-09-21.yang (limited to 'integration-test') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index 03ef9c2fd..40fe03018 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -79,6 +79,7 @@ class CpsIntegrationSpecBase extends Specification { def static BOOKSTORE_SCHEMA_SET = 'bookstoreSchemaSet' def static initialized = false + def now = OffsetDateTime.now() def setup() { if (!initialized) { @@ -120,4 +121,31 @@ class CpsIntegrationSpecBase extends Specification { cpsDataService.saveData(dataspaceName, anchorNamePrefix + it, data.replace("Easons", "Easons-"+it.toString()), OffsetDateTime.now()) } } + + def createJsonArray(name, numberOfElements, keyName, keyValuePrefix, dataPerKey) { + def json = '{"' + name + '":[' + (1..numberOfElements).each { + json += '{"' + keyName + '":"' + keyValuePrefix + '-' + it + '"' + if (!dataPerKey.isEmpty()) { + json += ',' + dataPerKey + } + json += '}' + if (it < numberOfElements) { + json += ',' + } + } + json += ']}' + } + + def createLeafList(name, numberOfElements, valuePrefix) { + def json = '"' + name + '":[' + (1..numberOfElements).each { + json += '"' + valuePrefix + '-' + it + '"' + if (it < numberOfElements) { + json += ',' + } + } + json += ']' + } + } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy index 2fe275383..12c97ed40 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/CpsDataServiceIntegrationSpec.groovy @@ -44,7 +44,6 @@ class CpsDataServiceIntegrationSpec extends FunctionalSpecBase { CpsDataService objectUnderTest def originalCountBookstoreChildNodes def originalCountBookstoreTopLevelListNodes - def now = OffsetDateTime.now() def setup() { objectUnderTest = cpsDataService diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy new file mode 100644 index 000000000..f5d7c5e15 --- /dev/null +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy @@ -0,0 +1,89 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.integration.performance.base + +import java.time.OffsetDateTime + +class NcmpPerfTestBase extends PerfTestBase { + + def static NCMP_PERFORMANCE_TEST_DATASPACE = 'ncmpPerformance' + def static REGISTRY_ANCHOR = 'ncmp-registry' + def static REGISTRY_SCHEMA_SET = 'registrySchemaSet' + def static CM_DATA_SUBSCRIPTIONS_ANCHOR = 'cm-data-subscriptions' + def static CM_DATA_SUBSCRIPTIONS_SCHEMA_SET = 'cmDataSubscriptionsSchemaSet' + + def datastore1cmHandlePlaceHolder = '{"datastores":{"datastore":[{"name":"ds-1","cm-handles":{"cm-handle":[]}}]}}' + def xPathForDataStore1CmHandles = '/datastores/datastore[@name="ds-1"]/cm-handles' + def numberOfCmDataSubscribers = 200 + def numberOfFiltersPerCmHandle = 10 + def numberOfCmHandlesPerCmDataSubscription = 200 + +// SHORT versions for easier debugging +// def subscriberIdPrefix = 'sub' +// def xpathPrefix = 'f' +// def cmHandlePrefix = 'ch' + + +// LONG versions for performance testing + def subscriberIdPrefix = 'some really long subscriber id to see if this makes any difference to the performance' + def xpathPrefix = 'some really long xpath/with/loads/of/children/grandchildren/and/whatever/else/I/can/think/of to see if this makes any difference to the performance' + def cmHandlePrefix = 'some really long cm handle id to see if this makes any difference to the performance' + + def printTitle() { + println('## N C M P P E R F O R M A N C E T E S T R E S U L T S ##') + } + + def isInitialised() { + return dataspaceExists(NCMP_PERFORMANCE_TEST_DATASPACE) + } + + def setupPerformanceInfraStructure() { + cpsAdminService.createDataspace(NCMP_PERFORMANCE_TEST_DATASPACE) + createRegistrySchemaSet() + createCmDataSubscriptionsSchemaSet() + addCmSubscriptionData() + } + + def createInitialData() { + cpsAdminService.createAnchor(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_SCHEMA_SET, REGISTRY_ANCHOR) + def data = readResourceDataFile('ncmp-registry/1000-cmhandles.json') + cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, data, OffsetDateTime.now()) + } + + def createRegistrySchemaSet() { + def modelAsString = readResourceDataFile('ncmp-registry/dmi-registry@2022-05-10.yang') + cpsModuleService.createSchemaSet(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_SCHEMA_SET, [registry: modelAsString]) + } + + def createCmDataSubscriptionsSchemaSet() { + def modelAsString = readResourceDataFile('cm-data-subscriptions/cm-data-subscriptions@2023-09-21.yang') + cpsModuleService.createSchemaSet(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_SCHEMA_SET, [registry: modelAsString]) + } + + def addCmSubscriptionData() { + cpsAdminService.createAnchor(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_SCHEMA_SET, CM_DATA_SUBSCRIPTIONS_ANCHOR) + cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, datastore1cmHandlePlaceHolder, now) + def subscribers = createLeafList('subscribers',numberOfCmDataSubscribers, subscriberIdPrefix) + def filters = '"filters":' + createJsonArray('filter',numberOfFiltersPerCmHandle,'xpath',xpathPrefix,subscribers) + def cmHandles = createJsonArray('cm-handle',numberOfCmHandlesPerCmDataSubscription,'id',cmHandlePrefix, filters) + cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, xPathForDataStore1CmHandles, cmHandles, now) + } +} diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpRegistryPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpRegistryPerfTestBase.groovy deleted file mode 100644 index d169bd757..000000000 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpRegistryPerfTestBase.groovy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the 'License'); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.integration.performance.base - -import java.time.OffsetDateTime - -import org.onap.cps.integration.base.CpsIntegrationSpecBase - -class NcmpRegistryPerfTestBase extends PerfTestBase { - - def static REGISTRY_ANCHOR = 'ncmp-registry' - def static REGISTRY_SCHEMA_SET = 'registrySchemaSet' - def static NCMP_PERFORMANCE_TEST_DATASPACE = 'ncmpPerformacne' - - def printTitle() { - println('## N C M P P E R F O R M A N C E T E S T R E S U L T S ##') - } - - def isInitialised() { - return dataspaceExists(NCMP_PERFORMANCE_TEST_DATASPACE) - } - - def setupPerformanceInfraStructure() { - cpsAdminService.createDataspace(NCMP_PERFORMANCE_TEST_DATASPACE) - def modelAsString = readResourceDataFile('ncmp-registry/dmi-registry@2022-05-10.yang') - cpsModuleService.createSchemaSet(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_SCHEMA_SET, [registry: modelAsString]) - } - - def createInitialData() { - def data = readResourceDataFile('ncmp-registry/1000-cmhandles.json') - cpsAdminService.createAnchor(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_SCHEMA_SET, REGISTRY_ANCHOR) - cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, data, OffsetDateTime.now()) - } - - -} diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy new file mode 100644 index 000000000..00e2d07b3 --- /dev/null +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy @@ -0,0 +1,113 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.integration.performance.ncmp + +import org.onap.cps.api.CpsQueryService +import org.onap.cps.integration.performance.base.NcmpPerfTestBase +import org.onap.cps.spi.model.DataNode + +import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS + +class CmDataSubscribersPerfTest extends NcmpPerfTestBase { + + def datastore1cmHandlePlaceHolder = '{"datastores":{"datastore":[{"name":"ds-1","cm-handles":{"cm-handle":[]}}]}}' + def xPathForDataStore1CmHandles = '/datastores/datastore[@name="ds-1"]/cm-handles' + + CpsQueryService objectUnderTest + + def setup() { objectUnderTest = cpsQueryService } + + def totalNumberOfEntries = numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription + + def random = new Random() + + def 'Find many subscribers in large dataset.'() { + when: 'all filters are queried' + stopWatch.start() + def cpsPath = '//filter' + def result = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) + then: 'got all filter entries' + result.size() == totalNumberOfEntries + then: 'find a random subscriptions by iteration (worst case: whole subscription matches previous entries)' + def matches = querySubscriptionsByIteration(result, -1) + stopWatch.stop() + matches.size() == numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription + and: 'query all subscribers within 1 second' + def durationInMillis = stopWatch.getTotalTimeMillis() + recordAndAssertPerformance("Query all subscribers", 1_000, durationInMillis) + } + + def 'Worst case new subscription (200x10 new entries).'() { + given: 'a new subscription with non-matching data' + def subscribers = createLeafList('subscribers',1, subscriberIdPrefix) + def filters = '"filters":' + createJsonArray('filter',numberOfFiltersPerCmHandle,'xpath','other_' + xpathPrefix,subscribers) + def cmHandles = createJsonArray('cm-handle',numberOfCmHandlesPerCmDataSubscription,'id','other' + cmHandlePrefix, filters) + when: 'Insert a new subscription' + stopWatch.start() + cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, xPathForDataStore1CmHandles, cmHandles, now) + stopWatch.stop() + def durationInMillis = stopWatch.getTotalTimeMillis() + then: 'insert new subscription with 1 second' + recordAndAssertPerformance("Insert new subscription", 1_000, durationInMillis) + } + + def 'Worst case subscription update (200x10 matching entries).'() { + given: 'all filters are queried' + def cpsPath = '//filter' + def result = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) + and: 'find all entries for an existing subscriptions' + def matches = querySubscriptionsByIteration(result, 1) + when: 'Update all subscriptions found' + stopWatch.start() + /* the production code version of this should manipulate the original subscribersAsArray of course + but for the (performance) poc creating another array with one extra element suffices + */ + def jsonPerPath = [:] + matches.each { xpath, subscribersAsArray -> + def updatedSubscribers = createLeafList('subscribers', 1 + numberOfCmDataSubscribers, subscriberIdPrefix) + def filterEntry = '{"filter": {"xpath":"' + xpath + '", ' + updatedSubscribers + ' } }' + def parentPath = xpath.toString().substring(0, xpath.toString().indexOf('/filter[@xpath=')) + jsonPerPath.put(parentPath, filterEntry) + } + cpsDataService.updateDataNodesAndDescendants(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, jsonPerPath, now) + stopWatch.stop() + def durationInMillis = stopWatch.getTotalTimeMillis() + then: 'Update matching subscription within 8 seconds' + //TODO Toine check with Daniel if this can be optimized quickly without really changing production code + // ie is there a better way of doing these 2,000 updates + recordAndAssertPerformance("Update matching subscription", 8_000, durationInMillis) + } + + def querySubscriptionsByIteration(Collection allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) { + def matches = [:] + allSubscriptionsAsDataNodes.each { + String[] subscribersAsArray = it.leaves.get('subscribers') + Set subscribersAsSet = new HashSet<>(Arrays.asList(subscribersAsArray)) + def targetSubscriptionId = subscriberIdPrefix + '-' + ( targetSubscriptionSequenceNumber > 0 ? targetSubscriptionSequenceNumber + : 1 + random.nextInt(numberOfCmDataSubscribers) ) + if (subscribersAsSet.contains(targetSubscriptionId)) { + matches.put(it.xpath, subscribersAsArray) + } + } + return matches + } + +} diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index d01216ee5..54e56d873 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -22,11 +22,11 @@ package org.onap.cps.integration.performance.ncmp import java.util.stream.Collectors import org.onap.cps.api.CpsQueryService -import org.onap.cps.integration.performance.base.NcmpRegistryPerfTestBase +import org.onap.cps.integration.performance.base.NcmpPerfTestBase import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS -class CmHandleQueryPerfTest extends NcmpRegistryPerfTestBase { +class CmHandleQueryPerfTest extends NcmpPerfTestBase { CpsQueryService objectUnderTest diff --git a/integration-test/src/test/resources/data/cm-data-subscriptions/cm-data-subscriptions@2023-09-21.yang b/integration-test/src/test/resources/data/cm-data-subscriptions/cm-data-subscriptions@2023-09-21.yang new file mode 100644 index 000000000..552f13715 --- /dev/null +++ b/integration-test/src/test/resources/data/cm-data-subscriptions/cm-data-subscriptions@2023-09-21.yang @@ -0,0 +1,49 @@ +module cm-data-subscriptions { + yang-version 1.1; + namespace "org:onap:cps:ncmp"; + + prefix cmds; + + revision "2023-09-21" { + description + "First release, Proof of Concept & Performance"; + } + + container datastores { + + list datastore { + key "name"; + + leaf name { + type string; + } + + container cm-handles { + + list cm-handle { + key "id"; + + leaf id { + type string; + } + + container filters { + + list filter { + key "xpath"; + + leaf xpath { + type string; + } + + leaf-list subscribers { + type string; + } + + } + } + } + } + } + } +} -- cgit 1.2.3-korg