Merge "Performance tests of alternate-id/module-set-tag lookup"

5 files changed, 161 insertions, 40 deletions

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy
index 68dfb4a790..a03155e9d1 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy
@@ -76,17 +76,17 @@ class CpsPerfTestBase extends PerfTestBase {
             ']}}'
     }
 
-    def 'Warm the database'() {
+    def 'CPS pre-load test data'() {
         when: 'dummy get data nodes runs so that populating the DB does not get included in other test timings'
             resourceMeter.start()
             def result = cpsDataService.getDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm1', '/', FetchDescendantsOption.OMIT_DESCENDANTS)
-            assert countDataNodesInTree(result) == 1
             resourceMeter.stop()
-            def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
-        then: 'memory used is within #peakMemoryUsage'
-            assert resourceMeter.getTotalMemoryUsageInMB() <= 30
-        and: 'all data is read within expected time'
-            recordAndAssertResourceUsage("Warming database", 100, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB())
+        then: 'expected data exists'
+            assert result.xpath == ['/openroadm-devices']
+        then: 'operation completes within expected time'
+            recordAndAssertResourceUsage('CPS pre-load test data',
+                    100, resourceMeter.totalTimeInSeconds,
+                    600, resourceMeter.totalMemoryUsageInMB)
     }
 
 }
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy
index ab5c0efcee..4b39e53273 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy
@@ -1,6 +1,6 @@
 /*
  *  ============LICENSE_START=======================================================
- *  Copyright (C) 2023 Nordix Foundation
+ *  Copyright (C) 2023-2024 Nordix Foundation
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the 'License');
  *  you may not use this file except in compliance with the License.
@@ -20,14 +20,15 @@
 
 package org.onap.cps.integration.performance.base
 
-import java.time.OffsetDateTime
 import org.onap.cps.integration.ResourceMeter
+import org.onap.cps.spi.FetchDescendantsOption
 
 class NcmpPerfTestBase extends PerfTestBase {
 
     def static NCMP_PERFORMANCE_TEST_DATASPACE = 'ncmpPerformance'
     def static REGISTRY_ANCHOR = 'ncmp-registry'
     def static REGISTRY_SCHEMA_SET = 'registrySchemaSet'
+    def static TOTAL_CM_HANDLES = 20_000
     def static CM_DATA_SUBSCRIPTIONS_ANCHOR = 'cm-data-subscriptions'
     def static CM_DATA_SUBSCRIPTIONS_SCHEMA_SET = 'cmDataSubscriptionsSchemaSet'
 
@@ -39,13 +40,6 @@ class NcmpPerfTestBase extends PerfTestBase {
 
     ResourceMeter resourceMeter = new ResourceMeter()
 
-// SHORT versions for easier debugging
-//    def subscriberIdPrefix = 'sub'
-//    def xpathPrefix = 'f'
-//    def cmHandlePrefix = 'ch'
-
-
-// LONG versions for performance testing
     def subscriberIdPrefix = 'some really long subscriber id to see if this makes any difference to the performance'
     def xpathPrefix = 'some really long xpath/with/loads/of/children/grandchildren/and/whatever/else/I/can/think/of to see if this makes any difference to the performance'
     def cmHandlePrefix = 'some really long cm handle id to see if this makes any difference to the performance'
@@ -62,13 +56,11 @@ class NcmpPerfTestBase extends PerfTestBase {
         cpsDataspaceService.createDataspace(NCMP_PERFORMANCE_TEST_DATASPACE)
         createRegistrySchemaSet()
         createCmDataSubscriptionsSchemaSet()
-        addCmSubscriptionData()
     }
 
     def createInitialData() {
-        cpsAnchorService.createAnchor(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_SCHEMA_SET, REGISTRY_ANCHOR)
-        def data = readResourceDataFile('ncmp-registry/1000-cmhandles.json')
-        cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, data, OffsetDateTime.now())
+        addRegistryData()
+        addCmSubscriptionData()
     }
 
     def createRegistrySchemaSet() {
@@ -76,6 +68,17 @@ class NcmpPerfTestBase extends PerfTestBase {
         cpsModuleService.createSchemaSet(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_SCHEMA_SET, [registry: modelAsString])
     }
 
+    def addRegistryData() {
+        cpsAnchorService.createAnchor(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_SCHEMA_SET, REGISTRY_ANCHOR)
+        cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, '{"dmi-registry": []}', now)
+        def innerNodeJsonTemplate = readResourceDataFile('ncmp-registry/innerNode.json')
+        def batchSize = 100
+        for (def i = 0; i < TOTAL_CM_HANDLES; i += batchSize) {
+            def data = '{ "cm-handles": [' + (1..batchSize).collect { innerNodeJsonTemplate.replace('CMHANDLE_ID_HERE', (it + i).toString()) }.join(',') + ']}'
+            cpsDataService.saveListElements(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, '/dmi-registry', data, now)
+        }
+    }
+
     def createCmDataSubscriptionsSchemaSet() {
         def modelAsString = readResourceDataFile('cm-data-subscriptions/cm-data-subscriptions@2023-09-21.yang')
         cpsModuleService.createSchemaSet(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_SCHEMA_SET, [registry: modelAsString])
@@ -89,4 +92,18 @@ class NcmpPerfTestBase extends PerfTestBase {
         def cmHandles = createJsonArray('cm-handle',numberOfCmHandlesPerCmDataSubscription,'id',cmHandlePrefix, filters)
         cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, xPathForDataStore1CmHandles, cmHandles, now)
     }
+
+    def 'NCMP pre-load test data'() {
+        when: 'dummy get data nodes runs so that populating the DB does not get included in other test timings'
+            resourceMeter.start()
+            def result = cpsDataService.getDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, '/', FetchDescendantsOption.OMIT_DESCENDANTS)
+            resourceMeter.stop()
+        then: 'expected data exists'
+            assert result.xpath == ['/dmi-registry']
+        and: 'operation completes within expected time'
+            recordAndAssertResourceUsage('NCMP pre-load test data',
+                    15, resourceMeter.totalTimeInSeconds,
+                    600, resourceMeter.totalMemoryUsageInMB)
+    }
+
 }
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
index ce0aab45b0..66beb603da 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
@@ -58,7 +58,7 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase {
         if (shortTitle.length() > 40) {
             shortTitle = shortTitle.substring(0, 40)
         }
-        def record = String.format('%2d.%-40s limit %8.2f took %8.2f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, thresholdInSec, recordedTimeInSec, memoryUsageInMB)
+        def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, thresholdInSec, recordedTimeInSec, memoryUsageInMB)
         record += pass ? 'PASS' : 'FAIL'
         PERFORMANCE_RECORD.add(record)
         assert recordedTimeInSec <= thresholdInSec
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
index c36ec834a7..96f85ffd7b 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
@@ -41,7 +41,6 @@ class WritePerfTest extends CpsPerfTestBase {
                     expectedDuration, resourceMeter.getTotalTimeInSeconds(),
                     memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         cleanup:
-            cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, OffsetDateTime.now())
             cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
         where:
             totalNodes || expectedDuration | memoryLimit
@@ -64,17 +63,16 @@ class WritePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
         then: 'the operation takes less than #expectedDuration and memory used is within limit'
             recordAndAssertResourceUsage("Writing ${totalBooks} books",
-                    expectedDuration, resourceMeter.getTotalTimeInSeconds(),
-                    memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+                    expectedDuration, resourceMeter.totalTimeInSeconds,
+                    memoryLimit, resourceMeter.totalMemoryUsageInMB)
         cleanup:
-            cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, OffsetDateTime.now())
             cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
         where:
             totalBooks || expectedDuration | memoryLimit
             800        || 0.5              | 50
             1600       || 1.5              | 100
-            3200       || 6                | 150
-            6400       || 18               | 200
+            3200       || 6.0              | 150
+            6400       || 18.0             | 200
     }
 
     def 'Writing openroadm list data using saveListElements.'() {
@@ -93,17 +91,16 @@ class WritePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
         then: 'the operation takes less than #expectedDuration and memory used is within limit'
             recordAndAssertResourceUsage("Saving list of ${totalNodes} devices",
-                    expectedDuration, resourceMeter.getTotalTimeInSeconds(),
-                    memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+                    expectedDuration, resourceMeter.totalTimeInSeconds,
+                    memoryLimit, resourceMeter.totalMemoryUsageInMB)
         cleanup:
-            cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, OffsetDateTime.now())
             cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
         where:
             totalNodes || expectedDuration | memoryLimit
-            50         || 4                | 200
-            100        || 7                | 200
-            200        || 14               | 250
-            400        || 28               | 250
+            50         || 2                | 100
+            100        || 4                | 200
+            200        || 7                | 400
+            400        || 14               | 500
     }
 
 }
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
index a5a6acb7a1..d95ac73195 100644
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
@@ -1,6 +1,6 @@
 /*
  *  ============LICENSE_START=======================================================
- *  Copyright (C) 2023 Nordix Foundation
+ *  Copyright (C) 2023-2024 Nordix Foundation
  *  ================================================================================
  *  Licensed under the Apache License, Version 2.0 (the 'License');
  *  you may not use this file except in compliance with the License.
@@ -20,20 +20,48 @@
 
 package org.onap.cps.integration.performance.ncmp
 
-import org.onap.cps.integration.ResourceMeter
+import org.apache.commons.lang3.StringUtils
+import org.onap.cps.ncmp.api.impl.inventory.CmHandleState
+import org.onap.cps.ncmp.api.impl.inventory.sync.ModuleSyncService
+import org.onap.cps.ncmp.api.impl.utils.YangDataConverter
+import org.onap.cps.spi.FetchDescendantsOption
+import org.onap.cps.spi.model.DataNode
+import org.springframework.beans.factory.annotation.Autowired
+
 import java.util.stream.Collectors
 import org.onap.cps.api.CpsQueryService
+import org.onap.cps.integration.ResourceMeter
 import org.onap.cps.integration.performance.base.NcmpPerfTestBase
+
 import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS
 import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS
 
 class CmHandleQueryPerfTest extends NcmpPerfTestBase {
 
+    static def MILLISECONDS = 0.001
+
     CpsQueryService objectUnderTest
     ResourceMeter resourceMeter = new ResourceMeter()
 
     def setup() { objectUnderTest = cpsQueryService }
 
+    def 'JVM warmup.'() {
+        when: 'the JVM is warmed up'
+            def iterations = 2500 // set this to 15000 for very accurate results (but test takes much longer)
+            resourceMeter.start()
+            (1..iterations).forEach {
+                cpsDataService.getDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR,
+                        '/dmi-registry/cm-handles[@id="cm-' + it + '"]', OMIT_DESCENDANTS)
+                objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR,
+                        '/dmi-registry/cm-handles[@alternate-id="alt-' + it + '"]', OMIT_DESCENDANTS)
+            }
+            resourceMeter.stop()
+        then: 'resource usage is as expected'
+            recordAndAssertResourceUsage('JVM warmup for CmHandleQueryPerfTest',
+                    30, resourceMeter.totalTimeInSeconds,
+                    300, resourceMeter.totalMemoryUsageInMB)
+    }
+
     def 'Query CM Handle IDs by a property name and value.'() {
         when: 'a cps-path query on name-value pair is performed (without getting descendants)'
             resourceMeter.start()
@@ -46,11 +74,90 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the required operations are performed within required time'
-            recordAndAssertResourceUsage("CpsPath Registry attributes Query", 0.4, durationInSeconds, 50, resourceMeter.getTotalMemoryUsageInMB())
-        and: 'all but 1 (other node) are returned'
-            result.size() == 999
+            recordAndAssertResourceUsage("CpsPath Registry attributes Query", 2, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
+        and: 'all nodes are returned'
+            result.size() == TOTAL_CM_HANDLES
         and: 'the tree contains all the expected descendants too'
-            assert countDataNodesInTree(result) == 5 * 999
+            assert countDataNodesInTree(result) == 5 * TOTAL_CM_HANDLES
+    }
+
+    def 'CM-handle is looked up by id.'() {
+        when: 'CM-handles are looked up by cm-handle-id 100 times'
+            int count = 0
+            resourceMeter.start()
+            (1..100).each {
+                count += cpsDataService.getDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR,
+                        '/dmi-registry/cm-handles[@id="cm-' + it + '"]', OMIT_DESCENDANTS).size()
+            }
+            resourceMeter.stop()
+        then:
+            assert count == 100
+        and: 'average performance is as expected'
+            def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
+            recordAndAssertResourceUsage('Look up CM-handle by id',
+                    expectedAverageResponseTime, averageResponseTime,
+                    15, resourceMeter.totalMemoryUsageInMB)
+        where:
+            expectedAverageResponseTime = 1 * MILLISECONDS
+    }
+
+    def 'CM-handle is looked up by alternate-id.'() {
+        when: 'CM-handles are looked up by alternate-id 100 times'
+            int count = 0
+            resourceMeter.start()
+            (1..100).each {
+                count += cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR,
+                        '/dmi-registry/cm-handles[@alternate-id="alt-' + it + '"]', OMIT_DESCENDANTS).size()
+            }
+            resourceMeter.stop()
+        then:
+            assert count == 100
+        and: 'average performance is as expected'
+            def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
+            recordAndAssertResourceUsage('Look up CM-handle by alternate-id',
+                    expectedAverageResponseTime, averageResponseTime,
+                    15, resourceMeter.totalMemoryUsageInMB)
+        where:
+            expectedAverageResponseTime = 10 * MILLISECONDS
+    }
+
+    def 'A batch of CM-handles is looked up by alternate-id.'() {
+        given: 'a CPS Path Query to look up 100 alternate-ids in a single operation'
+            def cpsPathQuery = '/dmi-registry/cm-handles[' + (1..100).collect { "@alternate-id='alt-${it}'" }.join(' or ') + ']'
+        when: 'CM-handles are looked up by alternate-ids in a single query'
+            resourceMeter.start()
+            def count = cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, cpsPathQuery, OMIT_DESCENDANTS).size()
+            resourceMeter.stop()
+        then: 'expected amount of data was returned'
+            assert count == 100
+        then: 'average performance is as expected'
+            def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
+            recordAndAssertResourceUsage('Batch look up CM-handle by alternate-id',
+                    expectedAverageResponseTime, averageResponseTime,
+                    15, resourceMeter.totalMemoryUsageInMB)
+        where:
+            expectedAverageResponseTime = 1 * MILLISECONDS
+    }
+
+    def 'Find any CM-handle given moduleSetTag when there are 20K READY handles with same moduleSetTag.'() {
+        given:
+            def cpsPathQuery = "/dmi-registry/cm-handles[@module-set-tag='my-module-set-tag']"
+        when: 'CM-handles are looked up by module-set-tag 100 times'
+            int count = 0
+            resourceMeter.start()
+            (1..100).each {
+                count += cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, cpsPathQuery, OMIT_DESCENDANTS).size()
+            }
+            resourceMeter.stop()
+        then:
+            assert count == TOTAL_CM_HANDLES * 100
+        then: 'average performance is as expected'
+            def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
+            recordAndAssertResourceUsage('Look up CM-handles by module-set-tag',
+                    expectedAverageResponseTime, averageResponseTime,
+                    500, resourceMeter.totalMemoryUsageInMB)
+        where:
+            expectedAverageResponseTime = 100 * MILLISECONDS
     }
 
 }