Merge "Temp Table Creation improvements"

7 files changed, 231 insertions, 60 deletions

diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java
index 5e4de7fec4..00e53aa00f 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java
@@ -28,6 +28,5 @@ import org.onap.cps.spi.model.ModuleReference;
  */
 public interface ModuleReferenceQuery {
 
-    Collection<ModuleReference> identifyNewModuleReferences(
-        final Collection<ModuleReference> moduleReferencesToCheck);
+    Collection<ModuleReference> identifyNewModuleReferences(final Collection<ModuleReference> moduleReferencesToCheck);
 }
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java
index f70e218373..ef701bc7dc 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java
@@ -20,16 +20,10 @@
 
 package org.onap.cps.spi.repository;
 
-import java.util.Collection;
 import org.onap.cps.spi.entities.YangResourceEntity;
-import org.onap.cps.spi.model.ModuleReference;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
 @Repository
-public interface ModuleReferenceRepository extends JpaRepository<YangResourceEntity, Long>, ModuleReferenceQuery {
+public interface ModuleReferenceRepository extends JpaRepository<YangResourceEntity, Long>, ModuleReferenceQuery {}
 
-    Collection<ModuleReference> identifyNewModuleReferences(
-        final Collection<ModuleReference> moduleReferencesToCheck);
-
-}
\ No newline at end of file
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java
index 681bbcddec..48982d51ff 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java
@@ -23,8 +23,8 @@ package org.onap.cps.spi.repository;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
-import java.util.UUID;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
 import lombok.AllArgsConstructor;
@@ -41,6 +41,8 @@ public class ModuleReferenceRepositoryImpl implements ModuleReferenceQuery {
     @PersistenceContext
     private EntityManager entityManager;
 
+    private TempTableCreator tempTableCreator;
+
     @Override
     @SneakyThrows
     public Collection<ModuleReference> identifyNewModuleReferences(
@@ -50,42 +52,18 @@ public class ModuleReferenceRepositoryImpl implements ModuleReferenceQuery {
             return Collections.emptyList();
         }
 
-        final String tempTableName = "moduleReferencesToCheckTemp"
-                + UUID.randomUUID().toString().replace("-", "");
-
-        createTemporaryTable(tempTableName);
-        insertDataIntoTable(tempTableName, moduleReferencesToCheck);
-
-        return identifyNewModuleReferencesForCmHandle(tempTableName);
-    }
-
-    private void createTemporaryTable(final String tempTableName) {
-        final StringBuilder sqlStringBuilder = new StringBuilder("CREATE TEMPORARY TABLE " + tempTableName + "(");
-        sqlStringBuilder.append(" id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,");
-        sqlStringBuilder.append(" module_name varchar NOT NULL,");
-        sqlStringBuilder.append(" revision varchar NOT NULL");
-        sqlStringBuilder.append(");");
-
-        entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate();
-    }
-
-    private void insertDataIntoTable(final String tempTableName, final Collection<ModuleReference> moduleReferences) {
-        final StringBuilder sqlStringBuilder = new StringBuilder("INSERT INTO  " + tempTableName);
-        sqlStringBuilder.append(" (module_name, revision) ");
-        sqlStringBuilder.append(" VALUES ");
-
-        for (final ModuleReference moduleReference : moduleReferences) {
-            sqlStringBuilder.append("('");
-            sqlStringBuilder.append(moduleReference.getModuleName());
-            sqlStringBuilder.append("', '");
-            sqlStringBuilder.append(moduleReference.getRevision());
-            sqlStringBuilder.append("'),");
+        final Collection<List<String>> sqlData = new HashSet<>(moduleReferencesToCheck.size());
+        for (final ModuleReference moduleReference : moduleReferencesToCheck) {
+            final List<String> row = new ArrayList<>(2);
+            row.add(moduleReference.getModuleName());
+            row.add(moduleReference.getRevision());
+            sqlData.add(row);
         }
 
-        // replace last ',' with ';'
-        sqlStringBuilder.replace(sqlStringBuilder.length() - 1, sqlStringBuilder.length(), ";");
+        final String tempTableName = tempTableCreator.createTemporaryTable(
+            "moduleReferencesToCheckTemp", sqlData, "module_name", "revision");
 
-        entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate();
+        return identifyNewModuleReferencesForCmHandle(tempTableName);
     }
 
     private Collection<ModuleReference> identifyNewModuleReferencesForCmHandle(final String tempTableName) {
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java
new file mode 100644
index 0000000000..8cad9f5e4c
--- /dev/null
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java
@@ -0,0 +1,97 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2022 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.spi.repository;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+
+@Slf4j
+@Transactional
+@AllArgsConstructor
+@Component
+public class TempTableCreator {
+
+    @PersistenceContext
+    private EntityManager entityManager;
+
+    /**
+     * Create a uniquely named temporary table.
+     *
+     * @param prefix      prefix for the table name (so you can recognize it)
+     * @param sqlData     data to insert (strings only) the inner List present a row of data
+     * @param columnNames column names (in same order as data in rows in sqlData)
+     * @return a unique temporary table name with given prefix
+     */
+    public String createTemporaryTable(final String prefix,
+                                       final Collection<List<String>> sqlData,
+                                       final String... columnNames) {
+        final String tempTableName = prefix + UUID.randomUUID().toString().replace("-", "");
+        final StringBuilder sqlStringBuilder = new StringBuilder("CREATE TEMPORARY TABLE ");
+        sqlStringBuilder.append(tempTableName);
+        defineColumns(sqlStringBuilder, columnNames);
+        insertData(sqlStringBuilder, tempTableName, columnNames, sqlData);
+        entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate();
+        return tempTableName;
+    }
+
+    private static void defineColumns(final StringBuilder sqlStringBuilder, final String[] columnNames) {
+        sqlStringBuilder.append('(');
+        final Iterator<String> it = Arrays.stream(columnNames).iterator();
+        while (it.hasNext()) {
+            final String columnName = it.next();
+            sqlStringBuilder.append(" ");
+            sqlStringBuilder.append(columnName);
+            sqlStringBuilder.append(" varchar NOT NULL");
+            if (it.hasNext()) {
+                sqlStringBuilder.append(",");
+            }
+        }
+        sqlStringBuilder.append(");");
+    }
+
+    private static void insertData(final StringBuilder sqlStringBuilder,
+                                   final String tempTableName,
+                                   final String[] columnNames,
+                                   final Collection<List<String>> sqlData) {
+        final Collection<String> sqlInserts = new HashSet<>(sqlData.size());
+        for (final Collection<String> row : sqlData) {
+            sqlInserts.add("('" + String.join("','", row) + "')");
+        }
+        sqlStringBuilder.append("INSERT INTO ");
+        sqlStringBuilder.append(tempTableName);
+        sqlStringBuilder.append(" (");
+        sqlStringBuilder.append(String.join(",", columnNames));
+        sqlStringBuilder.append(") VALUES ");
+        sqlStringBuilder.append(String.join(",", sqlInserts));
+        sqlStringBuilder.append(";");
+    }
+
+}
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy
index bcb080726a..4c67f7e972 100644
--- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy
+++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy
@@ -230,8 +230,9 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase
     def 'Identifying new module references where #scenario'() {
         when: 'identifyNewModuleReferences is called'
             def result = objectUnderTest.identifyNewModuleReferences(moduleReferences)
-        then: 'the correct module reference collection is returned'
-            assert result == expectedResult
+        then: 'the correct module references are returned'
+            assert result.size() == expectedResult.size()
+            assert result.containsAll(expectedResult)
         where: 'the following data is used'
             scenario                              | moduleReferences                                                                                  || expectedResult
             'new module references exist'         | toModuleReference([['some module 1' : 'some revision 1'], ['some module 2' : 'some revision 2']]) || toModuleReference([['some module 1' : 'some revision 1'], ['some module 2' : 'some revision 2']])
@@ -304,7 +305,7 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase
         def moduleReferences = [].withDefault { [:] }
         moduleReferenceAsMap.forEach(property ->
             property.forEach((moduleName, revision) -> {
-                moduleReferences.add(new ModuleReference('moduleName' : moduleName, 'revision' : revision))
+                moduleReferences.add(new ModuleReference(moduleName, revision))
             }))
         return moduleReferences
     }
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsModuleReferenceRepositoryPerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsModuleReferenceRepositoryPerfTest.groovy
new file mode 100644
index 0000000000..8da6c3a0ba
--- /dev/null
+++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsModuleReferenceRepositoryPerfTest.groovy
@@ -0,0 +1,103 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2022 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.spi.performance
+
+import org.onap.cps.spi.CpsModulePersistenceService
+import org.onap.cps.spi.entities.SchemaSetEntity
+import org.onap.cps.spi.impl.CpsPersistenceSpecBase
+import org.onap.cps.spi.model.ModuleReference
+import org.onap.cps.spi.repository.DataspaceRepository
+import org.onap.cps.spi.repository.ModuleReferenceRepository
+import org.onap.cps.spi.repository.SchemaSetRepository
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.test.context.jdbc.Sql
+import org.springframework.util.StopWatch
+
+import java.util.concurrent.ThreadLocalRandom
+
+class CpsModuleReferenceRepositoryPerfTest extends CpsPersistenceSpecBase {
+
+    static final String PERF_TEST_DATA = '/data/perf-test.sql'
+
+    def NEW_RESOURCE_CONTENT = 'module stores {\n' +
+        '    yang-version 1.1;\n' +
+        '    namespace "org:onap:ccsdk:sample";\n' +
+        '\n' +
+        '    prefix book-store;\n' +
+        '\n' +
+        '    revision "2020-09-15" {\n' +
+        '        description\n' +
+        '        "Sample Model";\n' +
+        '    }' +
+        '}'
+
+    @Autowired
+    CpsModulePersistenceService objectUnderTest
+
+    @Autowired
+    DataspaceRepository dataspaceRepository
+
+    @Autowired
+    SchemaSetRepository schemaSetRepository
+
+    @Autowired
+    ModuleReferenceRepository moduleReferenceRepository
+
+    @Sql([CLEAR_DATA, PERF_TEST_DATA])
+    def 'Store new schema set with many modules'() {
+        when: 'a new schema set with 200 modules is stored'
+            def newYangResourcesNameToContentMap = [:]
+            (1..200).each {
+                def year = 2000 + it
+                def resourceName = "module${it}".toString()
+                def moduleName = "stores${it}"
+                def content = NEW_RESOURCE_CONTENT.replace('2020',String.valueOf(year)).replace('stores',moduleName)
+                newYangResourcesNameToContentMap.put(resourceName, content)
+            }
+            objectUnderTest.storeSchemaSet('PERF-DATASPACE', 'perfSchemaSet', newYangResourcesNameToContentMap)
+        then: 'the schema set is persisted correctly'
+            def dataspaceEntity = dataspaceRepository.getByName('PERF-DATASPACE')
+            SchemaSetEntity result = schemaSetRepository.getByDataspaceAndName(dataspaceEntity, 'perfSchemaSet')
+            result.yangResources.size() == 200
+        and: 'identification of new module resources is fast enough (1,000 executions less then 5,000 milliseconds)'
+            def stopWatch = new StopWatch()
+            1000.times() {
+                def moduleReferencesToCheck = createModuleReferencesWithRandomMatchingExistingModuleReferences()
+                stopWatch.start()
+                def newModuleReferences = moduleReferenceRepository.identifyNewModuleReferences(moduleReferencesToCheck)
+                stopWatch.stop()
+                assert newModuleReferences.size() > 0 && newModuleReferences.size() < 300
+            }
+            assert stopWatch.getTotalTimeMillis() < 5000
+    }
+
+    def createModuleReferencesWithRandomMatchingExistingModuleReferences() {
+        def moduleReferences = []
+        (1..250).each {
+            def randomNumber = ThreadLocalRandom.current().nextInt(1, 300)
+            def year = 2000 + randomNumber
+            def moduleName = "stores${randomNumber}"
+            moduleReferences.add(new ModuleReference(moduleName, "${year}-09-15"))
+        }
+        return moduleReferences
+    }
+
+}
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy
index 33e83f1013..265c5fc5fd 100644
--- a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy
+++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy
@@ -47,26 +47,25 @@ class CpsToDataNodePerfTest extends CpsPersistenceSpecBase {
     static def ALLOWED_SETUP_TIME_MS = TimeUnit.SECONDS.toMillis(10)
     static def ALLOWED_READ_TIME_AL_NODES_MS = 500
 
-    def readStopWatch = new StopWatch()
+    def stopWatch = new StopWatch()
 
     @Sql([CLEAR_DATA, PERF_TEST_DATA])
     def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() {
         given: 'a node with a large number of descendants is created'
-            def setupStopWatch = new StopWatch()
-            setupStopWatch.start()
+            stopWatch.start()
             createLineage()
-            setupStopWatch.stop()
-            def setupDurationInMillis = setupStopWatch.getTotalTimeMillis()
+            stopWatch.stop()
+            def setupDurationInMillis = stopWatch.getTotalTimeMillis()
         and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds'
             assert setupDurationInMillis < ALLOWED_SETUP_TIME_MS
     }
 
     def 'Get data node with many descendants by xpath #scenario'() {
         when: 'get parent is executed with all descendants'
-            readStopWatch.start()
+            stopWatch.start()
             def result = objectUnderTest.getDataNode('PERF-DATASPACE', 'PERF-ANCHOR', xpath, INCLUDE_ALL_DESCENDANTS)
-            readStopWatch.stop()
-            def readDurationInMillis = readStopWatch.getTotalTimeMillis()
+            stopWatch.stop()
+            def readDurationInMillis = stopWatch.getTotalTimeMillis()
         then: 'read duration is under 500 milliseconds'
             assert readDurationInMillis < ALLOWED_READ_TIME_AL_NODES_MS
         and: 'data node is returned with all the descendants populated'
@@ -79,10 +78,10 @@ class CpsToDataNodePerfTest extends CpsPersistenceSpecBase {
 
     def 'Query parent data node with many descendants by cps-path'() {
         when: 'query is executed with all descendants'
-            readStopWatch.start()
+            stopWatch.start()
             def result = objectUnderTest.queryDataNodes('PERF-DATASPACE', 'PERF-ANCHOR', '//perf-parent-1' , INCLUDE_ALL_DESCENDANTS)
-            readStopWatch.stop()
-            def readDurationInMillis = readStopWatch.getTotalTimeMillis()
+            stopWatch.stop()
+            def readDurationInMillis = stopWatch.getTotalTimeMillis()
         then: 'read duration is under 500 milliseconds'
             assert readDurationInMillis < ALLOWED_READ_TIME_AL_NODES_MS
         and: 'data node is returned with all the descendants populated'
@@ -91,10 +90,10 @@ class CpsToDataNodePerfTest extends CpsPersistenceSpecBase {
 
     def 'Query many descendants by cps-path with #scenario'() {
         when: 'query is executed with all descendants'
-            readStopWatch.start()
+            stopWatch.start()
             def result = objectUnderTest.queryDataNodes('PERF-DATASPACE', 'PERF-ANCHOR',  '//perf-test-grand-child-1', descendantsOption)
-            readStopWatch.stop()
-            def readDurationInMillis = readStopWatch.getTotalTimeMillis()
+            stopWatch.stop()
+            def readDurationInMillis = stopWatch.getTotalTimeMillis()
         then: 'read duration is under 500 milliseconds'
             assert readDurationInMillis < alowedDuration
         and: 'data node is returned with all the descendants populated'