diff options
9 files changed, 236 insertions, 64 deletions
diff --git a/cps-path-parser/src/test/groovy/org/onap/cps/cpspath/parser/CpsPathUtilSpec.groovy b/cps-path-parser/src/test/groovy/org/onap/cps/cpspath/parser/CpsPathUtilSpec.groovy index df2e9d72c6..f1a878d63b 100644 --- a/cps-path-parser/src/test/groovy/org/onap/cps/cpspath/parser/CpsPathUtilSpec.groovy +++ b/cps-path-parser/src/test/groovy/org/onap/cps/cpspath/parser/CpsPathUtilSpec.groovy @@ -74,16 +74,16 @@ class CpsPathUtilSpec extends Specification { def 'CPS Path Processing Performance Test.'() { when: '200,000 paths are processed' - def setupStopWatch = new StopWatch() - setupStopWatch.start() + def stopWatch = new StopWatch() + stopWatch.start() (1..100000).each { CpsPathUtil.getNormalizedXpath('/long/path/to/see/if/it/adds/paring/time/significantly/parent/child[@common-leaf-name="123"]') CpsPathUtil.getNormalizedXpath('//child[@other-leaf=1]/leaf-name[text()="search"]/ancestor::parent') } - setupStopWatch.stop() + stopWatch.stop() then: 'it takes less then 10,000 milliseconds' // In CI this actually takes about 3-5 sec which is approx. 50+ parser executions per millisecond! - assert setupStopWatch.getTotalTimeMillis() < 10000 + assert stopWatch.getTotalTimeMillis() < 10000 } } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java index 5e4de7fec4..00e53aa00f 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java @@ -28,6 +28,5 @@ import org.onap.cps.spi.model.ModuleReference; */ public interface ModuleReferenceQuery { - Collection<ModuleReference> identifyNewModuleReferences( - final Collection<ModuleReference> moduleReferencesToCheck); + Collection<ModuleReference> identifyNewModuleReferences(final Collection<ModuleReference> moduleReferencesToCheck); } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java index f70e218373..ef701bc7dc 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java @@ -20,16 +20,10 @@ package org.onap.cps.spi.repository; -import java.util.Collection; import org.onap.cps.spi.entities.YangResourceEntity; -import org.onap.cps.spi.model.ModuleReference; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @Repository -public interface ModuleReferenceRepository extends JpaRepository<YangResourceEntity, Long>, ModuleReferenceQuery { +public interface ModuleReferenceRepository extends JpaRepository<YangResourceEntity, Long>, ModuleReferenceQuery {} - Collection<ModuleReference> identifyNewModuleReferences( - final Collection<ModuleReference> moduleReferencesToCheck); - -}
\ No newline at end of file diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java index 681bbcddec..48982d51ff 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java @@ -23,8 +23,8 @@ package org.onap.cps.spi.repository; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.UUID; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import lombok.AllArgsConstructor; @@ -41,6 +41,8 @@ public class ModuleReferenceRepositoryImpl implements ModuleReferenceQuery { @PersistenceContext private EntityManager entityManager; + private TempTableCreator tempTableCreator; + @Override @SneakyThrows public Collection<ModuleReference> identifyNewModuleReferences( @@ -50,42 +52,18 @@ public class ModuleReferenceRepositoryImpl implements ModuleReferenceQuery { return Collections.emptyList(); } - final String tempTableName = "moduleReferencesToCheckTemp" - + UUID.randomUUID().toString().replace("-", ""); - - createTemporaryTable(tempTableName); - insertDataIntoTable(tempTableName, moduleReferencesToCheck); - - return identifyNewModuleReferencesForCmHandle(tempTableName); - } - - private void createTemporaryTable(final String tempTableName) { - final StringBuilder sqlStringBuilder = new StringBuilder("CREATE TEMPORARY TABLE " + tempTableName + "("); - sqlStringBuilder.append(" id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,"); - sqlStringBuilder.append(" module_name varchar NOT NULL,"); - sqlStringBuilder.append(" revision varchar NOT NULL"); - sqlStringBuilder.append(");"); - - entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate(); - } - - private void insertDataIntoTable(final String tempTableName, final Collection<ModuleReference> moduleReferences) { - final StringBuilder sqlStringBuilder = new StringBuilder("INSERT INTO " + tempTableName); - sqlStringBuilder.append(" (module_name, revision) "); - sqlStringBuilder.append(" VALUES "); - - for (final ModuleReference moduleReference : moduleReferences) { - sqlStringBuilder.append("('"); - sqlStringBuilder.append(moduleReference.getModuleName()); - sqlStringBuilder.append("', '"); - sqlStringBuilder.append(moduleReference.getRevision()); - sqlStringBuilder.append("'),"); + final Collection<List<String>> sqlData = new HashSet<>(moduleReferencesToCheck.size()); + for (final ModuleReference moduleReference : moduleReferencesToCheck) { + final List<String> row = new ArrayList<>(2); + row.add(moduleReference.getModuleName()); + row.add(moduleReference.getRevision()); + sqlData.add(row); } - // replace last ',' with ';' - sqlStringBuilder.replace(sqlStringBuilder.length() - 1, sqlStringBuilder.length(), ";"); + final String tempTableName = tempTableCreator.createTemporaryTable( + "moduleReferencesToCheckTemp", sqlData, "module_name", "revision"); - entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate(); + return identifyNewModuleReferencesForCmHandle(tempTableName); } private Collection<ModuleReference> identifyNewModuleReferencesForCmHandle(final String tempTableName) { diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java new file mode 100644 index 0000000000..8cad9f5e4c --- /dev/null +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java @@ -0,0 +1,97 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2022 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.spi.repository; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Transactional +@AllArgsConstructor +@Component +public class TempTableCreator { + + @PersistenceContext + private EntityManager entityManager; + + /** + * Create a uniquely named temporary table. + * + * @param prefix prefix for the table name (so you can recognize it) + * @param sqlData data to insert (strings only) the inner List present a row of data + * @param columnNames column names (in same order as data in rows in sqlData) + * @return a unique temporary table name with given prefix + */ + public String createTemporaryTable(final String prefix, + final Collection<List<String>> sqlData, + final String... columnNames) { + final String tempTableName = prefix + UUID.randomUUID().toString().replace("-", ""); + final StringBuilder sqlStringBuilder = new StringBuilder("CREATE TEMPORARY TABLE "); + sqlStringBuilder.append(tempTableName); + defineColumns(sqlStringBuilder, columnNames); + insertData(sqlStringBuilder, tempTableName, columnNames, sqlData); + entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate(); + return tempTableName; + } + + private static void defineColumns(final StringBuilder sqlStringBuilder, final String[] columnNames) { + sqlStringBuilder.append('('); + final Iterator<String> it = Arrays.stream(columnNames).iterator(); + while (it.hasNext()) { + final String columnName = it.next(); + sqlStringBuilder.append(" "); + sqlStringBuilder.append(columnName); + sqlStringBuilder.append(" varchar NOT NULL"); + if (it.hasNext()) { + sqlStringBuilder.append(","); + } + } + sqlStringBuilder.append(");"); + } + + private static void insertData(final StringBuilder sqlStringBuilder, + final String tempTableName, + final String[] columnNames, + final Collection<List<String>> sqlData) { + final Collection<String> sqlInserts = new HashSet<>(sqlData.size()); + for (final Collection<String> row : sqlData) { + sqlInserts.add("('" + String.join("','", row) + "')"); + } + sqlStringBuilder.append("INSERT INTO "); + sqlStringBuilder.append(tempTableName); + sqlStringBuilder.append(" ("); + sqlStringBuilder.append(String.join(",", columnNames)); + sqlStringBuilder.append(") VALUES "); + sqlStringBuilder.append(String.join(",", sqlInserts)); + sqlStringBuilder.append(";"); + } + +} diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy index bcb080726a..4c67f7e972 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy @@ -230,8 +230,9 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase def 'Identifying new module references where #scenario'() { when: 'identifyNewModuleReferences is called' def result = objectUnderTest.identifyNewModuleReferences(moduleReferences) - then: 'the correct module reference collection is returned' - assert result == expectedResult + then: 'the correct module references are returned' + assert result.size() == expectedResult.size() + assert result.containsAll(expectedResult) where: 'the following data is used' scenario | moduleReferences || expectedResult 'new module references exist' | toModuleReference([['some module 1' : 'some revision 1'], ['some module 2' : 'some revision 2']]) || toModuleReference([['some module 1' : 'some revision 1'], ['some module 2' : 'some revision 2']]) @@ -304,7 +305,7 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase def moduleReferences = [].withDefault { [:] } moduleReferenceAsMap.forEach(property -> property.forEach((moduleName, revision) -> { - moduleReferences.add(new ModuleReference('moduleName' : moduleName, 'revision' : revision)) + moduleReferences.add(new ModuleReference(moduleName, revision)) })) return moduleReferences } diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsModuleReferenceRepositoryPerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsModuleReferenceRepositoryPerfTest.groovy new file mode 100644 index 0000000000..8da6c3a0ba --- /dev/null +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsModuleReferenceRepositoryPerfTest.groovy @@ -0,0 +1,103 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2022 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.spi.performance + +import org.onap.cps.spi.CpsModulePersistenceService +import org.onap.cps.spi.entities.SchemaSetEntity +import org.onap.cps.spi.impl.CpsPersistenceSpecBase +import org.onap.cps.spi.model.ModuleReference +import org.onap.cps.spi.repository.DataspaceRepository +import org.onap.cps.spi.repository.ModuleReferenceRepository +import org.onap.cps.spi.repository.SchemaSetRepository +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.test.context.jdbc.Sql +import org.springframework.util.StopWatch + +import java.util.concurrent.ThreadLocalRandom + +class CpsModuleReferenceRepositoryPerfTest extends CpsPersistenceSpecBase { + + static final String PERF_TEST_DATA = '/data/perf-test.sql' + + def NEW_RESOURCE_CONTENT = 'module stores {\n' + + ' yang-version 1.1;\n' + + ' namespace "org:onap:ccsdk:sample";\n' + + '\n' + + ' prefix book-store;\n' + + '\n' + + ' revision "2020-09-15" {\n' + + ' description\n' + + ' "Sample Model";\n' + + ' }' + + '}' + + @Autowired + CpsModulePersistenceService objectUnderTest + + @Autowired + DataspaceRepository dataspaceRepository + + @Autowired + SchemaSetRepository schemaSetRepository + + @Autowired + ModuleReferenceRepository moduleReferenceRepository + + @Sql([CLEAR_DATA, PERF_TEST_DATA]) + def 'Store new schema set with many modules'() { + when: 'a new schema set with 200 modules is stored' + def newYangResourcesNameToContentMap = [:] + (1..200).each { + def year = 2000 + it + def resourceName = "module${it}".toString() + def moduleName = "stores${it}" + def content = NEW_RESOURCE_CONTENT.replace('2020',String.valueOf(year)).replace('stores',moduleName) + newYangResourcesNameToContentMap.put(resourceName, content) + } + objectUnderTest.storeSchemaSet('PERF-DATASPACE', 'perfSchemaSet', newYangResourcesNameToContentMap) + then: 'the schema set is persisted correctly' + def dataspaceEntity = dataspaceRepository.getByName('PERF-DATASPACE') + SchemaSetEntity result = schemaSetRepository.getByDataspaceAndName(dataspaceEntity, 'perfSchemaSet') + result.yangResources.size() == 200 + and: 'identification of new module resources is fast enough (1,000 executions less then 5,000 milliseconds)' + def stopWatch = new StopWatch() + 1000.times() { + def moduleReferencesToCheck = createModuleReferencesWithRandomMatchingExistingModuleReferences() + stopWatch.start() + def newModuleReferences = moduleReferenceRepository.identifyNewModuleReferences(moduleReferencesToCheck) + stopWatch.stop() + assert newModuleReferences.size() > 0 && newModuleReferences.size() < 300 + } + assert stopWatch.getTotalTimeMillis() < 5000 + } + + def createModuleReferencesWithRandomMatchingExistingModuleReferences() { + def moduleReferences = [] + (1..250).each { + def randomNumber = ThreadLocalRandom.current().nextInt(1, 300) + def year = 2000 + randomNumber + def moduleName = "stores${randomNumber}" + moduleReferences.add(new ModuleReference(moduleName, "${year}-09-15")) + } + return moduleReferences + } + +} diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy index 33e83f1013..265c5fc5fd 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsToDataNodePerfTest.groovy @@ -47,26 +47,25 @@ class CpsToDataNodePerfTest extends CpsPersistenceSpecBase { static def ALLOWED_SETUP_TIME_MS = TimeUnit.SECONDS.toMillis(10) static def ALLOWED_READ_TIME_AL_NODES_MS = 500 - def readStopWatch = new StopWatch() + def stopWatch = new StopWatch() @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() { given: 'a node with a large number of descendants is created' - def setupStopWatch = new StopWatch() - setupStopWatch.start() + stopWatch.start() createLineage() - setupStopWatch.stop() - def setupDurationInMillis = setupStopWatch.getTotalTimeMillis() + stopWatch.stop() + def setupDurationInMillis = stopWatch.getTotalTimeMillis() and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' assert setupDurationInMillis < ALLOWED_SETUP_TIME_MS } def 'Get data node with many descendants by xpath #scenario'() { when: 'get parent is executed with all descendants' - readStopWatch.start() + stopWatch.start() def result = objectUnderTest.getDataNode('PERF-DATASPACE', 'PERF-ANCHOR', xpath, INCLUDE_ALL_DESCENDANTS) - readStopWatch.stop() - def readDurationInMillis = readStopWatch.getTotalTimeMillis() + stopWatch.stop() + def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'read duration is under 500 milliseconds' assert readDurationInMillis < ALLOWED_READ_TIME_AL_NODES_MS and: 'data node is returned with all the descendants populated' @@ -79,10 +78,10 @@ class CpsToDataNodePerfTest extends CpsPersistenceSpecBase { def 'Query parent data node with many descendants by cps-path'() { when: 'query is executed with all descendants' - readStopWatch.start() + stopWatch.start() def result = objectUnderTest.queryDataNodes('PERF-DATASPACE', 'PERF-ANCHOR', '//perf-parent-1' , INCLUDE_ALL_DESCENDANTS) - readStopWatch.stop() - def readDurationInMillis = readStopWatch.getTotalTimeMillis() + stopWatch.stop() + def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'read duration is under 500 milliseconds' assert readDurationInMillis < ALLOWED_READ_TIME_AL_NODES_MS and: 'data node is returned with all the descendants populated' @@ -91,10 +90,10 @@ class CpsToDataNodePerfTest extends CpsPersistenceSpecBase { def 'Query many descendants by cps-path with #scenario'() { when: 'query is executed with all descendants' - readStopWatch.start() + stopWatch.start() def result = objectUnderTest.queryDataNodes('PERF-DATASPACE', 'PERF-ANCHOR', '//perf-test-grand-child-1', descendantsOption) - readStopWatch.stop() - def readDurationInMillis = readStopWatch.getTotalTimeMillis() + stopWatch.stop() + def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'read duration is under 500 milliseconds' assert readDurationInMillis < alowedDuration and: 'data node is returned with all the descendants populated' diff --git a/cps-service/src/main/java/org/onap/cps/spi/model/ModuleReference.java b/cps-service/src/main/java/org/onap/cps/spi/model/ModuleReference.java index 569f0a06ed..18d55d5230 100644 --- a/cps-service/src/main/java/org/onap/cps/spi/model/ModuleReference.java +++ b/cps-service/src/main/java/org/onap/cps/spi/model/ModuleReference.java @@ -49,4 +49,5 @@ public class ModuleReference implements Serializable { this.revision = revision; this.namespace = ""; } + } |