summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorToineSiebelink <toine.siebelink@est.tech>2022-08-18 13:15:22 +0100
committerToineSiebelink <toine.siebelink@est.tech>2022-08-22 13:25:26 +0100
commita6633c02eefedcf85639c7d756661f3e756c6634 (patch)
tree03880652b66e8048a89019296a3b95d220d385c9
parent06dc882a49e6754f68ff626f246cfbe3136305af (diff)
Performance Improvement: Insert Yang Resources
Insert batches of yang resources for a schema set into one operation add batch-sized tests to ensure batch-size logic is covered for all branches fixed legacy issues in changed test classes Issue-ID: CPS-1208 Issue-ID: CPS-1126 Signed-off-by: ToineSiebelink <toine.siebelink@est.tech> Change-Id: I5b2a54ed0895999079975d777ba89d589ed4688b
-rw-r--r--cps-ri/src/main/java/org/onap/cps/spi/repository/SchemaSetYangResourceRepositoryImpl.java34
-rw-r--r--cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy131
-rw-r--r--cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistenceSpecBase.groovy1
-rw-r--r--cps-ri/src/test/resources/data/clear-all.sql9
4 files changed, 104 insertions, 71 deletions
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/SchemaSetYangResourceRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/SchemaSetYangResourceRepositoryImpl.java
index 04eaa453ef..c87e15adbf 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/SchemaSetYangResourceRepositoryImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/SchemaSetYangResourceRepositoryImpl.java
@@ -1,6 +1,6 @@
/*-
* ============LICENSE_START=======================================================
- * Copyright (C) 2021 Nordix Foundation.
+ * Copyright (C) 2021-2022 Nordix Foundation.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,26 +20,40 @@
package org.onap.cps.spi.repository;
+import java.sql.PreparedStatement;
import java.util.List;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
+import org.hibernate.Session;
import org.springframework.transaction.annotation.Transactional;
+
@Transactional
public class SchemaSetYangResourceRepositoryImpl implements SchemaSetYangResourceRepository {
+ private static final int MAX_INSERT_BATCH_SIZE = 100;
+
@PersistenceContext
private EntityManager entityManager;
@Override
- public void insertSchemaSetIdYangResourceId(final Integer schemaSetId, final List<Long> yangResourceId) {
- final var query = "INSERT INTO SCHEMA_SET_YANG_RESOURCES (SCHEMA_SET_ID, YANG_RESOURCE_ID) "
- + "VALUES ( :schemaSetId, :yangResourceId)";
- yangResourceId.forEach(id ->
- entityManager.createNativeQuery(query)
- .setParameter("schemaSetId", schemaSetId)
- .setParameter("yangResourceId", id)
- .executeUpdate()
- );
+ public void insertSchemaSetIdYangResourceId(final Integer schemaSetId, final List<Long> yangResourceIds) {
+ final Session session = entityManager.unwrap(Session.class);
+ session.doWork(connection -> {
+ try (PreparedStatement preparedStatement = connection.prepareStatement(
+ "INSERT INTO SCHEMA_SET_YANG_RESOURCES (SCHEMA_SET_ID, YANG_RESOURCE_ID) VALUES ( ?, ?)")) {
+ int sqlQueryCount = 1;
+ for (final long yangResourceId : yangResourceIds) {
+ preparedStatement.setInt(1, schemaSetId);
+ preparedStatement.setLong(2, yangResourceId);
+ preparedStatement.addBatch();
+ if (sqlQueryCount % MAX_INSERT_BATCH_SIZE == 0 || sqlQueryCount == yangResourceIds.size()) {
+ preparedStatement.executeBatch();
+ }
+ sqlQueryCount++;
+ }
+ }
+ });
}
}
+
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy
index 3249d51ad1..eac28873e7 100644
--- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy
+++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsModulePersistenceServiceIntegrationSpec.groovy
@@ -30,6 +30,7 @@ import org.onap.cps.spi.model.ModuleDefinition
import org.onap.cps.spi.model.ModuleReference
import org.onap.cps.spi.repository.AnchorRepository
import org.onap.cps.spi.repository.SchemaSetRepository
+import org.onap.cps.spi.repository.SchemaSetYangResourceRepositoryImpl
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.test.context.jdbc.Sql
@@ -47,13 +48,13 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase
@Autowired
CpsAdminPersistenceService cpsAdminPersistenceService
- static final String SET_DATA = '/data/schemaset.sql'
- static final String EXISTING_SCHEMA_SET_NAME = SCHEMA_SET_NAME1
- static final String SCHEMA_SET_NAME_NO_ANCHORS = 'SCHEMA-SET-100'
- static final String SCHEMA_SET_NAME_NEW = 'SCHEMA-SET-NEW'
+ final static String SET_DATA = '/data/schemaset.sql'
- static final String NEW_RESOURCE_NAME = 'some new resource'
- static final String NEW_RESOURCE_CONTENT = 'module stores {\n' +
+ def static EXISTING_SCHEMA_SET_NAME = SCHEMA_SET_NAME1
+ def SCHEMA_SET_NAME_NO_ANCHORS = 'SCHEMA-SET-100'
+ def NEW_SCHEMA_SET_NAME = 'SCHEMA-SET-NEW'
+ def NEW_RESOURCE_NAME = 'some new resource'
+ def NEW_RESOURCE_CONTENT = 'module stores {\n' +
' yang-version 1.1;\n' +
' namespace "org:onap:ccsdk:sample";\n' +
'\n' +
@@ -64,12 +65,9 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase
' "Sample Model";\n' +
' }' +
'}'
- static final String NEW_RESOURCE_CHECKSUM = 'b13faef573ed1374139d02c40d8ce09c80ea1dc70e63e464c1ed61568d48d539'
- static final String NEW_RESOURCE_MODULE_NAME = 'stores'
- static final String NEW_RESOURCE_REVISION = '2020-09-15'
- static final ModuleReference newModuleReference = ModuleReference.builder().moduleName(NEW_RESOURCE_MODULE_NAME)
- .revision(NEW_RESOURCE_REVISION).build()
-
+ def NEW_RESOURCE_CHECKSUM = 'b13faef573ed1374139d02c40d8ce09c80ea1dc70e63e464c1ed61568d48d539'
+ def NEW_RESOURCE_MODULE_NAME = 'stores'
+ def NEW_RESOURCE_REVISION = '2020-09-15'
def newYangResourcesNameToContentMap = [(NEW_RESOURCE_NAME):NEW_RESOURCE_CONTENT]
def dataspaceEntity
@@ -91,37 +89,62 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase
}
@Sql([CLEAR_DATA, SET_DATA])
- def 'Store new schema set.'() {
- when: 'a new schemaset is stored'
- objectUnderTest.storeSchemaSet(DATASPACE_NAME, SCHEMA_SET_NAME_NEW, newYangResourcesNameToContentMap)
+ def 'Store new schema set with one module'() {
+ when: 'a new schema set with one module is stored'
+ objectUnderTest.storeSchemaSet(DATASPACE_NAME, NEW_SCHEMA_SET_NAME, newYangResourcesNameToContentMap)
then: 'the schema set is persisted correctly'
- assertSchemaSetPersisted(DATASPACE_NAME, SCHEMA_SET_NAME_NEW, NEW_RESOURCE_NAME,
- NEW_RESOURCE_CONTENT, NEW_RESOURCE_CHECKSUM, NEW_RESOURCE_MODULE_NAME, NEW_RESOURCE_REVISION)
+ assertSchemaSetWithOneModuleIsPersistedCorrectly(NEW_RESOURCE_NAME,
+ NEW_RESOURCE_MODULE_NAME, NEW_RESOURCE_REVISION, NEW_RESOURCE_CONTENT, NEW_RESOURCE_CHECKSUM)
+ }
+
+ @Sql([CLEAR_DATA, SET_DATA])
+ def 'Store new schema set with multiple modules.'() {
+ given: 'a new schema set with #numberOfModules modules'
+ def numberOfModules = 2
+ String schemaSetName = "NewSchemaWith${numberOfModules}Modules"
+ def newYangResourcesNameToContentMap = [:]
+ (1..numberOfModules).each {
+ def uniqueRevision = String.valueOf(2000 + it) + '-01-01'
+ def uniqueContent = NEW_RESOURCE_CONTENT.replace(NEW_RESOURCE_REVISION, uniqueRevision)
+ newYangResourcesNameToContentMap.put(uniqueRevision, uniqueContent)
+ }
+ when: 'the new schema set is stored'
+ objectUnderTest.storeSchemaSet(DATASPACE_NAME, schemaSetName, newYangResourcesNameToContentMap)
+ then: 'the correct number of modules are persisted'
+ assert getYangResourceEntities(schemaSetName).size() == numberOfModules
}
@Sql([CLEAR_DATA, SET_DATA])
def 'Store and retrieve new schema set from new modules and existing modules.'() {
- given: 'map of new modules, a list of existing modules, module reference'
- def mapOfNewModules = [newModule1: 'module newmodule { yang-version 1.1; revision "2021-10-12" { } }']
- def moduleReferenceForExistingModule = new ModuleReference("test","2021-10-12")
- def listOfExistingModulesModuleReference = [moduleReferenceForExistingModule]
- def mapOfExistingModule = [test: 'module test { yang-version 1.1; revision "2021-10-12" { } }']
- objectUnderTest.storeSchemaSet(DATASPACE_NAME, "someSchemaSetName", mapOfExistingModule)
+ given: 'a new module'
+ def mapOfNewModules = [newModule: 'module newmodule { yang-version 1.1; revision "2022-08-19" { } }']
+ and: 'there are more existing modules in the db than the batch size (100)'
+ def listOfExistingModulesModuleReference = []
+ def mapOfExistingModule = [:]
+ def numberOfModules = 1 + SchemaSetYangResourceRepositoryImpl.MAX_INSERT_BATCH_SIZE
+ (1..numberOfModules).each {
+ def uniqueRevision = String.valueOf(2000 + it) + '-01-01'
+ def uniqueContent = "module test { yang-version 1.1; revision \"${uniqueRevision}\" { } }".toString()
+ mapOfNewModules.put( 'test' + it, uniqueContent)
+ listOfExistingModulesModuleReference.add(new ModuleReference('test',uniqueRevision))
+ }
+ objectUnderTest.storeSchemaSet(DATASPACE_NAME, 'existing schema set ', mapOfExistingModule)
when: 'a new schema set is created from these new modules and existing modules'
- objectUnderTest.storeSchemaSetFromModules(DATASPACE_NAME, "newSchemaSetName" , mapOfNewModules, listOfExistingModulesModuleReference)
+ objectUnderTest.storeSchemaSetFromModules(DATASPACE_NAME, NEW_SCHEMA_SET_NAME , mapOfNewModules, listOfExistingModulesModuleReference)
then: 'the schema set can be retrieved'
- def expectedYangResourcesMapAfterSchemaSetHasBeenCreated = mapOfNewModules + mapOfExistingModule
def actualYangResourcesMapAfterSchemaSetHasBeenCreated =
- objectUnderTest.getYangSchemaResources(DATASPACE_NAME, "newSchemaSetName")
- actualYangResourcesMapAfterSchemaSetHasBeenCreated == expectedYangResourcesMapAfterSchemaSetHasBeenCreated
+ objectUnderTest.getYangSchemaResources(DATASPACE_NAME, NEW_SCHEMA_SET_NAME)
+ and: 'it has all the new and existing modules'
+ def expectedYangResourcesMapAfterSchemaSetHasBeenCreated = mapOfNewModules + mapOfExistingModule
+ assert actualYangResourcesMapAfterSchemaSetHasBeenCreated == expectedYangResourcesMapAfterSchemaSetHasBeenCreated
}
@Sql([CLEAR_DATA, SET_DATA])
def 'Retrieving schema set (resources) by anchor.'() {
given: 'a new schema set is stored'
- objectUnderTest.storeSchemaSet(DATASPACE_NAME, SCHEMA_SET_NAME_NEW, newYangResourcesNameToContentMap)
+ objectUnderTest.storeSchemaSet(DATASPACE_NAME, NEW_SCHEMA_SET_NAME, newYangResourcesNameToContentMap)
and: 'an anchor is created with that schema set'
- cpsAdminPersistenceService.createAnchor(DATASPACE_NAME, SCHEMA_SET_NAME_NEW, ANCHOR_NAME1)
+ cpsAdminPersistenceService.createAnchor(DATASPACE_NAME, NEW_SCHEMA_SET_NAME, ANCHOR_NAME1)
when: 'the schema set resources for that anchor is retrieved'
def result = objectUnderTest.getYangSchemaSetResources(DATASPACE_NAME, ANCHOR_NAME1)
then: 'the correct resources are returned'
@@ -157,15 +180,13 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase
def existingResourceContent = 'module test { yang-version 1.1; revision "2020-09-15"; }'
def newYangResourcesNameToContentMap = [(NEW_RESOURCE_NAME):existingResourceContent]
when: 'the schema set with duplicate resource is stored'
- objectUnderTest.storeSchemaSet(DATASPACE_NAME, SCHEMA_SET_NAME_NEW, newYangResourcesNameToContentMap)
- then: 'the schema persisted (re)uses the existing name and has the same checksum'
- def existingResourceName = 'module1@2020-02-02.yang'
+ objectUnderTest.storeSchemaSet(DATASPACE_NAME, NEW_SCHEMA_SET_NAME, newYangResourcesNameToContentMap)
+ then: 'the schema persisted has the new name and has the same checksum'
def existingResourceChecksum = 'bea1afcc3d1517e7bf8cae151b3b6bfbd46db77a81754acdcb776a50368efa0a'
- def existingResourceModelName = 'test'
+ def existingResourceModuleName = 'test'
def existingResourceRevision = '2020-09-15'
- assertSchemaSetPersisted(DATASPACE_NAME, SCHEMA_SET_NAME_NEW, existingResourceName,
- existingResourceContent, existingResourceChecksum,
- existingResourceModelName, existingResourceRevision)
+ assertSchemaSetWithOneModuleIsPersistedCorrectly(NEW_RESOURCE_NAME, existingResourceModuleName,
+ existingResourceRevision, existingResourceContent, existingResourceChecksum)
}
@Sql([CLEAR_DATA, SET_DATA])
@@ -223,31 +244,31 @@ class CpsModulePersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase
result.sort() == [new ModuleDefinition('MODULE-NAME-004', 'REVISION-004', 'CONTENT-004')]
}
- def assertSchemaSetPersisted(expectedDataspaceName,
- expectedSchemaSetName,
- expectedYangResourceName,
- expectedYangResourceContent,
- expectedYangResourceChecksum,
- expectedYangResourceModuleName,
- expectedYangResourceRevision) {
- // assert the schema set is persisted
- def schemaSetEntity = schemaSetRepository
- .findByDataspaceAndName(dataspaceEntity, expectedSchemaSetName).orElseThrow()
- assert schemaSetEntity.name == expectedSchemaSetName
- assert schemaSetEntity.dataspace.name == expectedDataspaceName
+ def assertSchemaSetWithOneModuleIsPersistedCorrectly(expectedYangResourceName,
+ expectedYangResourceModuleName,
+ expectedYangResourceRevision,
+ expectedYangResourceContent,
+ expectedYangResourceChecksum) {
// assert the attached yang resource is persisted
- def yangResourceEntities = schemaSetEntity.getYangResources()
- yangResourceEntities.size() == 1
+ def yangResourceEntities = getYangResourceEntities(NEW_SCHEMA_SET_NAME)
+ assert yangResourceEntities.size() == 1
// assert the attached yang resource content
YangResourceEntity yangResourceEntity = yangResourceEntities.iterator().next()
assert yangResourceEntity.id != null
- yangResourceEntity.fileName == expectedYangResourceName
- yangResourceEntity.content == expectedYangResourceContent
- yangResourceEntity.checksum == expectedYangResourceChecksum
- yangResourceEntity.moduleName == expectedYangResourceModuleName
- yangResourceEntity.revision == expectedYangResourceRevision
+ assert yangResourceEntity.fileName == expectedYangResourceName
+ assert yangResourceEntity.moduleName == expectedYangResourceModuleName
+ assert yangResourceEntity.revision == expectedYangResourceRevision
+ assert yangResourceEntity.content == expectedYangResourceContent
+ assert yangResourceEntity.checksum == expectedYangResourceChecksum
+ return true
+ }
+
+ def getYangResourceEntities(schemaSetName) {
+ def schemaSetEntity = schemaSetRepository
+ .findByDataspaceAndName(dataspaceEntity, schemaSetName).orElseThrow()
+ return schemaSetEntity.getYangResources()
}
def toModuleReference(moduleReferenceAsMap) {
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistenceSpecBase.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistenceSpecBase.groovy
index 8e13be2ae1..d6f10d809d 100644
--- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistenceSpecBase.groovy
+++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistenceSpecBase.groovy
@@ -61,7 +61,6 @@ class CpsPersistenceSpecBase extends Specification {
static final String CLEAR_DATA = '/data/clear-all.sql'
static final String DATASPACE_NAME = 'DATASPACE-001'
- static final String DATASPACE_NAME2 = 'DATASPACE-002'
static final String SCHEMA_SET_NAME1 = 'SCHEMA-SET-001'
static final String SCHEMA_SET_NAME2 = 'SCHEMA-SET-002'
static final String ANCHOR_NAME1 = 'ANCHOR-001'
diff --git a/cps-ri/src/test/resources/data/clear-all.sql b/cps-ri/src/test/resources/data/clear-all.sql
index 8a5e8444e7..07c8a7aab5 100644
--- a/cps-ri/src/test/resources/data/clear-all.sql
+++ b/cps-ri/src/test/resources/data/clear-all.sql
@@ -1,7 +1,7 @@
/*
============LICENSE_START=======================================================
Copyright (C) 2020-2021 Pantheon.tech
- Modifications Copyright (C) 2020 Nordix Foundation.
+ Modifications Copyright (C) 2020,2022 Nordix Foundation.
Modifications Copyright (C) 2020 Bell Canada.
================================================================================
Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,7 +23,6 @@
DELETE FROM FRAGMENT;
DELETE FROM ANCHOR;
DELETE FROM DATASPACE;
--- following tables are cleared by CASCADE constraint:
--- SCHEMA_SET
--- SCHEMA_SET_YANG_RESOURCES
-DELETE FROM YANG_RESOURCE;
+DELETE FROM YANG_RESOURCE
+-- following tables are cleared by CASCADE constraint: SCHEMA_SET, SCHEMA_SET_YANG_RESOURCES
+