From e0643ab5130dde375c229989e216341e623a9c55 Mon Sep 17 00:00:00 2001 From: JosephKeenan Date: Tue, 15 Feb 2022 18:23:05 +0000 Subject: CPS-887 Decreasing performance of cmHandle registration - Creating temporary table module references from the node - Query returning new moduleReferences for node to be added to CPS - Modified registration for node to use new implementation (incl. tests) - Refactored some interface to use collection instead of list - Modified SQL statements to use String.format() - Fixed tests Issue-ID: CPS-887 Signed-off-by: JosephKeenan Change-Id: I58b17d29e8b9d6f7a644d33bf6cf8bf46f892580 --- .../spi/impl/CpsModulePersistenceServiceImpl.java | 44 ++++----- .../cps/spi/repository/ModuleReferenceQuery.java | 34 +++++++ .../spi/repository/ModuleReferenceRepository.java | 36 +++++++ .../repository/ModuleReferenceRepositoryImpl.java | 107 +++++++++++++++++++++ .../cps/spi/repository/YangResourceRepository.java | 2 +- 5 files changed, 199 insertions(+), 24 deletions(-) create mode 100644 cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java create mode 100644 cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java create mode 100644 cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java (limited to 'cps-ri/src/main/java') diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsModulePersistenceServiceImpl.java index 86d5de6d0f..ec720b8a96 100755 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsModulePersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsModulePersistenceServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020 Nordix Foundation + * Copyright (C) 2020-2022 Nordix Foundation * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * ================================================================================ @@ -40,6 +40,7 @@ import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.transaction.Transactional; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; @@ -53,9 +54,8 @@ import org.onap.cps.spi.exceptions.AlreadyDefinedException; import org.onap.cps.spi.exceptions.DuplicatedYangResourceException; import org.onap.cps.spi.exceptions.ModelValidationException; import org.onap.cps.spi.model.ModuleReference; -import org.onap.cps.spi.repository.AnchorRepository; import org.onap.cps.spi.repository.DataspaceRepository; -import org.onap.cps.spi.repository.FragmentRepository; +import org.onap.cps.spi.repository.ModuleReferenceRepository; import org.onap.cps.spi.repository.SchemaSetRepository; import org.onap.cps.spi.repository.YangResourceRepository; import org.opendaylight.yangtools.yang.common.Revision; @@ -63,15 +63,14 @@ import org.opendaylight.yangtools.yang.model.parser.api.YangSyntaxErrorException import org.opendaylight.yangtools.yang.model.repo.api.RevisionSourceIdentifier; import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.YangModelDependencyInfo; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.retry.annotation.Backoff; import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Component; - -@Component @Slf4j +@Component +@AllArgsConstructor public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceService { private static final String YANG_RESOURCE_CHECKSUM_CONSTRAINT_NAME = "yang_resource_checksum_key"; @@ -79,24 +78,16 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ private static final Pattern RFC6020_RECOMMENDED_FILENAME_PATTERN = Pattern .compile("([\\w-]+)@(\\d{4}-\\d{2}-\\d{2})(?:\\.yang)?", Pattern.CASE_INSENSITIVE); - @Autowired private YangResourceRepository yangResourceRepository; - @Autowired private SchemaSetRepository schemaSetRepository; - @Autowired private DataspaceRepository dataspaceRepository; - @Autowired - private AnchorRepository anchorRepository; - - @Autowired - private FragmentRepository fragmentRepository; - - @Autowired private CpsAdminPersistenceService cpsAdminPersistenceService; + private ModuleReferenceRepository moduleReferenceRepository; + @Override public Map getYangSchemaResources(final String dataspaceName, final String schemaSetName) { final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName); @@ -137,9 +128,9 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Retryable(value = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) public void storeSchemaSet(final String dataspaceName, final String schemaSetName, - final Map yangResourcesNameToContentMap) { + final Map moduleReferenceNameToContentMap) { final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName); - final var yangResourceEntities = synchronizeYangResources(yangResourcesNameToContentMap); + final var yangResourceEntities = synchronizeYangResources(moduleReferenceNameToContentMap); final var schemaSetEntity = new SchemaSetEntity(); schemaSetEntity.setName(schemaSetName); schemaSetEntity.setDataspace(dataspaceEntity); @@ -158,9 +149,9 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ @Retryable(value = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) public void storeSchemaSetFromModules(final String dataspaceName, final String schemaSetName, - final Map newYangResourcesModuleNameToContentMap, - final List moduleReferences) { - storeSchemaSet(dataspaceName, schemaSetName, newYangResourcesModuleNameToContentMap); + final Map newModuleNameToContentMap, + final Collection moduleReferences) { + storeSchemaSet(dataspaceName, schemaSetName, newModuleNameToContentMap); final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName); final var schemaSetEntity = schemaSetRepository.getByDataspaceAndName(dataspaceEntity, schemaSetName); @@ -186,8 +177,15 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ yangResourceRepository.deleteOrphans(); } - private Set synchronizeYangResources(final Map yangResourcesNameToContentMap) { - final Map checksumToEntityMap = yangResourcesNameToContentMap.entrySet().stream() + @Override + public Collection identifyNewModuleReferences( + final Collection moduleReferencesToCheck) { + return moduleReferenceRepository.identifyNewModuleReferences(moduleReferencesToCheck); + } + + private Set synchronizeYangResources( + final Map moduleReferenceNameToContentMap) { + final Map checksumToEntityMap = moduleReferenceNameToContentMap.entrySet().stream() .map(entry -> { final String checksum = DigestUtils.sha256Hex(entry.getValue().getBytes(StandardCharsets.UTF_8)); final Map moduleNameAndRevisionMap = createModuleNameAndRevisionMap(entry.getKey(), diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java new file mode 100644 index 0000000000..6551937e10 --- /dev/null +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceQuery.java @@ -0,0 +1,34 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2022 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.spi.repository; + +import java.util.Collection; +import org.onap.cps.spi.model.ModuleReference; + +/** + * This interface is used in conjunction with {@link ModuleReferenceRepository} to create native sql queries. + */ +public interface ModuleReferenceQuery { + + Collection identifyNewModuleReferences( + final Collection moduleReferencesToCheck); + +} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java new file mode 100644 index 0000000000..ce2bfe7847 --- /dev/null +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepository.java @@ -0,0 +1,36 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2022 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.spi.repository; + +import java.util.Collection; +import org.onap.cps.spi.entities.YangResourceEntity; +import org.onap.cps.spi.model.ModuleReference; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface ModuleReferenceRepository extends + JpaRepository, ModuleReferenceQuery { + + Collection identifyNewModuleReferences( + final Collection moduleReferencesToCheck); + +} \ No newline at end of file diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java new file mode 100644 index 0000000000..f4078ffec7 --- /dev/null +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/ModuleReferenceRepositoryImpl.java @@ -0,0 +1,107 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2022 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.spi.repository; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.onap.cps.spi.model.ModuleReference; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Transactional +public class ModuleReferenceRepositoryImpl implements ModuleReferenceQuery { + + @PersistenceContext + private EntityManager entityManager; + + @Override + @SneakyThrows + public Collection identifyNewModuleReferences( + final Collection moduleReferencesToCheck) { + + if (moduleReferencesToCheck == null || moduleReferencesToCheck.isEmpty()) { + return Collections.EMPTY_LIST; + } + + final String tempTableName = "moduleReferencesToCheckTemp" + + UUID.randomUUID().toString().replaceAll("-", ""); + + createTemporaryTable(tempTableName); + insertDataIntoTable(tempTableName, moduleReferencesToCheck); + + return identifyNewModuleReferencesForCmHandle(tempTableName); + } + + private void createTemporaryTable(final String tempTableName) { + final StringBuilder sqlStringBuilder = new StringBuilder("CREATE TEMPORARY TABLE " + tempTableName + "("); + sqlStringBuilder.append(" id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,"); + sqlStringBuilder.append(" module_name varchar NOT NULL,"); + sqlStringBuilder.append(" revision varchar NOT NULL"); + sqlStringBuilder.append(");"); + + entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate(); + } + + private void insertDataIntoTable(final String tempTableName, final Collection moduleReferences) { + final StringBuilder sqlStringBuilder = new StringBuilder("INSERT INTO " + tempTableName); + sqlStringBuilder.append(" (module_name, revision) "); + sqlStringBuilder.append(" VALUES "); + + for (final ModuleReference moduleReference : moduleReferences) { + sqlStringBuilder.append("('"); + sqlStringBuilder.append(moduleReference.getModuleName()); + sqlStringBuilder.append("', '"); + sqlStringBuilder.append(moduleReference.getRevision()); + sqlStringBuilder.append("'),"); + } + + // replace last ',' with ';' + sqlStringBuilder.replace(sqlStringBuilder.length() - 1, sqlStringBuilder.length(), ";"); + + entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate(); + } + + private Collection identifyNewModuleReferencesForCmHandle(final String tempTableName) { + final String sql = String.format( + "SELECT %1$s.module_name, %1$s.revision" + + " FROM %1$s LEFT JOIN yang_resource" + + " ON yang_resource.module_name=%1$s.module_name" + + " AND yang_resource.revision=%1$s.revision" + + " WHERE yang_resource.module_name IS NULL;", tempTableName); + + final List resultsAsObjects = + entityManager.createNativeQuery(sql).getResultList(); + + final List resultsAsModuleReferences = new ArrayList<>(resultsAsObjects.size()); + for (final Object[] row : resultsAsObjects) { + resultsAsModuleReferences.add(new ModuleReference((String) row[0], (String) row[1])); + } + + return resultsAsModuleReferences; + } +} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/YangResourceRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/YangResourceRepository.java index 3f5c43decc..895937b60a 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/YangResourceRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/YangResourceRepository.java @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2020 Pantheon.tech - * Modifications Copyright (C) Nordix Foundation + * Modifications Copyright (C) 2021-2022 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. -- cgit 1.2.3-korg