diff options
19 files changed, 226 insertions, 99 deletions
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/data/DmiDataOperations.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/data/DmiDataOperations.java index 41348aea3f..1e66ff6cd2 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/data/DmiDataOperations.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/data/DmiDataOperations.java @@ -52,7 +52,7 @@ import org.onap.cps.ncmp.impl.models.DmiRequestBody; import org.onap.cps.ncmp.impl.utils.http.RestServiceUrlTemplateBuilder; import org.onap.cps.ncmp.impl.utils.http.UrlTemplateParameters; import org.onap.cps.spi.exceptions.CpsException; -import org.onap.cps.spi.exceptions.DataNodeNotFoundException; +import org.onap.cps.spi.exceptions.DataValidationException; import org.onap.cps.utils.JsonObjectMapper; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; @@ -285,7 +285,7 @@ public class DmiDataOperations { String cmHandleId = cmResourceAddress.getCmHandleReference(); try { return getYangModelCmHandle(cmHandleId); - } catch (final DataNodeNotFoundException ignored) { + } catch (final DataValidationException ignored) { cmHandleId = cmResourceAddress.resolveCmHandleReferenceToId(); return getYangModelCmHandle(cmHandleId); } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/data/DmiDataOperationsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/data/DmiDataOperationsSpec.groovy index 71054dce41..0d1cfb7c2d 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/data/DmiDataOperationsSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/data/DmiDataOperationsSpec.groovy @@ -36,6 +36,7 @@ import org.onap.cps.ncmp.impl.utils.AlternateIdMatcher import org.onap.cps.ncmp.impl.utils.http.UrlTemplateParameters import org.onap.cps.ncmp.utils.TestUtils import org.onap.cps.spi.exceptions.DataNodeNotFoundException +import org.onap.cps.spi.exceptions.DataValidationException import org.onap.cps.utils.JsonObjectMapper import org.spockframework.spring.SpringBean import org.springframework.beans.factory.annotation.Autowired @@ -214,17 +215,21 @@ class DmiDataOperationsSpec extends DmiOperationsBaseSpec { assert objectUnderTest.resolveYangModelCmHandleFromCmHandleReference(cmResourceAddress) == yangModelCmHandle } - def 'Resolving cm handle references with alternate id.'() { + def 'Resolving cm handle references with alternate id #scenario.'() { given: 'a resource with a alternate id' - def cmResourceAddress = new CmResourceAddress('some store', 'alternate-id', 'some resource') - and: 'the alternate id cannot be found in the inventory directly and that results in a data node not found exception' - mockInventoryPersistence.getYangModelCmHandle('alternate-id') >> { throw new DataNodeNotFoundException('','') } + def cmResourceAddress = new CmResourceAddress('some store', alternateId, 'some resource') + and: 'the alternate id cannot be found in the inventory directly and that results in an exception' + mockInventoryPersistence.getYangModelCmHandle(alternateId) >> { throw errorThrownDuringCmHandleIdSearch } and: 'the alternate id can be matched to a cm handle id' - alternateIdMatcher.getCmHandleId('alternate-id') >> 'cm-handle-id' + alternateIdMatcher.getCmHandleId(alternateId) >> 'cm-handle-id' and: 'that cm handle id is available in the inventory' mockInventoryPersistence.getYangModelCmHandle('cm-handle-id') >> yangModelCmHandle expect: 'resolving that cm handle id returns the cm handle' assert objectUnderTest.resolveYangModelCmHandleFromCmHandleReference(cmResourceAddress) == yangModelCmHandle + where: 'the following alternate ids are used' + scenario | alternateId | errorThrownDuringCmHandleIdSearch + 'alternate id with no special characters' | 'alternate-id' | new DataNodeNotFoundException('','') + 'alternate id with special characters' | 'alternate#id' | new DataValidationException('','') } diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java index c419a81245..ec71c30a75 100644 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java @@ -25,9 +25,9 @@ package org.onap.cps.rest.controller; import io.micrometer.core.annotation.Timed; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.CpsQueryService; @@ -126,17 +126,8 @@ public class QueryRestController implements CpsQueryApi { : (int) Math.ceil((double) totalAnchors / paginationOption.getPageSize()); } - private Map<String, List<DataNode>> groupDataNodesPerAnchor(final Collection<DataNode> dataNodes) { - final Map<String, List<DataNode>> dataNodesMapForAnchor = new HashMap<>(); - for (final DataNode dataNode : dataNodes) { - List<DataNode> dataNodesInAnchor = dataNodesMapForAnchor.get(dataNode.getAnchorName()); - if (dataNodesInAnchor == null) { - dataNodesInAnchor = new ArrayList<>(); - dataNodesMapForAnchor.put(dataNode.getAnchorName(), dataNodesInAnchor); - } - dataNodesInAnchor.add(dataNode); - } - return dataNodesMapForAnchor; + private static Map<String, List<DataNode>> groupDataNodesPerAnchor(final Collection<DataNode> dataNodes) { + return dataNodes.stream().collect(Collectors.groupingBy(DataNode::getAnchorName)); } private ResponseEntity<Object> executeNodesByDataspaceQueryAndCreateResponse(final String dataspaceName, diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java index bdbdc7cf36..cacdba93cc 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsDataPersistenceServiceImpl.java @@ -39,8 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -81,8 +79,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService private final JsonObjectMapper jsonObjectMapper; private final SessionManager sessionManager; - private static final String REG_EX_FOR_OPTIONAL_LIST_INDEX = "(\\[@.+?])?)"; - @Override public void storeDataNodes(final String dataspaceName, final String anchorName, final Collection<DataNode> dataNodes) { @@ -228,13 +224,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService final FetchDescendantsOption fetchDescendantsOption) { final AnchorEntity anchorEntity = getAnchorEntity(dataspaceName, anchorName); final CpsPathQuery cpsPathQuery = getCpsPathQuery(cpsPath); - - Collection<FragmentEntity> fragmentEntities; - fragmentEntities = fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery); - if (cpsPathQuery.hasAncestorAxis()) { - final Collection<String> ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery); - fragmentEntities = fragmentRepository.findByAnchorAndXpathIn(anchorEntity, ancestorXpaths); - } + final Collection<FragmentEntity> fragmentEntities = + fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery); return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); } @@ -246,7 +237,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService final PaginationOption paginationOption) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); final CpsPathQuery cpsPathQuery = getCpsPathQuery(cpsPath); - final List<Long> anchorIds; if (paginationOption == NO_PAGINATION) { anchorIds = Collections.emptyList(); @@ -256,17 +246,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService return Collections.emptyList(); } } - Collection<FragmentEntity> fragmentEntities = - fragmentRepository.findByDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery, anchorIds); - - if (cpsPathQuery.hasAncestorAxis()) { - final Collection<String> ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery); - if (anchorIds.isEmpty()) { - fragmentEntities = fragmentRepository.findByDataspaceAndXpathIn(dataspaceEntity, ancestorXpaths); - } else { - fragmentEntities = fragmentRepository.findByAnchorIdsAndXpathIn(anchorIds, ancestorXpaths); - } - } + final Collection<FragmentEntity> fragmentEntities = + fragmentRepository.findByDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery, anchorIds); return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); } @@ -668,21 +649,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService .collect(Collectors.toMap(FragmentEntity::getXpath, fragmentEntity -> fragmentEntity)); } - private static Set<String> processAncestorXpath(final Collection<FragmentEntity> fragmentEntities, - final CpsPathQuery cpsPathQuery) { - final Set<String> ancestorXpath = new HashSet<>(); - final Pattern pattern = - Pattern.compile("(.*/" + Pattern.quote(cpsPathQuery.getAncestorSchemaNodeIdentifier()) - + REG_EX_FOR_OPTIONAL_LIST_INDEX + "/.*"); - for (final FragmentEntity fragmentEntity : fragmentEntities) { - final Matcher matcher = pattern.matcher(fragmentEntity.getXpath()); - if (matcher.matches()) { - ancestorXpath.add(matcher.group(1)); - } - } - return ancestorXpath; - } - private static boolean isRootXpath(final String xpath) { return "/".equals(xpath) || "".equals(xpath); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java index 3368aee148..e5853abf39 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java @@ -230,8 +230,6 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ updateAllModuleReferences(allModuleReferences, schemaSetEntity.getId()); } - - @Override @Transactional public void deleteUnusedYangResourceModules() { diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java index b8bbf59c23..e35440e29a 100644 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentQueryBuilder.java @@ -30,8 +30,10 @@ import java.util.List; import java.util.Map; import java.util.Queue; import lombok.RequiredArgsConstructor; +import org.apache.commons.text.StringSubstitutor; import org.onap.cps.cpspath.parser.CpsPathPrefixType; import org.onap.cps.cpspath.parser.CpsPathQuery; +import org.onap.cps.cpspath.parser.CpsPathUtil; import org.onap.cps.ri.models.AnchorEntity; import org.onap.cps.ri.models.DataspaceEntity; import org.onap.cps.ri.models.FragmentEntity; @@ -43,6 +45,7 @@ import org.springframework.stereotype.Component; @RequiredArgsConstructor @Component public class FragmentQueryBuilder { + private static final String DESCENDANT_PATH = "//"; @PersistenceContext private EntityManager entityManager; @@ -58,9 +61,10 @@ public class FragmentQueryBuilder { final StringBuilder sqlStringBuilder = new StringBuilder(); final Map<String, Object> queryParameters = new HashMap<>(); - sqlStringBuilder.append("SELECT fragment.* FROM fragment"); + addSearchPrefix(cpsPathQuery, sqlStringBuilder); addWhereClauseForAnchor(anchorEntity, sqlStringBuilder, queryParameters); addNodeSearchConditions(cpsPathQuery, sqlStringBuilder, queryParameters, false); + addSearchSuffix(cpsPathQuery, sqlStringBuilder, queryParameters); return getQuery(sqlStringBuilder.toString(), queryParameters, FragmentEntity.class); } @@ -78,13 +82,14 @@ public class FragmentQueryBuilder { final StringBuilder sqlStringBuilder = new StringBuilder(); final Map<String, Object> queryParameters = new HashMap<>(); - sqlStringBuilder.append("SELECT fragment.* FROM fragment"); + addSearchPrefix(cpsPathQuery, sqlStringBuilder); if (anchorIdsForPagination.isEmpty()) { addWhereClauseForDataspace(dataspaceEntity, sqlStringBuilder, queryParameters); } else { addWhereClauseForAnchorIds(anchorIdsForPagination, sqlStringBuilder, queryParameters); } addNodeSearchConditions(cpsPathQuery, sqlStringBuilder, queryParameters, true); + addSearchSuffix(cpsPathQuery, sqlStringBuilder, queryParameters); return getQuery(sqlStringBuilder.toString(), queryParameters, FragmentEntity.class); } @@ -143,7 +148,8 @@ public class FragmentQueryBuilder { final Map<String, Object> queryParameters, final boolean acrossAnchors) { addAbsoluteParentXpathSearchCondition(cpsPathQuery, sqlStringBuilder, queryParameters, acrossAnchors); - addXpathSearchCondition(cpsPathQuery, sqlStringBuilder, queryParameters); + sqlStringBuilder.append(" AND "); + addXpathSearchCondition(cpsPathQuery, sqlStringBuilder, queryParameters, "baseXpath"); addLeafConditions(cpsPathQuery, sqlStringBuilder); addTextFunctionCondition(cpsPathQuery, sqlStringBuilder, queryParameters); addContainsFunctionCondition(cpsPathQuery, sqlStringBuilder, queryParameters); @@ -151,13 +157,35 @@ public class FragmentQueryBuilder { private static void addXpathSearchCondition(final CpsPathQuery cpsPathQuery, final StringBuilder sqlStringBuilder, - final Map<String, Object> queryParameters) { - sqlStringBuilder.append(" AND (xpath LIKE :escapedXpath OR " - + "(xpath LIKE :escapedXpath||'[@%]' AND xpath NOT LIKE :escapedXpath||'[@%]/%[@%]'))"); + final Map<String, Object> queryParameters, + final String parameterName) { + queryParameters.put(parameterName, escapeXpathForSqlLike(cpsPathQuery)); + final String sqlForXpathLikeContainerOrList = """ + ( + (xpath LIKE :${xpathParamName}) + OR + (xpath LIKE :${xpathParamName}||'[@%]' AND xpath NOT LIKE :${xpathParamName}||'[@%]/%[@%]') + ) + """; + sqlStringBuilder.append(substitute(sqlForXpathLikeContainerOrList, Map.of("xpathParamName", parameterName))); + } + + /** + * Returns a pattern suitable for use in an SQL LIKE expression, matching the xpath (absolute or descendant). + * For an absolute path such as "/bookstore/categories[@name='10% off']", + * the output would be "/bookstore/categories[@name='10\% off']". + * For a descendant path such as "//categories[@name='10% off']", + * the output would be "%/categories[@name='10\% off']". + * Note: percent sign '%' means match anything in SQL LIKE, while underscore '_' means match any single character. + * + * @param cpsPathQuery Cps Path Query + * @return a pattern suitable for use in an SQL LIKE expression. + */ + private static String escapeXpathForSqlLike(final CpsPathQuery cpsPathQuery) { if (CpsPathPrefixType.ABSOLUTE.equals(cpsPathQuery.getCpsPathPrefixType())) { - queryParameters.put("escapedXpath", EscapeUtils.escapeForSqlLike(cpsPathQuery.getXpathPrefix())); + return EscapeUtils.escapeForSqlLike(cpsPathQuery.getXpathPrefix()); } else { - queryParameters.put("escapedXpath", "%/" + EscapeUtils.escapeForSqlLike(cpsPathQuery.getDescendantName())); + return "%/" + EscapeUtils.escapeForSqlLike(cpsPathQuery.getDescendantName()); } } @@ -261,6 +289,55 @@ public class FragmentQueryBuilder { } } + private static void addSearchPrefix(final CpsPathQuery cpsPathQuery, final StringBuilder sqlStringBuilder) { + if (cpsPathQuery.hasAncestorAxis()) { + sqlStringBuilder.append(""" + WITH RECURSIVE ancestors AS ( + SELECT parentFragment.* FROM fragment parentFragment + WHERE parentFragment.id IN ( + SELECT parent_id FROM fragment"""); + } else { + sqlStringBuilder.append("SELECT fragment.* FROM fragment"); + } + } + + private static void addSearchSuffix(final CpsPathQuery cpsPathQuery, + final StringBuilder sqlStringBuilder, + final Map<String, Object> queryParameters) { + if (cpsPathQuery.hasAncestorAxis()) { + sqlStringBuilder.append(""" + ) + UNION + SELECT fragment.* + FROM fragment + JOIN ancestors ON ancestors.parent_id = fragment.id + ) + SELECT * FROM ancestors + WHERE"""); + + final String ancestorPath = DESCENDANT_PATH + cpsPathQuery.getAncestorSchemaNodeIdentifier(); + final CpsPathQuery ancestorCpsPathQuery = CpsPathUtil.getCpsPathQuery(ancestorPath); + addAncestorNodeSearchCondition(ancestorCpsPathQuery, sqlStringBuilder, queryParameters); + } + } + + private static void addAncestorNodeSearchCondition(final CpsPathQuery ancestorCpsPathQuery, + final StringBuilder sqlStringBuilder, + final Map<String, Object> queryParameters) { + if (ancestorCpsPathQuery.hasLeafConditions()) { + final String pathWithoutSlashes = ancestorCpsPathQuery.getNormalizedXpath().substring(2); + queryParameters.put("ancestorXpath", "%/" + EscapeUtils.escapeForSqlLike(pathWithoutSlashes)); + sqlStringBuilder.append(" xpath LIKE :ancestorXpath"); + } else { + addXpathSearchCondition(ancestorCpsPathQuery, sqlStringBuilder, queryParameters, "ancestorXpath"); + } + } + + private static <V> String substitute(final String template, final Map<String, V> valueMap) { + final StringSubstitutor stringSubstitutor = new StringSubstitutor(valueMap); + return stringSubstitutor.replace(template); + } + private static void setQueryParameters(final Query query, final Map<String, Object> queryParameters) { for (final Map.Entry<String, Object> queryParameter : queryParameters.entrySet()) { query.setParameter(queryParameter.getKey(), queryParameter.getValue()); diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java index a8c1fd2d4e..d95d322d37 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java @@ -26,7 +26,6 @@ package org.onap.cps.ri.repository; import java.util.Collection; import java.util.List; import org.onap.cps.ri.models.AnchorEntity; -import org.onap.cps.ri.models.DataspaceEntity; import org.onap.cps.ri.models.FragmentEntity; import org.onap.cps.ri.utils.EscapeUtils; import org.springframework.data.jpa.repository.JpaRepository; @@ -63,21 +62,6 @@ public interface FragmentRepository extends JpaRepository<FragmentEntity, Long>, return findListByAnchorIdAndEscapedXpath(anchorEntity.getId(), escapedXpath); } - @Query(value = "SELECT fragment.* FROM fragment JOIN anchor ON anchor.id = fragment.anchor_id " - + "WHERE dataspace_id = :dataspaceId AND xpath IN (:xpaths)", nativeQuery = true) - List<FragmentEntity> findByDataspaceIdAndXpathIn(@Param("dataspaceId") int dataspaceId, - @Param("xpaths") Collection<String> xpaths); - - default List<FragmentEntity> findByDataspaceAndXpathIn(final DataspaceEntity dataspaceEntity, - final Collection<String> xpaths) { - return findByDataspaceIdAndXpathIn(dataspaceEntity.getId(), xpaths); - } - - @Query(value = "SELECT * FROM fragment WHERE anchor_id IN (:anchorIds)" - + " AND xpath IN (:xpaths)", nativeQuery = true) - List<FragmentEntity> findByAnchorIdsAndXpathIn(@Param("anchorIds") Collection<Long> anchorIds, - @Param("xpaths") Collection<String> xpaths); - @Modifying @Query(value = "DELETE FROM fragment WHERE anchor_id IN (:anchorIds)", nativeQuery = true) void deleteByAnchorIdIn(@Param("anchorIds") Collection<Long> anchorIds); diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java index bbfe496d85..304ed288f5 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsModuleService.java @@ -188,4 +188,9 @@ public interface CpsModuleService { final Map<String, String> parentAttributes, final Map<String, String> childAttributes); + /** + * Remove any Yang Resource Modules from the DB that are no longer referenced by any schema set. + */ + void deleteUnusedYangResourceModules(); + } diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java index 4063a7f769..9f3f2cc571 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java @@ -106,7 +106,6 @@ public class CpsModuleServiceImpl implements CpsModuleService { } cpsModulePersistenceService.deleteSchemaSet(dataspaceName, schemaSetName); yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName); - cpsModulePersistenceService.deleteUnusedYangResourceModules(); } @Override @@ -119,7 +118,6 @@ public class CpsModuleServiceImpl implements CpsModuleService { .stream().map(Anchor::getName).collect(Collectors.toSet()); cpsAnchorService.deleteAnchors(dataspaceName, anchorNames); cpsModulePersistenceService.deleteSchemaSets(dataspaceName, schemaSetNames); - cpsModulePersistenceService.deleteUnusedYangResourceModules(); for (final String schemaSetName : schemaSetNames) { yangTextSchemaSourceSetCache.removeFromCache(dataspaceName, schemaSetName); } @@ -182,6 +180,11 @@ public class CpsModuleServiceImpl implements CpsModuleService { childAttributes); } + @Override + public void deleteUnusedYangResourceModules() { + cpsModulePersistenceService.deleteUnusedYangResourceModules(); + } + private boolean isCascadeDeleteProhibited(final CascadeDeleteAllowed cascadeDeleteAllowed) { return CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED == cascadeDeleteAllowed; } diff --git a/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java b/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java new file mode 100644 index 0000000000..6bd3e1f204 --- /dev/null +++ b/cps-service/src/main/java/org/onap/cps/init/DbCleaner.java @@ -0,0 +1,48 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.init; + +import java.util.concurrent.TimeUnit; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.onap.cps.api.CpsModuleService; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@RequiredArgsConstructor +@Service +public class DbCleaner { + + private final CpsModuleService cpsModuleService; + + /** + * This method will clean up the db during application start up. + * It wil run once and currently only removes unused yang resource modules. + * + */ + @Scheduled(initialDelay = 1, timeUnit = TimeUnit.SECONDS) + public void cleanDbAtStartUp() { + log.info("CPS Application started, commencing DB clean up"); + cpsModuleService.deleteUnusedYangResourceModules(); + log.info("DB clean up completed"); + } +} diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy index 1831506563..c02b06fd80 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy @@ -142,8 +142,6 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset') and: 'schema set will be removed from the cache' 1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset') - and: 'orphan yang resources are deleted' - 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() and: 'the CpsValidator is called on the dataspaceName and schemaSetName' 1 * mockCpsValidator.validateNameCharacters('my-dataspace', _) where: 'following parameters are used' @@ -161,8 +159,6 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsModulePersistenceService.deleteSchemaSet('my-dataspace', 'my-schemaset') and: 'schema set will be removed from the cache' 1 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', 'my-schemaset') - and: 'orphan yang resources are deleted' - 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() and: 'the CpsValidator is called on the dataspaceName and schemaSetName' 1 * mockCpsValidator.validateNameCharacters('my-dataspace', 'my-schemaset') } @@ -187,8 +183,6 @@ class CpsModuleServiceImplSpec extends Specification { mockCpsModulePersistenceService.deleteSchemaSets('my-dataspace', _) and: 'schema sets will be removed from the cache' 2 * mockYangTextSchemaSourceSetCache.removeFromCache('my-dataspace', _) - and: 'orphan yang resources are deleted' - 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() and: 'the CpsValidator is called on the dataspaceName' 1 * mockCpsValidator.validateNameCharacters('my-dataspace') and: 'the CpsValidator is called on the schemaSetNames' @@ -276,6 +270,13 @@ class CpsModuleServiceImplSpec extends Specification { 1 * mockCpsValidator.validateNameCharacters('some-dataspace-name', 'some-anchor-name') } + def 'Delete unused yang resource modules.'() { + when: 'deleting unused yang resource modules' + objectUnderTest.deleteUnusedYangResourceModules() + then: 'it is delegated to the module persistence service' + 1 * mockCpsModulePersistenceService.deleteUnusedYangResourceModules() + } + def getModuleReferences() { return [new ModuleReference('some module name','some revision name')] } diff --git a/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy new file mode 100644 index 0000000000..5106d29fa5 --- /dev/null +++ b/cps-service/src/test/groovy/org/onap/cps/init/DbCleanerSpec.groovy @@ -0,0 +1,38 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.init + +import org.onap.cps.api.CpsModuleService +import spock.lang.Specification + +class DbCleanerSpec extends Specification { + + def mockCpsModuleService = Mock(CpsModuleService) + + def objectUnderTest = new DbCleaner(mockCpsModuleService) + + def 'DB clean up.'() { + when: 'scheduled method is triggered' + objectUnderTest.cleanDbAtStartUp() + then: 'the unused yang resource modules are deleted' + 1 * mockCpsModuleService.deleteUnusedYangResourceModules() + } +} diff --git a/csit/plans/cps/test.properties b/csit/plans/cps/test.properties index e7b9519c2d..52e82bdb85 100644 --- a/csit/plans/cps/test.properties +++ b/csit/plans/cps/test.properties @@ -21,7 +21,7 @@ DMI_SERVICE_URL=http://$LOCAL_IP:$DMI_PORT DOCKER_REPO=nexus3.onap.org:10003 CPS_VERSION=latest -DMI_VERSION=1.5.1-SNAPSHOT-latest +DMI_VERSION=latest ADVISED_MODULES_SYNC_SLEEP_TIME_MS=2000 CMHANDLE_DATA_SYNC_SLEEP_TIME_MS=2000 diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index feb58d849d..ae34fc3606 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -142,7 +142,7 @@ services: ncmp-dmi-plugin-demo-and-csit-stub: container_name: ${NCMP_DMI_PLUGIN_DEMO_AND_CSIT_STUB_CONTAINER_NAME:-ncmp-dmi-plugin-demo-and-csit-stub} - image: ${DOCKER_REPO:-nexus3.onap.org:10003}/onap/dmi-plugin-demo-and-csit-stub:${DMI_DEMO_STUB_VERSION:-latest} + image: ${DOCKER_REPO:-nexus3.onap.org:10003}/onap/dmi-stub:${DMI_DEMO_STUB_VERSION:-latest} ports: - ${DMI_DEMO_STUB_PORT:-8784}:8092 environment: diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy index d69f6cca0c..f20e4e5cad 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy @@ -32,6 +32,8 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { def setup() { objectUnderTest = cpsDataspaceService } + def cleanup() { cpsModuleService.deleteUnusedYangResourceModules() } + def 'Dataspace CRUD operations.'() { when: 'a dataspace is created' objectUnderTest.createDataspace('newDataspace') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy index 9e51d80d9e..a50a59a3d7 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/ModuleServiceIntegrationSpec.groovy @@ -59,9 +59,9 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { def bookstoreModelFileContent = readResourceDataFile('bookstore/bookstore.yang') def bookstoreTypesFileContent = readResourceDataFile('bookstore/bookstore-types.yang') - def setup() { - objectUnderTest = cpsModuleService - } + def setup() { objectUnderTest = cpsModuleService } + + def cleanup() { objectUnderTest.deleteUnusedYangResourceModules() } /* C R E A T E S C H E M A S E T U S E - C A S E S @@ -77,7 +77,7 @@ class ModuleServiceIntegrationSpec extends FunctionalSpecBase { originalNumberOfModuleReferences + numberOfNewModules == yangResourceModuleReferences.size() cleanup: objectUnderTest.deleteSchemaSetsWithCascade(FUNCTIONAL_TEST_DATASPACE_1, [ 'newSchemaSet' ]) - where: 'the following parameters are use' + where: 'the following parameters are used' scenario | numberOfNewModules 'two valid new modules' | 2 'empty schema set' | 0 diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy index 5c2a4fc665..3b49cfc415 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/QueryServiceIntegrationSpec.groovy @@ -224,6 +224,7 @@ class QueryServiceIntegrationSpec extends FunctionalSpecBase { 'ancestor with parent that does not exist' | '//books/ancestor::parentDoesNoExist/categories' || [] 'ancestor does not exist' | '//books/ancestor::ancestorDoesNotExist' || [] 'ancestor combined with contains condition' | '//books[contains(@title,"Mat")]/ancestor::bookstore' || ["/bookstore"] + 'ancestor is the same as search target' | '//books/ancestor::books' || [] } def 'Query for attribute by cps path of type ancestor with #scenario descendants.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy index 20fa546eba..43bcbdb4f4 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy @@ -24,6 +24,7 @@ import io.micrometer.core.instrument.MeterRegistry import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.springframework.beans.factory.annotation.Autowired +import org.springframework.util.StopWatch import spock.util.concurrent.PollingConditions import java.util.concurrent.Executors @@ -61,12 +62,13 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { assert moduleSyncWorkQueue.isEmpty() } - def 'CPS-2478 Highlight module sync inefficiencies.'() { + def 'CPS-2478 Highlight (and improve) module sync inefficiencies.'() { given: 'register 250 cm handles with module set tag cps-2478-A' def numberOfTags = 2 def cmHandlesPerTag = 250 def totalCmHandles = numberOfTags * cmHandlesPerTag def offset = 1 + def minimumBatches = totalCmHandles / 100 registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset) and: 'register anther 250 cm handles with module set tag cps-2478-B' offset += cmHandlesPerTag @@ -86,7 +88,7 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)' def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() new PollingConditions().within(10, () -> { - assert dbStateUpdateTimer.count() >= 5 + assert dbStateUpdateTimer.count() >= minimumBatches }) and: 'the db has been queried for tags exactly 2 times.' def dbModuleQueriesTimer = meterRegistry.get('cps.module.service.module.reference.query.by.attribute').timer() @@ -100,7 +102,12 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets ') logInstrumentation(dbStateUpdateTimer, 'batch state updates ') cleanup: 'remove all cm handles' + // To properly measure performance the sample-size should be increased to 20,000 cm handles or higher (10,000 per tag) + def stopWatch = new StopWatch() + stopWatch.start() deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1) + stopWatch.stop() + println "*** CPS-2478, Deletion of $totalCmHandles cm handles took ${stopWatch.getTotalTimeMillis()} milliseconds" } def 'Populate module sync work queue simultaneously on two parallel threads (CPS-2403).'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy index add931a1ad..914f2030cc 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy @@ -100,6 +100,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, (i..i+100).collect {SCHEMA_SET_PREFIX + it}) } cpsModuleService.deleteSchemaSetsWithCascade(CPS_PERFORMANCE_TEST_DATASPACE, [SCHEMA_SET_PREFIX + '0']) + cpsModuleService.deleteUnusedYangResourceModules() } // This makes a Yang module of approximately target length in bytes by padding the description field with many '*' |