diff options
43 files changed, 599 insertions, 203 deletions
diff --git a/cps-dependencies/pom.xml b/cps-dependencies/pom.xml index ad1828ec5c..6c034c2d7a 100644 --- a/cps-dependencies/pom.xml +++ b/cps-dependencies/pom.xml @@ -219,7 +219,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> - <version>3.11</version> + <version>3.17.0</version> </dependency> <dependency> <groupId>org.apache.maven.plugins</groupId> @@ -249,7 +249,7 @@ <dependency> <groupId>org.liquibase</groupId> <artifactId>liquibase-core</artifactId> - <version>4.29.0</version> + <version>4.30.0</version> </dependency> <dependency> <groupId>org.mapstruct</groupId> diff --git a/cps-ncmp-rest/docs/openapi/components.yaml b/cps-ncmp-rest/docs/openapi/components.yaml index e564c6b0cf..637a1386f4 100644 --- a/cps-ncmp-rest/docs/openapi/components.yaml +++ b/cps-ncmp-rest/docs/openapi/components.yaml @@ -516,7 +516,7 @@ components: outputAlternateIdOptionInQuery: name: outputAlternateId in: query - description: Boolean parameter to determine if returned value(s) will be cm handle Ids or alternate Ids for a given query + description: Boolean parameter to determine if returned value(s) will be cm handle ids or alternate ids for a given query required: false schema: type: boolean diff --git a/cps-ncmp-rest/docs/openapi/ncmp.yml b/cps-ncmp-rest/docs/openapi/ncmp.yml index 4624bc1060..15b8b37231 100755 --- a/cps-ncmp-rest/docs/openapi/ncmp.yml +++ b/cps-ncmp-rest/docs/openapi/ncmp.yml @@ -417,7 +417,7 @@ getCmHandleStateById: searchCmHandleIds: post: - description: Execute cm handle query search and return a list of cm handle references. Any number of conditions can be applied. To be included in the result a cm-handle must fulfill ALL the conditions. An empty collection will be returned in the case that the cm handle does not match a condition. For more on cm handle query search please refer to <a href="https://docs.onap.org/projects/onap-cps/en/latest/ncmp-cmhandle-querying.html">cm handle query search Read the Docs</a>.<br/>By supplying a CPS Path it is possible to query on any data related to the cm handle. For more on CPS Path please refer to <a href="https://docs.onap.org/projects/onap-cps/en/latest/cps-path.html">CPS Path Read the Docs</a>. The cm handle ancestor is automatically returned for this query. + description: Execute cm handle query search and return a list of cm handle references. Any number of conditions can be applied. To be included in the result a cm handle must fulfill ALL the conditions. An empty collection will be returned in the case that the cm handle does not match a condition. For more on cm handle query search please refer to <a href="https://docs.onap.org/projects/onap-cps/en/latest/ncmp-cmhandle-querying.html">cm handle query search Read the Docs</a>.<br/>By supplying a CPS Path it is possible to query on any data related to the cm handle. For more on CPS Path please refer to <a href="https://docs.onap.org/projects/onap-cps/en/latest/cps-path.html">CPS Path Read the Docs</a>. The cm handle ancestor is automatically returned for this query. tags: - network-cm-proxy summary: Execute cm handle query upon a given set of query parameters diff --git a/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyRestExceptionHandler.java b/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyRestExceptionHandler.java index 6910003c54..7255743c67 100644 --- a/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyRestExceptionHandler.java +++ b/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyRestExceptionHandler.java @@ -26,6 +26,7 @@ import lombok.extern.slf4j.Slf4j; import org.onap.cps.ncmp.api.data.exceptions.InvalidDatastoreException; import org.onap.cps.ncmp.api.data.exceptions.InvalidOperationException; import org.onap.cps.ncmp.api.data.exceptions.OperationNotSupportedException; +import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException; import org.onap.cps.ncmp.api.exceptions.DmiClientRequestException; import org.onap.cps.ncmp.api.exceptions.DmiRequestException; import org.onap.cps.ncmp.api.exceptions.InvalidTopicException; @@ -90,8 +91,8 @@ public class NetworkCmProxyRestExceptionHandler { return buildErrorResponse(HttpStatus.CONFLICT, exception); } - @ExceptionHandler({DataNodeNotFoundException.class}) - public static ResponseEntity<Object> handleNotFoundExceptions(final Exception exception) { + @ExceptionHandler({CmHandleNotFoundException.class, DataNodeNotFoundException.class}) + public static ResponseEntity<Object> cmHandleNotFoundExceptions(final Exception exception) { return buildErrorResponse(HttpStatus.NOT_FOUND, exception); } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NcmpResponseStatus.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NcmpResponseStatus.java index 8cfad7dbf6..be22752882 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NcmpResponseStatus.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NcmpResponseStatus.java @@ -27,14 +27,14 @@ public enum NcmpResponseStatus { SUCCESS("0", "Successfully applied changes"), CM_DATA_SUBSCRIPTION_ACCEPTED("1", "ACCEPTED"), - CM_HANDLES_NOT_FOUND("100", "cm handle id(s) not found"), + CM_HANDLES_NOT_FOUND("100", "cm handle reference(s) not found"), CM_HANDLES_NOT_READY("101", "cm handle(s) not ready"), DMI_SERVICE_NOT_RESPONDING("102", "dmi plugin service is not responding"), UNABLE_TO_READ_RESOURCE_DATA("103", "dmi plugin service is not able to read resource data"), CM_DATA_SUBSCRIPTION_REJECTED("104", "REJECTED"), UNKNOWN_ERROR("108", "Unknown error"), CM_HANDLE_ALREADY_EXIST("109", "cm-handle already exists"), - CM_HANDLE_INVALID_ID("110", "cm-handle has an invalid character(s) in id"), + CM_HANDLE_INVALID_ID("110", "cm handle reference has an invalid character(s) in id"), ALTERNATE_ID_ALREADY_ASSOCIATED("111", "alternate id already associated"), MESSAGE_TOO_LARGE("112", "message too large"); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/exceptions/CmHandleNotFoundException.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/exceptions/CmHandleNotFoundException.java new file mode 100644 index 0000000000..715e1a00db --- /dev/null +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/exceptions/CmHandleNotFoundException.java @@ -0,0 +1,38 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.api.exceptions; + +public class CmHandleNotFoundException extends NcmpException { + + private static final String CM_HANDLE_NOT_FOUND_DETAILS_FORMAT = + "No cm handles found with reference %s"; + + /** + * Constructor. + * + * @param cmHandleReference cm handle reference either cm handle id or alternate id + */ + public CmHandleNotFoundException(final String cmHandleReference) { + super("Cm handle not found", String.format(CM_HANDLE_NOT_FOUND_DETAILS_FORMAT, + cmHandleReference)); + } + +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java index a8996874ff..ec440f4905 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/NetworkCmProxyInventoryFacade.java @@ -27,8 +27,10 @@ package org.onap.cps.ncmp.api.inventory; import static org.onap.cps.ncmp.impl.inventory.CmHandleQueryParametersValidator.validateCmHandleQueryParameters; import java.util.Collection; +import java.util.Collections; import java.util.Map; import lombok.RequiredArgsConstructor; +import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException; import org.onap.cps.ncmp.api.inventory.models.CmHandleQueryApiParameters; import org.onap.cps.ncmp.api.inventory.models.CmHandleQueryServiceParameters; import org.onap.cps.ncmp.api.inventory.models.CompositeState; @@ -78,7 +80,7 @@ public class NetworkCmProxyInventoryFacade { * Get all cm handle references by DMI plugin identifier. * * @param dmiPluginIdentifier DMI plugin identifier - * @param outputAlternateId Boolean for cm handle reference type either + * @param outputAlternateId boolean for cm handle reference type either * cm handle id (false) or alternate id (true) * @return collection of cm handle references */ @@ -91,7 +93,7 @@ public class NetworkCmProxyInventoryFacade { * Get all cm handle IDs by various properties. * * @param cmHandleQueryServiceParameters cm handle query parameters - * @param outputAlternateId Boolean for cm handle reference type either + * @param outputAlternateId boolean for cm handle reference type either * cm handle id (false) or alternate id (true) * @return collection of cm handle references */ @@ -111,8 +113,12 @@ public class NetworkCmProxyInventoryFacade { * @return a collection of modules names and revisions */ public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandleReference) { - final String cmHandleId = alternateIdMatcher.getCmHandleId(cmHandleReference); - return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId); + try { + final String cmHandleId = alternateIdMatcher.getCmHandleId(cmHandleReference); + return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId); + } catch (final CmHandleNotFoundException cmHandleNotFoundException) { + return Collections.emptyList(); + } } /** @@ -122,8 +128,12 @@ public class NetworkCmProxyInventoryFacade { * @return a collection of module definition (moduleName, revision and yang resource content) */ public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleReference(final String cmHandleReference) { - final String cmHandleId = alternateIdMatcher.getCmHandleId(cmHandleReference); - return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId); + try { + final String cmHandleId = alternateIdMatcher.getCmHandleId(cmHandleReference); + return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId); + } catch (final CmHandleNotFoundException cmHandleNotFoundException) { + return Collections.emptyList(); + } } /** @@ -137,8 +147,12 @@ public class NetworkCmProxyInventoryFacade { public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleAndModule(final String cmHandleReference, final String moduleName, final String moduleRevision) { - final String cmHandleId = alternateIdMatcher.getCmHandleId(cmHandleReference); - return inventoryPersistence.getModuleDefinitionsByCmHandleAndModule(cmHandleId, moduleName, moduleRevision); + try { + final String cmHandleId = alternateIdMatcher.getCmHandleId(cmHandleReference); + return inventoryPersistence.getModuleDefinitionsByCmHandleAndModule(cmHandleId, moduleName, moduleRevision); + } catch (final CmHandleNotFoundException cmHandleNotFoundException) { + return Collections.emptyList(); + } } /** @@ -162,7 +176,7 @@ public class NetworkCmProxyInventoryFacade { * Retrieve cm handle ids for the given query parameters. * * @param cmHandleQueryApiParameters cm handle query parameters - * @param outputAlternateId Boolean for cm handle reference type either cmHandleId (false) or AlternateId (true) + * @param outputAlternateId boolean for cm handle reference type either cmHandleId (false) or AlternateId (true) * @return cm handle ids */ public Collection<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters, @@ -227,4 +241,4 @@ public class NetworkCmProxyInventoryFacade { .getEffectiveTrustLevel(ncmpServiceCmHandle.getCmHandleId())); } -}
\ No newline at end of file +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java index 109a541cb3..345eefec2a 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfig.java @@ -51,6 +51,7 @@ public class HazelcastCacheConfig { protected HazelcastInstance getOrCreateHazelcastInstance(final NamedConfig namedConfig) { return Hazelcast.getOrCreateHazelcastInstance(defineInstanceConfig(instanceConfigName, namedConfig)); + } private Config defineInstanceConfig(final String instanceConfigName, final NamedConfig namedConfig) { diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryService.java index 74c04928ed..415153ddf7 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/CmHandleQueryService.java @@ -103,8 +103,8 @@ public interface CmHandleQueryService { * Get collection of all cm handles references by DMI plugin identifier and alternate id output option. * * @param dmiPluginIdentifier DMI plugin identifier - * @param outputAlternateId Boolean for cm handle reference type either cmHandleId (false) or AlternateId (true) - * @return collection of cm handle ids + * @param outputAlternateId boolean for cm handle reference type either cmHandleId (false) or AlternateId (true) + * @return collection of cm handle references */ Collection<String> getCmHandleReferencesByDmiPluginIdentifier(String dmiPluginIdentifier, boolean outputAlternateId); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistence.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistence.java index 61d7df923e..e5dd937cc0 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistence.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistence.java @@ -157,12 +157,12 @@ public interface InventoryPersistence extends NcmpPersistence { * get CM handles that has given module names. * * @param moduleNamesForQuery module names - * @param outputAlternateIds Boolean for cm handle reference type either + * @param outputAlternateId boolean for cm handle reference type either * cm handle id (false or null) or alternate id (true) - * @return Collection of CM handle Ids + * @return Collection of CM handle references */ Collection<String> getCmHandleReferencesWithGivenModules(Collection<String> moduleNamesForQuery, - boolean outputAlternateIds); + boolean outputAlternateId); /** * Check database if cm handle id exists if not return false. diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java index c4765ff53d..e468ed100a 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImpl.java @@ -40,6 +40,7 @@ import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; import org.onap.cps.impl.utils.CpsValidator; +import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException; import org.onap.cps.ncmp.api.inventory.models.CompositeState; import org.onap.cps.ncmp.api.inventory.models.CompositeStateBuilder; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; @@ -193,8 +194,7 @@ public class InventoryPersistenceImpl extends NcmpPersistenceImpl implements Inv final Collection<DataNode> dataNodes = cmHandleQueryService .queryNcmpRegistryByCpsPath(cpsPathForCmHandleByAlternateId, OMIT_DESCENDANTS); if (dataNodes.isEmpty()) { - throw new DataNodeNotFoundException(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, - cpsPathForCmHandleByAlternateId); + throw new CmHandleNotFoundException(alternateId); } return dataNodes.iterator().next(); } @@ -218,8 +218,8 @@ public class InventoryPersistenceImpl extends NcmpPersistenceImpl implements Inv @Override public Collection<String> getCmHandleReferencesWithGivenModules(final Collection<String> moduleNamesForQuery, - final boolean outputAlternateIds) { - if (outputAlternateIds) { + final boolean outputAlternateId) { + if (outputAlternateId) { final Collection<String> cmHandleIds = cpsAnchorService.queryAnchorNames(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, moduleNamesForQuery); return getAlternateIdsFromDataNodes(getCmHandleDataNodes(cmHandleIds)); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java index 8ba70b3a31..a056efd6ce 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DmiModelOperations.java @@ -26,6 +26,7 @@ import static org.onap.cps.ncmp.impl.models.RequiredDmiService.MODEL; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import io.micrometer.core.annotation.Timed; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -33,6 +34,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.onap.cps.ncmp.api.inventory.models.YangResource; import org.onap.cps.ncmp.impl.dmi.DmiProperties; import org.onap.cps.ncmp.impl.dmi.DmiRestClient; @@ -48,6 +50,7 @@ import org.springframework.stereotype.Service; /** * Operations class for DMI Model. */ +@Slf4j @RequiredArgsConstructor @Service public class DmiModelOperations { @@ -62,6 +65,8 @@ public class DmiModelOperations { * @param yangModelCmHandle the yang model cm handle * @return module references */ + @Timed(value = "cps.ncmp.inventory.module.references.from.dmi", + description = "Time taken to get all module references for a cm handle from dmi") public List<ModuleReference> getModuleReferences(final YangModelCmHandle yangModelCmHandle) { final DmiRequestBody dmiRequestBody = DmiRequestBody.builder() .moduleSetTag(yangModelCmHandle.getModuleSetTag()).build(); @@ -79,6 +84,8 @@ public class DmiModelOperations { * @param newModuleReferences the unknown module references * @return yang resources as map of module name to yang(re)source */ + @Timed(value = "cps.ncmp.inventory.yang.resources.from.dmi", + description = "Time taken to get list of yang resources from dmi") public Map<String, String> getNewYangResourcesFromDmi(final YangModelCmHandle yangModelCmHandle, final Collection<ModuleReference> newModuleReferences) { if (newModuleReferences.isEmpty()) { diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java index ca0f1c6a6d..ba50dd3c19 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncService.java @@ -26,16 +26,20 @@ import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT; import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME; +import com.hazelcast.collection.ISet; import java.time.OffsetDateTime; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import lombok.AllArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.util.Strings; import org.onap.cps.api.CpsAnchorService; import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; +import org.onap.cps.ncmp.api.exceptions.NcmpException; import org.onap.cps.ncmp.impl.inventory.models.CmHandleState; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; import org.onap.cps.spi.CascadeDeleteAllowed; @@ -50,12 +54,15 @@ import org.springframework.stereotype.Service; @RequiredArgsConstructor public class ModuleSyncService { + private static final Map<String, String> NO_NEW_MODULES = Collections.emptyMap(); + private final DmiModelOperations dmiModelOperations; private final CpsModuleService cpsModuleService; private final CpsDataService cpsDataService; private final CpsAnchorService cpsAnchorService; private final JsonObjectMapper jsonObjectMapper; - private static final Map<String, String> NO_NEW_MODULES = Collections.emptyMap(); + private final ISet<String> moduleSetTagsBeingProcessed; + private final Map<String, ModuleDelta> privateModuleSetCache = new HashMap<>(); @AllArgsConstructor private static final class ModuleDelta { @@ -69,11 +76,37 @@ public class ModuleSyncService { * @param yangModelCmHandle the yang model of cm handle. */ public void syncAndCreateSchemaSetAndAnchor(final YangModelCmHandle yangModelCmHandle) { - final ModuleDelta moduleDelta = getModuleDelta(yangModelCmHandle, yangModelCmHandle.getModuleSetTag()); - final String cmHandleId = yangModelCmHandle.getId(); - cpsModuleService.createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, + final String moduleSetTag = yangModelCmHandle.getModuleSetTag(); + final ModuleDelta moduleDelta; + boolean isNewModuleSetTag = Strings.isNotBlank(moduleSetTag); + try { + if (privateModuleSetCache.containsKey(moduleSetTag)) { + moduleDelta = privateModuleSetCache.get(moduleSetTag); + } else { + if (isNewModuleSetTag) { + if (moduleSetTagsBeingProcessed.add(moduleSetTag)) { + log.info("Processing new module set tag {}", moduleSetTag); + } else { + isNewModuleSetTag = false; + throw new NcmpException("Concurrent processing of module set tag " + moduleSetTag, + moduleSetTag + " already being processed for cm handle " + yangModelCmHandle.getId()); + } + } + moduleDelta = getModuleDelta(yangModelCmHandle, moduleSetTag); + } + final String cmHandleId = yangModelCmHandle.getId(); + cpsModuleService.createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, moduleDelta.newModuleNameToContentMap, moduleDelta.allModuleReferences); - cpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, cmHandleId); + cpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, cmHandleId); + if (isNewModuleSetTag) { + final ModuleDelta noModuleDelta = new ModuleDelta(moduleDelta.allModuleReferences, NO_NEW_MODULES); + privateModuleSetCache.put(moduleSetTag, noModuleDelta); + } + } finally { + if (isNewModuleSetTag) { + moduleSetTagsBeingProcessed.remove(moduleSetTag); + } + } } /** @@ -105,6 +138,10 @@ public class ModuleSyncService { } } + public void clearPrivateModuleSetCache() { + privateModuleSetCache.clear(); + } + private ModuleDelta getModuleDelta(final YangModelCmHandle yangModelCmHandle, final String targetModuleSetTag) { final Map<String, String> newYangResources; Collection<ModuleReference> allModuleReferences = getModuleReferencesByModuleSetTag(targetModuleSetTag); @@ -120,7 +157,7 @@ public class ModuleSyncService { } private Collection<ModuleReference> getModuleReferencesByModuleSetTag(final String moduleSetTag) { - if (moduleSetTag == null || moduleSetTag.trim().isEmpty()) { + if (Strings.isBlank(moduleSetTag)) { return Collections.emptyList(); } return cpsModuleService.getModuleReferencesByAttribute(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java index 31fcbad08b..7cc74a3b55 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasks.java @@ -63,7 +63,8 @@ public class ModuleSyncTasks { try { cmHandlesAsDataNodes.forEach(cmHandleAsDataNode -> { final YangModelCmHandle yangModelCmHandle = YangDataConverter.toYangModelCmHandle(cmHandleAsDataNode); - cmHandleStatePerCmHandle.put(yangModelCmHandle, processCmHandle(yangModelCmHandle)); + final CmHandleState cmHandleState = processCmHandle(yangModelCmHandle); + cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState); }); } finally { batchCounter.getAndDecrement(); @@ -127,4 +128,4 @@ public class ModuleSyncTasks { log.info("{} removed from in progress map", resetCmHandleId); } } -}
\ No newline at end of file +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java index 1f33cc349d..b98075c06c 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfig.java @@ -20,8 +20,10 @@ package org.onap.cps.ncmp.impl.inventory.sync; +import com.hazelcast.collection.ISet; import com.hazelcast.config.MapConfig; import com.hazelcast.config.QueueConfig; +import com.hazelcast.config.SetConfig; import com.hazelcast.map.IMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.locks.Lock; @@ -44,6 +46,8 @@ public class SynchronizationCacheConfig extends HazelcastCacheConfig { private static final QueueConfig commonQueueConfig = createQueueConfig("defaultQueueConfig"); private static final MapConfig moduleSyncStartedConfig = createMapConfig("moduleSyncStartedConfig"); private static final MapConfig dataSyncSemaphoresConfig = createMapConfig("dataSyncSemaphoresConfig"); + private static final SetConfig moduleSetTagsBeingProcessedConfig + = createSetConfig("moduleSetTagsBeingProcessedConfig"); private static final String LOCK_NAME_FOR_WORK_QUEUE = "workQueueLock"; /** @@ -63,8 +67,7 @@ public class SynchronizationCacheConfig extends HazelcastCacheConfig { */ @Bean public IMap<String, Object> moduleSyncStartedOnCmHandles() { - return getOrCreateHazelcastInstance(moduleSyncStartedConfig).getMap( - "moduleSyncStartedOnCmHandles"); + return getOrCreateHazelcastInstance(moduleSyncStartedConfig).getMap("moduleSyncStartedOnCmHandles"); } /** @@ -78,6 +81,17 @@ public class SynchronizationCacheConfig extends HazelcastCacheConfig { } /** + * Collection of (new) module set tags being processed. + * To prevent processing on multiple threads of same tag + * + * @return set of module set tags being processed + */ + @Bean + public ISet<String> moduleSetTagsBeingProcessed() { + return getOrCreateHazelcastInstance(moduleSetTagsBeingProcessedConfig).getSet("moduleSetTagsBeingProcessed"); + } + + /** * Retrieves a distributed lock used to control access to the work queue for module synchronization. * This lock ensures that the population and modification of the work queue are thread-safe and * protected from concurrent access across different nodes in the distributed system. diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/AlternateIdMatcher.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/AlternateIdMatcher.java index 9facd630a2..36c0cfa756 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/AlternateIdMatcher.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/utils/AlternateIdMatcher.java @@ -23,9 +23,9 @@ package org.onap.cps.ncmp.impl.utils; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException; import org.onap.cps.ncmp.exceptions.NoAlternateIdMatchFoundException; import org.onap.cps.ncmp.impl.inventory.InventoryPersistence; -import org.onap.cps.spi.exceptions.DataNodeNotFoundException; import org.onap.cps.spi.model.DataNode; import org.springframework.stereotype.Service; @@ -50,7 +50,7 @@ public class AlternateIdMatcher { while (StringUtils.isNotEmpty(bestMatch)) { try { return inventoryPersistence.getCmHandleDataNodeByAlternateId(bestMatch); - } catch (final DataNodeNotFoundException ignored) { + } catch (final CmHandleNotFoundException ignored) { bestMatch = getParentPath(bestMatch, separator); } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy index 0bd838437d..c08ff75a44 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/cache/HazelcastCacheConfigSpec.groovy @@ -22,12 +22,17 @@ package org.onap.cps.ncmp.impl.cache import com.hazelcast.config.Config import com.hazelcast.config.RestEndpointGroup +import com.hazelcast.core.Hazelcast import spock.lang.Specification class HazelcastCacheConfigSpec extends Specification { def objectUnderTest = new HazelcastCacheConfig() + def cleanupSpec() { + Hazelcast.getHazelcastInstanceByName('my instance config').shutdown() + } + def 'Create Hazelcast instance with a #scenario'() { given: 'a cluster name and instance config name' objectUnderTest.clusterName = 'my cluster' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy index 1beab20def..6213258303 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServicePropertyHandlerSpec.groovy @@ -162,9 +162,9 @@ class CmHandleRegistrationServicePropertyHandlerSpec extends Specification { } where: scenario | cmHandleId | exception || expectedError | expectedErrorText - 'Cm Handle does not exist' | 'cmHandleId' | new DataNodeNotFoundException(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR) || CM_HANDLES_NOT_FOUND | 'cm handle id(s) not found' + 'Cm Handle does not exist' | 'cmHandleId' | new DataNodeNotFoundException(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR) || CM_HANDLES_NOT_FOUND | 'cm handle reference(s) not found' 'Unknown' | 'cmHandleId' | new RuntimeException('Failed') || UNKNOWN_ERROR | 'Failed' - 'Invalid cm handle id' | 'cmHandleId with spaces' | new DataValidationException('Name Validation Error.', cmHandleId + 'contains an invalid character') || CM_HANDLE_INVALID_ID | 'cm-handle has an invalid character(s) in id' + 'Invalid cm handle id' | 'cmHandleId with spaces' | new DataValidationException('Name Validation Error.', cmHandleId + 'contains an invalid character') || CM_HANDLE_INVALID_ID | 'cm handle reference has an invalid character(s) in id' } def 'Multiple update operations in a single request'() { @@ -193,7 +193,7 @@ class CmHandleRegistrationServicePropertyHandlerSpec extends Specification { assert it.status == Status.FAILURE assert it.cmHandle == cmHandleId assert it.ncmpResponseStatus == CM_HANDLES_NOT_FOUND - assert it.errorText == 'cm handle id(s) not found' + assert it.errorText == 'cm handle reference(s) not found' } then: 'the replace list method is called twice' 2 * mockInventoryPersistence.replaceListContent(cmHandleXpath, _) diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy index 70e26d993c..a69721b6aa 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/CmHandleRegistrationServiceSpec.groovy @@ -399,10 +399,10 @@ class CmHandleRegistrationServiceSpec extends Specification { and: 'the cm handle state is not updated to "DELETED"' 0 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_, CmHandleState.DELETED) where: - scenario | cmHandleId | deleteListElementException || expectedError | expectedErrorText - 'cm-handle does not exist' | 'cmhandle' | new DataNodeNotFoundException('', '', '') || CM_HANDLES_NOT_FOUND | 'cm handle id(s) not found' - 'cm-handle has invalid name' | 'cm handle with space' | new DataValidationException('', '') || CM_HANDLE_INVALID_ID | 'cm-handle has an invalid character(s) in id' - 'an unexpected exception' | 'cmhandle' | new RuntimeException('Failed') || UNKNOWN_ERROR | 'Failed' + scenario | deleteListElementException || expectedError | expectedErrorText + 'cm-handle does not exist' | new DataNodeNotFoundException('', '', '') || CM_HANDLES_NOT_FOUND | 'cm handle reference(s) not found' + 'cm-handle has invalid name' | new DataValidationException('', '') || CM_HANDLE_INVALID_ID | 'cm handle reference has an invalid character(s) in id' + 'an unexpected exception' | new RuntimeException('Failed') || UNKNOWN_ERROR | 'Failed' } def 'Set Cm Handle Data Sync Enabled Flag where data sync flag is #scenario'() { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy index 00f092ff75..4d8855c4b2 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/InventoryPersistenceImplSpec.groovy @@ -27,12 +27,12 @@ import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsModuleService import org.onap.cps.impl.utils.CpsValidator +import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException import org.onap.cps.ncmp.api.inventory.models.CompositeState import org.onap.cps.ncmp.impl.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle import org.onap.cps.spi.CascadeDeleteAllowed import org.onap.cps.spi.FetchDescendantsOption -import org.onap.cps.spi.exceptions.DataNodeNotFoundException import org.onap.cps.spi.model.DataNode import org.onap.cps.spi.model.ModuleDefinition import org.onap.cps.spi.model.ModuleReference @@ -327,8 +327,9 @@ class InventoryPersistenceImplSpec extends Specification { when: 'getting the cm handle data node' objectUnderTest.getCmHandleDataNodeByAlternateId('alternate id') then: 'no data found exception thrown' - def thrownException = thrown(DataNodeNotFoundException) - assert thrownException.getMessage().contains('DataNode not found') + def thrownException = thrown(CmHandleNotFoundException) + assert thrownException.getMessage().contains('Cm handle not found') + assert thrownException.getDetails().contains('No cm handles found with reference alternate id') } def 'Get multiple cm handle data nodes by alternate ids, passing empty collection'() { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy index 6030e5debf..2f13a9a483 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncServiceSpec.groovy @@ -20,14 +20,17 @@ package org.onap.cps.ncmp.impl.inventory.sync +import com.hazelcast.collection.ISet import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsModuleService +import org.onap.cps.ncmp.api.exceptions.NcmpException import org.onap.cps.ncmp.api.inventory.models.CompositeStateBuilder import org.onap.cps.ncmp.api.inventory.models.NcmpServiceCmHandle import org.onap.cps.ncmp.impl.inventory.CmHandleQueryService import org.onap.cps.ncmp.impl.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle +import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncService.ModuleDelta import org.onap.cps.spi.CascadeDeleteAllowed import org.onap.cps.spi.exceptions.SchemaSetNotFoundException import org.onap.cps.spi.model.ModuleReference @@ -45,18 +48,22 @@ class ModuleSyncServiceSpec extends Specification { def mockCmHandleQueries = Mock(CmHandleQueryService) def mockCpsDataService = Mock(CpsDataService) def mockJsonObjectMapper = Mock(JsonObjectMapper) + def mockModuleSetTagsBeingProcessed = Mock(ISet<String>); - def objectUnderTest = new ModuleSyncService(mockDmiModelOperations, mockCpsModuleService, - mockCpsDataService, mockCpsAnchorService, mockJsonObjectMapper) + def objectUnderTest = new ModuleSyncService(mockDmiModelOperations, mockCpsModuleService, mockCpsDataService, mockCpsAnchorService, mockJsonObjectMapper, mockModuleSetTagsBeingProcessed) def expectedDataspaceName = NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME - def 'Sync model for a NEW cm handle using module set tags: #scenario.'() { - given: 'a cm handle state to be synced' - def ncmpServiceCmHandle = new NcmpServiceCmHandle() - ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withCmHandleState(CmHandleState.ADVISED).build()) - ncmpServiceCmHandle.cmHandleId = 'ch-1' - def yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle('some service name', '', '', ncmpServiceCmHandle, moduleSetTag, '', '') + def setup() { + // Allow tags for al test except 'duplicate-processing-tag' to be added to processing semaphore + mockModuleSetTagsBeingProcessed.add('new-tag') >> true + mockModuleSetTagsBeingProcessed.add('same-tag') >> true + mockModuleSetTagsBeingProcessed.add('cached-tag') >> true + } + + def 'Sync models for a NEW cm handle using module set tags: #scenario.'() { + given: 'a cm handle to be synced' + def yangModelCmHandle = createAdvisedCmHandle(moduleSetTag) and: 'DMI operations returns some module references' def moduleReferences = [ new ModuleReference('module1','1'), new ModuleReference('module2','2') ] mockDmiModelOperations.getModuleReferences(yangModelCmHandle) >> moduleReferences @@ -75,10 +82,60 @@ class ModuleSyncServiceSpec extends Specification { where: 'the following parameters are used' scenario | identifiedNewModuleReferences | newModuleNameContentToMap | moduleSetTag | existingModuleReferences 'one new module, new tag' | [new ModuleReference('module1', '1')] | [module1: 'some yang source'] | '' | [] - 'no new module, new tag' | [] | [:] | 'new-tag-1' | [] + 'no new module, new tag' | [] | [:] | 'new-tag' | [] 'same tag' | [] | [:] | 'same-tag' | [new ModuleReference('module1', '1'), new ModuleReference('module2', '2')] } + def 'Attempt Sync models for a cm handle with exception and #scenario module set tag'() { + given: 'a cm handle to be synced' + def yangModelCmHandle = createAdvisedCmHandle(moduleSetTag) + and: 'the service returns a list of module references when queried with the specified attributes' + mockCpsModuleService.getModuleReferencesByAttribute(*_) >> [new ModuleReference('module1', '1')] + and: 'exception occurs when try to store result' + def testException = new RuntimeException('test') + mockCpsModuleService.createSchemaSetFromModules(*_) >> { throw testException } + when: 'module sync is triggered' + objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) + then: 'the same exception is thrown up' + def exceptionThrown = thrown(Exception) + assert testException == exceptionThrown + and: 'module set tag is removed from processing semaphores only when needed' + expectedCallsToRemoveTag * mockModuleSetTagsBeingProcessed.remove('new-tag') + where: 'following module set tags are used' + scenario | moduleSetTag || expectedCallsToRemoveTag + 'with' | 'new-tag' || 1 + 'without' | ' ' || 0 + } + + def 'Sync models for a cm handle with previously cached module set tag.'() { + given: 'a cm handle to be synced' + def yangModelCmHandle = createAdvisedCmHandle('cached-tag') + and: 'The module set tag exist in the private cache' + def moduleReferences = [ new ModuleReference('module1','1') ] + def cachedModuleDelta = new ModuleDelta(moduleReferences, [:]) + objectUnderTest.privateModuleSetCache.put('cached-tag', cachedModuleDelta) + when: 'module sync is triggered' + objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) + then: 'create schema set from module is invoked with correct parameters' + 1 * mockCpsModuleService.createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'ch-1', [:], moduleReferences) + and: 'anchor is created with the correct parameters' + 1 * mockCpsAnchorService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, 'ch-1', 'ch-1') + } + + def 'Attempt to sync using a module set tag already being processed by a different instance or thread.'() { + given: 'a cm handle to be synced' + def yangModelCmHandle = createAdvisedCmHandle('duplicateTag') + and: 'The module set tag already exist in the processing semaphore set' + mockModuleSetTagsBeingProcessed.add('duplicate-processing-tag') > false + when: 'module sync is triggered' + objectUnderTest.syncAndCreateSchemaSetAndAnchor(yangModelCmHandle) + then: 'a ncmp exception is thrown with the relevant details' + def exceptionThrown = thrown(NcmpException) + assert exceptionThrown.message.contains('duplicateTag') + assert exceptionThrown.details.contains('duplicateTag') + assert exceptionThrown.details.contains('ch-1') + } + def 'Upgrade model for an existing cm handle with Module Set Tag where the modules are #scenario'() { given: 'a cm handle being upgraded to module set tag: tag-1' def ncmpServiceCmHandle = new NcmpServiceCmHandle() @@ -113,7 +170,7 @@ class ModuleSyncServiceSpec extends Specification { 'in database' | [new ModuleReference('module1', '1')] } - def 'upgrade model for a existing cm handle'() { + def 'upgrade model for an existing cm handle'() { given: 'a cm handle that is ready but locked for upgrade' def ncmpServiceCmHandle = new NcmpServiceCmHandle() ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder() @@ -159,4 +216,20 @@ class ModuleSyncServiceSpec extends Specification { result == unsupportedOperationException } + def 'Clear module set cache.'() { + given: 'something in the module set cache' + objectUnderTest.privateModuleSetCache.put('test',new ModuleDelta([],[:])) + when: 'the cache is cleared' + objectUnderTest.clearPrivateModuleSetCache() + then: 'the cache is empty' + objectUnderTest.privateModuleSetCache.isEmpty() + } + + def createAdvisedCmHandle(moduleSetTag) { + def ncmpServiceCmHandle = new NcmpServiceCmHandle() + ncmpServiceCmHandle.setCompositeState(new CompositeStateBuilder().withCmHandleState(CmHandleState.ADVISED).build()) + ncmpServiceCmHandle.cmHandleId = 'ch-1' + return YangModelCmHandle.toYangModelCmHandle('some service name', '', '', ncmpServiceCmHandle, moduleSetTag, '', '') + } + } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy index 8ce1e934f2..e21c868bbf 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncTasksSpec.groovy @@ -26,6 +26,7 @@ import ch.qos.logback.classic.Logger import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.core.read.ListAppender import com.hazelcast.config.Config +import com.hazelcast.core.Hazelcast import com.hazelcast.instance.impl.HazelcastInstanceFactory import com.hazelcast.map.IMap import org.onap.cps.ncmp.api.inventory.models.CompositeState @@ -75,6 +76,10 @@ class ModuleSyncTasksSpec extends Specification { def objectUnderTest = new ModuleSyncTasks(mockInventoryPersistence, mockSyncUtils, mockModuleSyncService, mockLcmEventsCmHandleStateHandler, moduleSyncStartedOnCmHandles) + def cleanupSpec() { + Hazelcast.getHazelcastInstanceByName('hazelcastInstanceName').shutdown() + } + def 'Module Sync ADVISED cm handles.'() { given: 'cm handles in an ADVISED state' def cmHandle1 = cmHandleAsDataNodeByIdAndState('cm-handle-1', CmHandleState.ADVISED) diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy index 4c96d6b822..c2ecf927c8 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/SynchronizationCacheConfigSpec.groovy @@ -20,6 +20,7 @@ package org.onap.cps.ncmp.impl.inventory.sync +import com.hazelcast.collection.ISet import com.hazelcast.config.Config import com.hazelcast.core.Hazelcast import com.hazelcast.map.IMap @@ -38,13 +39,16 @@ import java.util.concurrent.TimeUnit class SynchronizationCacheConfigSpec extends Specification { @Autowired - private BlockingQueue<DataNode> moduleSyncWorkQueue + BlockingQueue<DataNode> moduleSyncWorkQueue @Autowired - private IMap<String, Object> moduleSyncStartedOnCmHandles + IMap<String, Object> moduleSyncStartedOnCmHandles @Autowired - private IMap<String, Boolean> dataSyncSemaphores + IMap<String, Boolean> dataSyncSemaphores + + @Autowired + ISet<String> moduleSetTagsBeingProcessed def cleanupSpec() { Hazelcast.getHazelcastInstanceByName('cps-and-ncmp-hazelcast-instance-test-config').shutdown() @@ -57,8 +61,11 @@ class SynchronizationCacheConfigSpec extends Specification { assert null != moduleSyncStartedOnCmHandles and: 'system is able to create an instance of a map to hold data sync semaphores' assert null != dataSyncSemaphores - and: 'they have the correct names (in any order)' - assert Hazelcast.allHazelcastInstances.name.contains('cps-and-ncmp-hazelcast-instance-test-config') + and: 'system is able to create an instance of a set to hold module set tags being processed' + assert null != moduleSetTagsBeingProcessed + and: 'there is only one instance with the correct name' + assert Hazelcast.allHazelcastInstances.size() == 1 + assert Hazelcast.allHazelcastInstances.name[0] == 'cps-and-ncmp-hazelcast-instance-test-config' } def 'Verify configs for Distributed objects'(){ @@ -103,7 +110,6 @@ class SynchronizationCacheConfigSpec extends Specification { then: 'applied properties are reflected' assert testConfig.networkConfig.join.kubernetesConfig.enabled assert testConfig.networkConfig.join.kubernetesConfig.properties.get('service-name') == 'test-service-name' - } def 'Time to Live Verify for Module Sync Semaphore'() { diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/utils/AlternateIdMatcherSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/utils/AlternateIdMatcherSpec.groovy index bd1faa2705..0a58039d8a 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/utils/AlternateIdMatcherSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/utils/AlternateIdMatcherSpec.groovy @@ -20,9 +20,9 @@ package org.onap.cps.ncmp.impl.utils +import org.onap.cps.ncmp.api.exceptions.CmHandleNotFoundException import org.onap.cps.ncmp.exceptions.NoAlternateIdMatchFoundException import org.onap.cps.ncmp.impl.inventory.InventoryPersistence -import org.onap.cps.spi.exceptions.DataNodeNotFoundException import org.onap.cps.spi.model.DataNode import spock.lang.Specification @@ -35,7 +35,7 @@ class AlternateIdMatcherSpec extends Specification { given: 'cm handle in the registry with alternate id /a/b' mockInventoryPersistence.getCmHandleDataNodeByAlternateId('/a/b') >> new DataNode() and: 'no other cm handle' - mockInventoryPersistence.getCmHandleDataNodeByAlternateId(_) >> { throw new DataNodeNotFoundException('', '') } + mockInventoryPersistence.getCmHandleDataNodeByAlternateId(_) >> { throw new CmHandleNotFoundException('') } } def 'Finding longest alternate id matches.'() { diff --git a/cps-ncmp-service/src/test/resources/dataOperationResponseEvent.json b/cps-ncmp-service/src/test/resources/dataOperationResponseEvent.json index 611d47d1a3..827250f5fd 100644 --- a/cps-ncmp-service/src/test/resources/dataOperationResponseEvent.json +++ b/cps-ncmp-service/src/test/resources/dataOperationResponseEvent.json @@ -1 +1 @@ -[{"operationId":"operational-14","ids":["unknown-cm-handle"],"resourceIdentifier":"some resource identifier","options":"some option","statusCode":"100","statusMessage":"cm handle id(s) not found"},{"operationId":"operational-14","ids":["non-ready-cm-handle"],"resourceIdentifier":"some resource identifier","options":"some option","statusCode":"101","statusMessage":"cm handle(s) not ready"},{"operationId":"running-12","ids":["non-ready-cm-handle"],"resourceIdentifier":"some resource identifier","options":"some option","statusCode":"101","statusMessage":"cm handle(s) not ready"}]
\ No newline at end of file +[{"operationId":"operational-14","ids":["unknown-cm-handle"],"resourceIdentifier":"some resource identifier","options":"some option","statusCode":"100","statusMessage":"cm handle reference(s) not found"},{"operationId":"operational-14","ids":["non-ready-cm-handle"],"resourceIdentifier":"some resource identifier","options":"some option","statusCode":"101","statusMessage":"cm handle(s) not ready"},{"operationId":"running-12","ids":["non-ready-cm-handle"],"resourceIdentifier":"some resource identifier","options":"some option","statusCode":"101","statusMessage":"cm handle(s) not ready"}]
\ No newline at end of file diff --git a/cps-rest/docs/openapi/cpsQueryV2.yml b/cps-rest/docs/openapi/cpsQueryV2.yml index 7f0ceff768..9aaa4193c3 100644 --- a/cps-rest/docs/openapi/cpsQueryV2.yml +++ b/cps-rest/docs/openapi/cpsQueryV2.yml @@ -1,5 +1,6 @@ # ============LICENSE_START======================================================= # Copyright (C) 2023 TechMahindra Ltd. +# Modifications Copyright (C) 2023-2024 TechMahindra Ltd. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,6 +29,7 @@ nodesByDataspaceAndAnchorAndCpsPath: - $ref: 'components.yml#/components/parameters/anchorNameInPath' - $ref: 'components.yml#/components/parameters/cpsPathInQuery' - $ref: 'components.yml#/components/parameters/descendantsInQuery' + - $ref: 'components.yml#/components/parameters/contentTypeInHeader' responses: '200': description: OK @@ -38,6 +40,14 @@ nodesByDataspaceAndAnchorAndCpsPath: examples: dataSample: $ref: 'components.yml#/components/examples/dataSample' + application/xml: + schema: + type: object + xml: + name: stores + examples: + dataSample: + $ref: 'components.yml#/components/examples/dataSampleXml' '400': $ref: 'components.yml#/components/responses/BadRequest' '403': diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java index 547be669ae..6823f6b03e 100644 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/QueryRestController.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2021-2024 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada. - * Modifications Copyright (C) 2022-2023 TechMahindra Ltd. + * Modifications Copyright (C) 2022-2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,9 +36,11 @@ import org.onap.cps.spi.FetchDescendantsOption; import org.onap.cps.spi.PaginationOption; import org.onap.cps.spi.model.Anchor; import org.onap.cps.spi.model.DataNode; +import org.onap.cps.utils.ContentType; import org.onap.cps.utils.DataMapUtils; import org.onap.cps.utils.JsonObjectMapper; import org.onap.cps.utils.PrefixResolver; +import org.onap.cps.utils.XmlFileUtils; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RequestMapping; @@ -62,18 +64,20 @@ public class QueryRestController implements CpsQueryApi { final FetchDescendantsOption fetchDescendantsOption = Boolean.TRUE.equals(includeDescendants) ? FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS : FetchDescendantsOption.OMIT_DESCENDANTS; return executeNodesByDataspaceQueryAndCreateResponse(dataspaceName, anchorName, cpsPath, - fetchDescendantsOption); + fetchDescendantsOption, ContentType.JSON); } @Override @Timed(value = "cps.data.controller.datanode.query.v2", description = "Time taken to query data nodes") public ResponseEntity<Object> getNodesByDataspaceAndAnchorAndCpsPathV2(final String dataspaceName, - final String anchorName, final String cpsPath, final String fetchDescendantsOptionAsString) { + final String anchorName, final String contentTypeInHeader, final String cpsPath, + final String fetchDescendantsOptionAsString) { + final ContentType contentType = ContentType.fromString(contentTypeInHeader); final FetchDescendantsOption fetchDescendantsOption = FetchDescendantsOption.getFetchDescendantsOption(fetchDescendantsOptionAsString); return executeNodesByDataspaceQueryAndCreateResponse(dataspaceName, anchorName, cpsPath, - fetchDescendantsOption); + fetchDescendantsOption, contentType); } @Override @@ -130,7 +134,8 @@ public class QueryRestController implements CpsQueryApi { } private ResponseEntity<Object> executeNodesByDataspaceQueryAndCreateResponse(final String dataspaceName, - final String anchorName, final String cpsPath, final FetchDescendantsOption fetchDescendantsOption) { + final String anchorName, final String cpsPath, final FetchDescendantsOption fetchDescendantsOption, + final ContentType contentType) { final Collection<DataNode> dataNodes = cpsQueryService.queryDataNodes(dataspaceName, anchorName, cpsPath, fetchDescendantsOption); final List<Map<String, Object>> dataNodesAsListOfMaps = new ArrayList<>(dataNodes.size()); @@ -143,6 +148,17 @@ public class QueryRestController implements CpsQueryApi { final Map<String, Object> dataMap = DataMapUtils.toDataMapWithIdentifier(dataNode, prefix); dataNodesAsListOfMaps.add(dataMap); } - return new ResponseEntity<>(jsonObjectMapper.asJsonString(dataNodesAsListOfMaps), HttpStatus.OK); + return buildResponseEntity(dataNodesAsListOfMaps, contentType); + } + + private ResponseEntity<Object> buildResponseEntity(final List<Map<String, Object>> dataNodesAsListOfMaps, + final ContentType contentType) { + final String responseData; + if (contentType == ContentType.XML) { + responseData = XmlFileUtils.convertDataMapsToXml(dataNodesAsListOfMaps); + } else { + responseData = jsonObjectMapper.asJsonString(dataNodesAsListOfMaps); + } + return new ResponseEntity<>(responseData, HttpStatus.OK); } } diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy index 80b287cda8..076ab32454 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/QueryRestControllerSpec.groovy @@ -3,7 +3,7 @@ * Copyright (C) 2021-2024 Nordix Foundation * Modifications Copyright (C) 2021-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech - * Modifications Copyright (C) 2022-2023 TechMahindra Ltd. + * Modifications Copyright (C) 2022-2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest import org.springframework.http.HttpStatus +import org.springframework.http.MediaType import org.springframework.test.web.servlet.MockMvc import spock.lang.Specification @@ -97,26 +98,52 @@ class QueryRestControllerSpec extends Specification { 'descendants' | 'true' || INCLUDE_ALL_DESCENDANTS } - def 'Query data node v2 api by cps path for the given dataspace and anchor with #scenario.'() { + def 'Query data node v2 API by cps path for the given dataspace and anchor with #scenario and media type JSON'() { given: 'service method returns a list containing a data node' - def dataNode1 = new DataNodeBuilder().withXpath('/xpath') + def dataNode = new DataNodeBuilder().withXpath('/xpath') .withLeaves([leaf: 'value', leafList: ['leaveListElement1', 'leaveListElement2']]).build() - mockCpsQueryService.queryDataNodes(dataspaceName, anchorName, cpsPath, { descendantsOption -> { - assert descendantsOption.depth == expectedDepth}}) >> [dataNode1, dataNode1] + mockCpsQueryService.queryDataNodes(dataspaceName, anchorName, cpsPath, { descendantsOption -> + assert descendantsOption.depth == expectedDepth + }) >> [dataNode, dataNode] when: 'query data nodes API is invoked' def response = mvc.perform( - get(dataNodeEndpointV2) - .param('cps-path', cpsPath) - .param('descendants', includeDescendantsOptionString)) - .andReturn().response - then: 'the response contains the the datanode in json format' + get(dataNodeEndpointV2) + .contentType(MediaType.APPLICATION_JSON) + .param('cps-path', cpsPath) + .param('descendants', includeDescendantsOptionString)) + .andReturn().response + then: 'the response contains the datanode in the expected JSON format' assert response.status == HttpStatus.OK.value() assert response.getContentAsString().contains('{"xpath":{"leaf":"value","leafList":["leaveListElement1","leaveListElement2"]}}') - where: 'the following options for include descendants are provided in the request' - scenario | includeDescendantsOptionString || expectedDepth - 'direct children' | 'direct' || 1 - 'descendants' | '2' || 2 + where: 'the following options for include descendants are provided in the request' + scenario | includeDescendantsOptionString || expectedDepth + 'direct children' | 'direct' || 1 + 'descendants' | '2' || 2 + } + + def 'Query data node v2 API by cps path for the given dataspace and anchor with #scenario and media type XML'() { + given: 'service method returns a list containing a data node' + def dataNode = new DataNodeBuilder().withXpath('/xpath') + .withLeaves([leaf: 'value', leafList: ['leaveListElement1', 'leaveListElement2']]).build() + mockCpsQueryService.queryDataNodes(dataspaceName, anchorName, cpsPath, { descendantsOption -> + assert descendantsOption.depth == expectedDepth + }) >> [dataNode, dataNode] + when: 'query data nodes API is invoked' + def response = + mvc.perform( + get(dataNodeEndpointV2) + .contentType(MediaType.APPLICATION_XML) + .param('cps-path', cpsPath) + .param('descendants', includeDescendantsOptionString)) + .andReturn().response + then: 'the response contains the datanode in the expected XML format' + assert response.status == HttpStatus.OK.value() + assert response.getContentAsString().contains('<xpath><leaf>value</leaf><leafList>leaveListElement1</leafList><leafList>leaveListElement2</leafList></xpath>') + where: 'the following options for include descendants are provided in the request' + scenario | includeDescendantsOptionString || expectedDepth + 'direct children' | 'direct' || 1 + 'descendants' | '2' || 2 } def 'Query data node by cps path for the given dataspace across all anchors with #scenario.'() { diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java index 6f491ba3b7..3368aee148 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsModulePersistenceServiceImpl.java @@ -27,6 +27,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; +import io.micrometer.core.annotation.Timed; import jakarta.transaction.Transactional; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -186,6 +187,8 @@ public class CpsModulePersistenceServiceImpl implements CpsModulePersistenceServ // can occur in case of specific concurrent requests. @Retryable(retryFor = DuplicatedYangResourceException.class, maxAttempts = 5, backoff = @Backoff(random = true, delay = 200, maxDelay = 2000, multiplier = 2)) + @Timed(value = "cps.module.persistence.schemaset.store", + description = "Time taken to store a schemaset (list of module references") public void storeSchemaSetFromModules(final String dataspaceName, final String schemaSetName, final Map<String, String> newModuleNameToContentMap, final Collection<ModuleReference> allModuleReferences) { diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java index a600b22b61..4063a7f769 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java @@ -171,8 +171,8 @@ public class CpsModuleServiceImpl implements CpsModuleService { return cpsModulePersistenceService.identifyNewModuleReferences(moduleReferencesToCheck); } - @Timed(value = "cps.module.service.module.reference.query", - description = "Time taken to query list of module references") + @Timed(value = "cps.module.service.module.reference.query.by.attribute", + description = "Time taken to query list of module references by attribute (e.g moduleSetTag)") @Override public Collection<ModuleReference> getModuleReferencesByAttribute(final String dataspaceName, final String anchorName, diff --git a/cps-service/src/main/java/org/onap/cps/utils/XmlFileUtils.java b/cps-service/src/main/java/org/onap/cps/utils/XmlFileUtils.java index 94b97bd88f..bbfb7f4d2e 100644 --- a/cps-service/src/main/java/org/onap/cps/utils/XmlFileUtils.java +++ b/cps-service/src/main/java/org/onap/cps/utils/XmlFileUtils.java @@ -189,30 +189,32 @@ public class XmlFileUtils { private static void createXmlElements(final Document document, final Node parentNode, final Map<String, Object> dataMap) { - for (final Map.Entry<String, Object> mapEntry : dataMap.entrySet()) { - if (mapEntry.getValue() instanceof List) { - appendList(document, parentNode, mapEntry); - } else if (mapEntry.getValue() instanceof Map) { - appendMap(document, parentNode, mapEntry); + for (final Map.Entry<String, Object> dataNodeMapEntry : dataMap.entrySet()) { + if (dataNodeMapEntry.getValue() instanceof List) { + appendList(document, parentNode, dataNodeMapEntry); + } else if (dataNodeMapEntry.getValue() instanceof Map) { + appendMap(document, parentNode, dataNodeMapEntry); } else { - appendObject(document, parentNode, mapEntry); + appendObject(document, parentNode, dataNodeMapEntry); } } } private static void appendList(final Document document, final Node parentNode, - final Map.Entry<String, Object> mapEntry) { - final List<Object> list = (List<Object>) mapEntry.getValue(); - if (list.isEmpty()) { - final Element listElement = document.createElement(mapEntry.getKey()); + final Map.Entry<String, Object> dataNodeMapEntry) { + final List<Object> dataNodeMaps = (List<Object>) dataNodeMapEntry.getValue(); + if (dataNodeMaps.isEmpty()) { + final Element listElement = document.createElement(dataNodeMapEntry.getKey()); parentNode.appendChild(listElement); } else { - for (final Object element : list) { - final Element listElement = document.createElement(mapEntry.getKey()); - if (element instanceof Map) { - createXmlElements(document, listElement, (Map<String, Object>) element); + for (final Object dataNodeMap : dataNodeMaps) { + final Element listElement = document.createElement(dataNodeMapEntry.getKey()); + if (dataNodeMap == null) { + parentNode.appendChild(listElement); + } else if (dataNodeMap instanceof Map) { + createXmlElements(document, listElement, (Map<String, Object>) dataNodeMap); } else { - listElement.appendChild(document.createTextNode(element.toString())); + listElement.appendChild(document.createTextNode(dataNodeMap.toString())); } parentNode.appendChild(listElement); } @@ -220,16 +222,18 @@ public class XmlFileUtils { } private static void appendMap(final Document document, final Node parentNode, - final Map.Entry<String, Object> mapEntry) { - final Element childElement = document.createElement(mapEntry.getKey()); - createXmlElements(document, childElement, (Map<String, Object>) mapEntry.getValue()); + final Map.Entry<String, Object> dataNodeMapEntry) { + final Element childElement = document.createElement(dataNodeMapEntry.getKey()); + createXmlElements(document, childElement, (Map<String, Object>) dataNodeMapEntry.getValue()); parentNode.appendChild(childElement); } private static void appendObject(final Document document, final Node parentNode, - final Map.Entry<String, Object> mapEntry) { - final Element element = document.createElement(mapEntry.getKey()); - element.appendChild(document.createTextNode(mapEntry.getValue().toString())); + final Map.Entry<String, Object> dataNodeMapEntry) { + final Element element = document.createElement(dataNodeMapEntry.getKey()); + if (dataNodeMapEntry.getValue() != null) { + element.appendChild(document.createTextNode(dataNodeMapEntry.getValue().toString())); + } parentNode.appendChild(element); } diff --git a/cps-service/src/test/groovy/org/onap/cps/utils/XmlFileUtilsSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/utils/XmlFileUtilsSpec.groovy index 3b21145293..9a932c9279 100644 --- a/cps-service/src/test/groovy/org/onap/cps/utils/XmlFileUtilsSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/utils/XmlFileUtilsSpec.groovy @@ -32,7 +32,7 @@ import static org.onap.cps.utils.XmlFileUtils.convertDataMapsToXml class XmlFileUtilsSpec extends Specification { - def 'Parse a valid xml content #scenario'(){ + def 'Parse a valid xml content #scenario'() { given: 'YANG model schema context' def yangResourceNameToContent = TestUtils.getYangResourcesAsMap('bookstore.yang') def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourceNameToContent).getSchemaContext() @@ -41,13 +41,13 @@ class XmlFileUtilsSpec extends Specification { then: 'the result xml is wrapped by root node defined in YANG schema' assert parsedXmlContent == expectedOutput where: - scenario | xmlData || expectedOutput - 'without root data node' | '<?xml version="1.0" encoding="UTF-8"?><class> </class>' || '<?xml version="1.0" encoding="UTF-8"?><stores xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><class> </class></stores>' - 'with root data node' | '<?xml version="1.0" encoding="UTF-8"?><stores><class> </class></stores>' || '<?xml version="1.0" encoding="UTF-8"?><stores><class> </class></stores>' - 'no xml header' | '<stores><class> </class></stores>' || '<stores><class> </class></stores>' + scenario | xmlData || expectedOutput + 'without root data node' | '<?xml version="1.0" encoding="UTF-8"?><class> </class>' || '<?xml version="1.0" encoding="UTF-8"?><stores xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><class> </class></stores>' + 'with root data node' | '<?xml version="1.0" encoding="UTF-8"?><stores><class> </class></stores>' || '<?xml version="1.0" encoding="UTF-8"?><stores><class> </class></stores>' + 'no xml header' | '<stores><class> </class></stores>' || '<stores><class> </class></stores>' } - def 'Parse a invalid xml content'(){ + def 'Parse a invalid xml content'() { given: 'YANG model schema context' def yangResourceNameToContent = TestUtils.getYangResourcesAsMap('bookstore.yang') def schemaContext = YangTextSchemaSourceSetBuilder.of(yangResourceNameToContent).getSchemaContext() @@ -84,9 +84,6 @@ class XmlFileUtilsSpec extends Specification { 'nested XML branch' | [['test-tree': [branch: [name: 'Left', nest: [name: 'Small', birds: 'Sparrow']]]]] || '<test-tree><branch><name>Left</name><nest><name>Small</name><birds>Sparrow</birds></nest></branch></test-tree>' 'list of branch within a test tree' | [['test-tree': [branch: [[name: 'Left', nest: [name: 'Small', birds: 'Sparrow']], [name: 'Right', nest: [name: 'Big', birds: 'Owl']]]]]] || '<test-tree><branch><name>Left</name><nest><name>Small</name><birds>Sparrow</birds></nest></branch><branch><name>Right</name><nest><name>Big</name><birds>Owl</birds></nest></branch></test-tree>' 'list of birds under a nest' | [['nest': ['name': 'Small', 'birds': ['Sparrow']]]] || '<nest><name>Small</name><birds>Sparrow</birds></nest>' - 'XML Content map with null key/value' | [['test-tree': [branch: [name: 'Left', nest: []]]]] || '<test-tree><branch><name>Left</name><nest/></branch></test-tree>' - 'XML Content list is empty' | [['nest': ['name': 'Small', 'birds': []]]] || '<nest><name>Small</name><birds/></nest>' - 'XML with mixed content in list' | [['branch': ['name': 'Left', 'nest': ['name': 'Small', 'birds': ['', 'Sparrow']]]]] || '<branch><name>Left</name><nest><name>Small</name><birds/><birds>Sparrow</birds></nest></branch>' } def 'Convert data maps to XML with null or empty maps and lists'() { @@ -95,11 +92,14 @@ class XmlFileUtilsSpec extends Specification { then: 'the result contains the expected XML or handles nulls correctly' assert result == expectedXmlOutput where: - scenario | dataMaps || expectedXmlOutput - 'null entry in map' | [['branch': []]] || '<branch/>' - 'list with null object' | [['branch': [name: 'Left', nest: [name: 'Small', birds: []]]]] || '<branch><name>Left</name><nest><name>Small</name><birds/></nest></branch>' - 'list containing null list' | [['test-tree': [branch: '']]] || '<test-tree><branch/></test-tree>' - 'nested map with null values' | [['test-tree': [branch: [name: 'Left', nest: '']]]] || '<test-tree><branch><name>Left</name><nest/></branch></test-tree>' + scenario | dataMaps || expectedXmlOutput + 'null entry in map' | [['branch': []]] || '<branch/>' + 'XML Content list is empty' | [['nest': ['name': 'Small', 'birds': [null]]]] || '<nest><name>Small</name><birds/></nest>' + 'XML with mixed content in list' | [['branch': ['name': 'Left', 'nest': ['name': 'Small', 'birds': [null, 'Sparrow']]]]] || '<branch><name>Left</name><nest><name>Small</name><birds/><birds>Sparrow</birds></nest></branch>' + 'list with null object' | [['branch': [name: 'Left', nest: [name: 'Small', birds: [null]]]]] || '<branch><name>Left</name><nest><name>Small</name><birds/></nest></branch>' + 'list containing null values' | [['branch': [null, null, null]]] || '<branch/><branch/><branch/>' + 'nested map with null values' | [['test-tree': [branch: [name: 'Left', nest: null]]]] || '<test-tree><branch><name>Left</name><nest/></branch></test-tree>' + 'mixed list with null values' | [['branch': ['name': 'Left', 'nest': ['name': 'Small', 'birds': [null, 'Sparrow', null]]]]] || '<branch><name>Left</name><nest><name>Small</name><birds/><birds>Sparrow</birds><birds/></nest></branch>' } def 'Converting data maps to xml with no data'() { @@ -109,7 +109,7 @@ class XmlFileUtilsSpec extends Specification { convertDataMapsToXml(dataMapWithNull) then: 'a validation exception is thrown' def exception = thrown(DataValidationException) - and:'the cause is a null pointer exception' + and: 'the cause is a null pointer exception' assert exception.cause instanceof NullPointerException } @@ -120,9 +120,9 @@ class XmlFileUtilsSpec extends Specification { convertDataMapsToXml(dataMap) then: 'a validation exception is thrown' def exception = thrown(DataValidationException) - and:'the cause is a document object model exception' + and: 'the cause is a document object model exception' assert exception.cause instanceof DOMException } -} +}
\ No newline at end of file diff --git a/docs/api/swagger/cps/openapi.yaml b/docs/api/swagger/cps/openapi.yaml index 3f889c1e6c..3b6bd43d6c 100644 --- a/docs/api/swagger/cps/openapi.yaml +++ b/docs/api/swagger/cps/openapi.yaml @@ -2283,6 +2283,15 @@ paths: default: none example: "3" type: string + - description: Content type in header + in: header + name: Content-Type + required: true + schema: + enum: + - application/json + - application/xml + type: string responses: "200": content: @@ -2293,6 +2302,15 @@ paths: value: null schema: type: object + application/xml: + examples: + dataSample: + $ref: '#/components/examples/dataSampleXml' + value: null + schema: + type: object + xml: + name: stores description: OK "400": content: diff --git a/docs/api/swagger/ncmp/openapi-inventory.yaml b/docs/api/swagger/ncmp/openapi-inventory.yaml index d358719a4b..ab83ed2ae6 100644 --- a/docs/api/swagger/ncmp/openapi-inventory.yaml +++ b/docs/api/swagger/ncmp/openapi-inventory.yaml @@ -97,7 +97,7 @@ paths: example: my-dmi-plugin type: string - description: Boolean parameter to determine if returned value(s) will be cm - handle Ids or alternate Ids for a given query + handle ids or alternate ids for a given query in: query name: outputAlternateId required: false @@ -145,7 +145,7 @@ paths: operationId: searchCmHandleIds parameters: - description: Boolean parameter to determine if returned value(s) will be cm - handle Ids or alternate Ids for a given query + handle ids or alternate ids for a given query in: query name: outputAlternateId required: false @@ -202,7 +202,7 @@ components: type: string outputAlternateIdOptionInQuery: description: Boolean parameter to determine if returned value(s) will be cm - handle Ids or alternate Ids for a given query + handle ids or alternate ids for a given query in: query name: outputAlternateId required: false diff --git a/docs/api/swagger/ncmp/openapi.yaml b/docs/api/swagger/ncmp/openapi.yaml index aa84e432e9..e7256c0836 100644 --- a/docs/api/swagger/ncmp/openapi.yaml +++ b/docs/api/swagger/ncmp/openapi.yaml @@ -1130,10 +1130,10 @@ paths: /v1/ch/id-searches: post: description: Execute cm handle query search and return a list of cm handle references. - Any number of conditions can be applied. To be included in the result a cm-handle - must fulfill ALL the conditions. An empty collection will be returned in the - case that the cm handle does not match a condition. For more on cm handle - query search please refer to <a href="https://docs.onap.org/projects/onap-cps/en/latest/ncmp-cmhandle-querying.html">cm + Any number of conditions can be applied. To be included in the result a cm + handle must fulfill ALL the conditions. An empty collection will be returned + in the case that the cm handle does not match a condition. For more on cm + handle query search please refer to <a href="https://docs.onap.org/projects/onap-cps/en/latest/ncmp-cmhandle-querying.html">cm handle query search Read the Docs</a>.<br/>By supplying a CPS Path it is possible to query on any data related to the cm handle. For more on CPS Path please refer to <a href="https://docs.onap.org/projects/onap-cps/en/latest/cps-path.html">CPS @@ -1142,7 +1142,7 @@ paths: operationId: searchCmHandleIds parameters: - description: Boolean parameter to determine if returned value(s) will be cm - handle Ids or alternate Ids for a given query + handle ids or alternate ids for a given query in: query name: outputAlternateId required: false @@ -1619,7 +1619,7 @@ components: type: string outputAlternateIdOptionInQuery: description: Boolean parameter to determine if returned value(s) will be cm - handle Ids or alternate Ids for a given query + handle ids or alternate ids for a given query in: query name: outputAlternateId required: false diff --git a/docs/ncmp-cmhandle-querying.rst b/docs/ncmp-cmhandle-querying.rst index 529297daa7..2e534d87ff 100644 --- a/docs/ncmp-cmhandle-querying.rst +++ b/docs/ncmp-cmhandle-querying.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2022-2023 Nordix Foundation +.. Copyright (C) 2022-2024 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmhandlequerying: @@ -19,9 +19,20 @@ For querying CM Handles we have two Post endpoints: - ncmp/v1/ch/searches Returns all CM Handles which match the query properties provided. Gives a JSON payload of the **details** of all matching CM Handles. -- ncmp/v1/ch/id-searches Returns all CM Handles IDs which match the query properties provided. Gives a JSON payload of the **ids** of all matching CM Handles. +- ncmp/v1/ch/id-searches Returns all CM Handles IDs or Alternate IDs which match the query properties provided. Gives a JSON payload of the **ids** of all matching CM Handles. -/searches returns whole CM Handle object (data) whereas /id-searches returns only CM Handle IDs. Otherwise these endpoints are intended to be functionally identical so both can be queried with the same request body. If no matching CM Handles are found an empty array is returned. +/searches returns whole CM Handle object (data) whereas /id-searches returns only CM Handle IDs or Alternate IDs. Otherwise these endpoints are intended to be functionally identical so both can be queried with the same request body. If no matching CM Handles are found an empty array is returned. + +Parameters +========== + +/id-searches can return either CM Handle IDs or Alternate IDs. This is controlled with an optional parameter outputAlternateId. + +- *outputAlternateId=true* returns Alternate IDs + +- *outputAlternateId=false* returns CM Handle IDs + +Note: Null values will default to false so /id-searches & /id-searches?outputAlternateId will both return CM Handle IDs Request Body ============ diff --git a/docs/ncmp-data-operation.rst b/docs/ncmp-data-operation.rst index 94d5ee9c0a..3352e03cf0 100644 --- a/docs/ncmp-data-operation.rst +++ b/docs/ncmp-data-operation.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. Copyright (C) 2023 Nordix Foundation +.. Copyright (C) 2023-2024 Nordix Foundation .. DO NOT CHANGE THIS LABEL FOR RELEASE NOTES - EVEN THOUGH IT GIVES A WARNING .. _cmHandleDataOperation: @@ -43,7 +43,7 @@ This endpoint executes data operation for given array of operations: | | | implementation. For ONAP DMI Plugin it will be RESTConf paths but it can| | | | really be anything. | +--------------------------+-------------+-------------------------------------------------------------------------+ - | targetIds | Yes | List of cm handle ids. | + | targetIds | Yes | List of cm handle references | +--------------------------+-------------+-------------------------------------------------------------------------+ The status codes used in the events resulting from these operations are defined here: diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index 759eccd966..02a10cfa6b 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -21,6 +21,7 @@ package org.onap.cps.integration.base +import com.hazelcast.collection.ISet import okhttp3.mockwebserver.MockWebServer import org.onap.cps.api.CpsAnchorService import org.onap.cps.api.CpsDataService @@ -37,13 +38,13 @@ import org.onap.cps.ncmp.impl.data.NetworkCmProxyQueryService import org.onap.cps.ncmp.impl.inventory.InventoryPersistence import org.onap.cps.ncmp.impl.inventory.ParameterizedCmHandleQueryService import org.onap.cps.ncmp.impl.inventory.models.CmHandleState +import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncService import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog import org.onap.cps.ncmp.impl.utils.AlternateIdMatcher import org.onap.cps.ri.repository.DataspaceRepository import org.onap.cps.ri.utils.SessionManager import org.onap.cps.spi.exceptions.DataspaceNotFoundException import org.onap.cps.spi.model.DataNode -import org.onap.cps.utils.ContentType import org.onap.cps.utils.JsonObjectMapper import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value @@ -61,13 +62,8 @@ import spock.lang.Specification import spock.util.concurrent.PollingConditions import java.time.OffsetDateTime -import java.time.format.DateTimeFormatter import java.util.concurrent.BlockingQueue -import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DATASPACE_NAME -import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_ANCHOR -import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT - @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, classes = [CpsDataspaceService]) @Testcontainers @EnableAutoConfiguration @@ -121,6 +117,9 @@ abstract class CpsIntegrationSpecBase extends Specification { ModuleSyncWatchdog moduleSyncWatchdog @Autowired + ModuleSyncService moduleSyncService + + @Autowired BlockingQueue<DataNode> moduleSyncWorkQueue @Autowired @@ -132,6 +131,8 @@ abstract class CpsIntegrationSpecBase extends Specification { @Autowired AlternateIdMatcher alternateIdMatcher + @Autowired + ISet<String> moduleSetTagsBeingProcessed @Value('${ncmp.policy-executor.server.port:8080}') private String policyServerPort; @@ -174,13 +175,13 @@ abstract class CpsIntegrationSpecBase extends Specification { DMI1_URL = String.format("http://%s:%s", mockDmiServer1.getHostName(), mockDmiServer1.getPort()) DMI2_URL = String.format("http://%s:%s", mockDmiServer2.getHostName(), mockDmiServer2.getPort()) - } def cleanup() { mockDmiServer1.shutdown() mockDmiServer2.shutdown() mockPolicyServer.shutdown() + moduleSetTagsBeingProcessed.clear() } def static readResourceDataFile(filename) { @@ -262,11 +263,16 @@ abstract class CpsIntegrationSpecBase extends Specification { networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, createdCmHandles: [cmHandleToCreate])) } - def registerSequenceOfCmHandlesWithoutWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles) { + def registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(dmiPlugin, moduleSetTag, numberOfCmHandles, offset) { def cmHandles = [] + def id = offset + def moduleReferences = (1..200).collect { moduleSetTag + '_Module_' + it.toString() } (1..numberOfCmHandles).each { - def cmHandle = new NcmpServiceCmHandle(cmHandleId: 'ch-'+it, moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID) - cmHandles.add(cmHandle) + def ncmpServiceCmHandle = new NcmpServiceCmHandle(cmHandleId: 'ch-'+id, moduleSetTag: moduleSetTag, alternateId: NO_ALTERNATE_ID) + cmHandles.add(ncmpServiceCmHandle) + dmiDispatcher1.moduleNamesPerCmHandleId[ncmpServiceCmHandle.cmHandleId] = moduleReferences + dmiDispatcher2.moduleNamesPerCmHandleId[ncmpServiceCmHandle.cmHandleId] = moduleReferences + id++ } networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, createdCmHandles: cmHandles)) } @@ -279,9 +285,10 @@ abstract class CpsIntegrationSpecBase extends Specification { networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, removedCmHandles: cmHandleIds)) } - def deregisterSequenceOfCmHandles(dmiPlugin, numberOfCmHandles) { + def deregisterSequenceOfCmHandles(dmiPlugin, numberOfCmHandles, offset) { def cmHandleIds = [] - (1..numberOfCmHandles).each { cmHandleIds.add('ch-'+it) } + def id = offset + (1..numberOfCmHandles).each { cmHandleIds.add('ch-' + id++) } networkCmProxyInventoryFacade.updateDmiRegistration(new DmiPluginRegistration(dmiPlugin: dmiPlugin, removedCmHandles: cmHandleIds)) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy index 64449371fe..a5e3daf289 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/CmHandleUpgradeSpec.groovy @@ -33,54 +33,55 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { NetworkCmProxyInventoryFacade objectUnderTest - static final CM_HANDLE_ID = 'ch-1' - static final CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG = 'ch-2' + def cmHandleId = 'ch-1' + def cmHandleIdWithExistingModuleSetTag = 'ch-2' def setup() { objectUnderTest = networkCmProxyInventoryFacade + moduleSyncService.clearPrivateModuleSetCache() } def 'Upgrade CM-handle with new moduleSetTag or no moduleSetTag.'() { given: 'a CM-handle is created with expected initial modules: M1 and M2' - dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2'] - registerCmHandle(DMI1_URL, CM_HANDLE_ID, initialModuleSetTag) - assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort() + dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2'] + registerCmHandle(DMI1_URL, cmHandleId, initialModuleSetTag) + assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort() when: "the CM-handle is upgraded with given moduleSetTag '${updatedModuleSetTag}'" - def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: updatedModuleSetTag) + def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: updatedModuleSetTag) def dmiPluginRegistrationResponse = objectUnderTest.updateDmiRegistration( new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade)) then: 'registration gives successful response' - assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(CM_HANDLE_ID)] + assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)] and: 'CM-handle is in LOCKED state due to MODULE_UPGRADE' - def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID) + def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId) assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE assert cmHandleCompositeState.lockReason.details == "Upgrade to ModuleSetTag: ${updatedModuleSetTag}" when: 'DMI will return different modules for upgrade: M1 and M3' - dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M3'] + dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M3'] and: 'the module sync watchdog is triggered twice' 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } then: 'CM-handle goes to READY state' new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID).cmHandleState + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState }) and: 'the CM-handle has expected moduleSetTag' - assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == updatedModuleSetTag + assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag and: 'CM-handle has expected updated modules: M1 and M3' - assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort() + assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort() cleanup: 'deregister CM-handle' - deregisterCmHandle(DMI1_URL, CM_HANDLE_ID) + deregisterCmHandle(DMI1_URL, cmHandleId) - where: + where: 'following module set tags are used' initialModuleSetTag | updatedModuleSetTag NO_MODULE_SET_TAG | NO_MODULE_SET_TAG NO_MODULE_SET_TAG | 'new' @@ -90,39 +91,39 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { def 'Upgrade CM-handle with existing moduleSetTag.'() { given: 'DMI will return modules for registration' - dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2'] - dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG] = ['M1', 'M3'] + dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2'] + dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleIdWithExistingModuleSetTag] = ['M1', 'M3'] and: "an existing CM-handle handle with moduleSetTag '${updatedModuleSetTag}'" - registerCmHandle(DMI1_URL, CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG, updatedModuleSetTag) - assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG).moduleName.sort() + registerCmHandle(DMI1_URL, cmHandleIdWithExistingModuleSetTag, updatedModuleSetTag) + assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleIdWithExistingModuleSetTag).moduleName.sort() and: "a CM-handle with moduleSetTag '${initialModuleSetTag}' which will be upgraded" - registerCmHandle(DMI1_URL, CM_HANDLE_ID, initialModuleSetTag) - assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort() + registerCmHandle(DMI1_URL, cmHandleId, initialModuleSetTag) + assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort() when: "CM-handle is upgraded to moduleSetTag '${updatedModuleSetTag}'" - def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: updatedModuleSetTag) + def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: updatedModuleSetTag) def dmiPluginRegistrationResponse = objectUnderTest.updateDmiRegistration( new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade)) then: 'registration gives successful response' - assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(CM_HANDLE_ID)] + assert dmiPluginRegistrationResponse.upgradedCmHandles == [CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)] and: 'the module sync watchdog is triggered twice' 2.times { moduleSyncWatchdog.moduleSyncAdvisedCmHandles() } and: 'CM-handle goes to READY state' new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID).cmHandleState + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState }) and: 'the CM-handle has expected moduleSetTag' - assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == updatedModuleSetTag + assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == updatedModuleSetTag and: 'CM-handle has expected updated modules: M1 and M3' - assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort() + assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort() cleanup: 'deregister CM-handle' - deregisterCmHandles(DMI1_URL, [CM_HANDLE_ID, CM_HANDLE_ID_WITH_EXISTING_MODULE_SET_TAG]) + deregisterCmHandles(DMI1_URL, [cmHandleId, cmHandleIdWithExistingModuleSetTag]) where: initialModuleSetTag | updatedModuleSetTag @@ -132,37 +133,37 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { def 'Skip upgrade of CM-handle with same moduleSetTag as before.'() { given: 'an existing CM-handle with expected initial modules: M1 and M2' - dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2'] - registerCmHandle(DMI1_URL, CM_HANDLE_ID, 'same') - assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort() + dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2'] + registerCmHandle(DMI1_URL, cmHandleId, 'same') + assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort() when: 'CM-handle is upgraded with the same moduleSetTag' - def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: 'same') + def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: 'same') objectUnderTest.updateDmiRegistration( new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade)) then: 'CM-handle remains in READY state' - assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID).cmHandleState + assert CmHandleState.READY == objectUnderTest.getCmHandleCompositeState(cmHandleId).cmHandleState and: 'the CM-handle has same moduleSetTag as before' - assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == 'same' + assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == 'same' then: 'CM-handle has same modules as before: M1 and M2' - assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort() + assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(cmHandleId).moduleName.sort() cleanup: 'deregister CM-handle' - deregisterCmHandle(DMI1_URL, CM_HANDLE_ID) + deregisterCmHandle(DMI1_URL, cmHandleId) } def 'Upgrade of CM-handle fails due to DMI error.'() { given: 'a CM-handle exists' - dmiDispatcher1.moduleNamesPerCmHandleId[CM_HANDLE_ID] = ['M1', 'M2'] - registerCmHandle(DMI1_URL, CM_HANDLE_ID, 'oldTag') + dmiDispatcher1.moduleNamesPerCmHandleId[cmHandleId] = ['M1', 'M2'] + registerCmHandle(DMI1_URL, cmHandleId, 'oldTag') and: 'DMI is not available for upgrade' dmiDispatcher1.isAvailable = false when: 'the CM-handle is upgraded' - def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: 'newTag') + def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [cmHandleId], moduleSetTag: 'newTag') objectUnderTest.updateDmiRegistration( new DmiPluginRegistration(dmiPlugin: DMI1_URL, upgradedCmHandles: cmHandlesToUpgrade)) @@ -171,16 +172,16 @@ class CmHandleUpgradeSpec extends CpsIntegrationSpecBase { then: 'CM-handle goes to LOCKED state with reason MODULE_UPGRADE_FAILED' new PollingConditions().within(MODULE_SYNC_WAIT_TIME_IN_SECONDS, () -> { - def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(CM_HANDLE_ID) + def cmHandleCompositeState = objectUnderTest.getCmHandleCompositeState(cmHandleId) assert cmHandleCompositeState.cmHandleState == CmHandleState.LOCKED assert cmHandleCompositeState.lockReason.lockReasonCategory == LockReasonCategory.MODULE_UPGRADE_FAILED }) and: 'the CM-handle has same moduleSetTag as before' - assert objectUnderTest.getNcmpServiceCmHandle(CM_HANDLE_ID).moduleSetTag == 'oldTag' + assert objectUnderTest.getNcmpServiceCmHandle(cmHandleId).moduleSetTag == 'oldTag' cleanup: 'deregister CM-handle' - deregisterCmHandle(DMI1_URL, CM_HANDLE_ID) + deregisterCmHandle(DMI1_URL, cmHandleId) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy index e0bb437a7c..963bc1fe61 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/ModuleSyncWatchdogIntegrationSpec.groovy @@ -20,26 +20,32 @@ package org.onap.cps.integration.functional.ncmp +import io.micrometer.core.instrument.MeterRegistry import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.impl.inventory.sync.ModuleSyncWatchdog +import org.springframework.beans.factory.annotation.Autowired +import spock.util.concurrent.PollingConditions import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { ModuleSyncWatchdog objectUnderTest + @Autowired + MeterRegistry meterRegistry + def executorService = Executors.newFixedThreadPool(2) - def SYNC_SAMPLE_SIZE = 100 + def PARALLEL_SYNC_SAMPLE_SIZE = 100 def setup() { objectUnderTest = moduleSyncWatchdog - registerSequenceOfCmHandlesWithoutWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, SYNC_SAMPLE_SIZE) } def cleanup() { try { - deregisterSequenceOfCmHandles(DMI1_URL, SYNC_SAMPLE_SIZE) + deregisterSequenceOfCmHandles(DMI1_URL, PARALLEL_SYNC_SAMPLE_SIZE, 1) moduleSyncWorkQueue.clear() } finally { executorService.shutdownNow() @@ -47,15 +53,60 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { } def 'Watchdog is disabled for test.'() { + given: + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, PARALLEL_SYNC_SAMPLE_SIZE, 1) when: 'wait a while but less then the initial delay of 10 minutes' Thread.sleep(3000) then: 'the work queue remains empty' assert moduleSyncWorkQueue.isEmpty() } + def 'CPS-2478 Highlight module sync inefficiencies.'() { + given: 'register 250 cm handles with module set tag cps-2478-A' + def numberOfTags = 2 + def cmHandlesPerTag = 250 + def totalCmHandles = numberOfTags * cmHandlesPerTag + def offset = 1 + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-A', cmHandlesPerTag, offset) + and: 'register anther 250 cm handles with module set tag cps-2478-B' + offset += cmHandlesPerTag + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, 'cps-2478-B', cmHandlesPerTag, offset) + and: 'clear any previous instrumentation' + meterRegistry.clear() + when: 'sync all advised cm handles' + objectUnderTest.moduleSyncAdvisedCmHandles() + Thread.sleep(100) + then: 'retry until all schema sets are stored in db (1 schema set for each cm handle)' + def dbSchemaSetStorageTimer = meterRegistry.get('cps.module.persistence.schemaset.store').timer() + new PollingConditions().within(10, () -> { + objectUnderTest.moduleSyncAdvisedCmHandles() + Thread.sleep(100) + assert dbSchemaSetStorageTimer.count() >= 500 + }) + then: 'wait till at least 5 batches of state updates are done (often more because of retries of locked cm handles)' + def dbStateUpdateTimer = meterRegistry.get('cps.ncmp.cmhandle.state.update.batch').timer() + new PollingConditions().within(10, () -> { + assert dbStateUpdateTimer.count() >= 5 + }) + and: 'the db has been queried for tags exactly 2 times.' + def dbModuleQueriesTimer = meterRegistry.get('cps.module.service.module.reference.query.by.attribute').timer() + assert dbModuleQueriesTimer.count() == 2 + and: 'exactly 2 calls to DMI to get module references' + def dmiModuleRetrievalTimer = meterRegistry.get('cps.ncmp.inventory.module.references.from.dmi').timer() + assert dmiModuleRetrievalTimer.count() == 2 + and: 'log the relevant instrumentation' + logInstrumentation(dbModuleQueriesTimer, 'query module references') + logInstrumentation(dmiModuleRetrievalTimer, 'get modules from DMI ') + logInstrumentation(dbSchemaSetStorageTimer, 'store schema sets ') + logInstrumentation(dbStateUpdateTimer, 'batch state updates ') + cleanup: 'remove all cm handles' + deregisterSequenceOfCmHandles(DMI1_URL, totalCmHandles, 1) + } + def 'Populate module sync work queue simultaneously on two parallel threads (CPS-2403).'() { // This test failed before bug https://lf-onap.atlassian.net/browse/CPS-2403 was fixed given: 'the queue is empty at the start' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, PARALLEL_SYNC_SAMPLE_SIZE, 1) assert moduleSyncWorkQueue.isEmpty() when: 'attempt to populate the queue on the main (test) and another parallel thread at the same time' objectUnderTest.populateWorkQueueIfNeeded() @@ -63,12 +114,13 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { and: 'wait a little (to give all threads time to complete their task)' Thread.sleep(50) then: 'the queue size is exactly the sample size' - assert moduleSyncWorkQueue.size() == SYNC_SAMPLE_SIZE + assert moduleSyncWorkQueue.size() == PARALLEL_SYNC_SAMPLE_SIZE } def 'Populate module sync work queue on two parallel threads with a slight difference in start time.'() { // This test proved that the issue in CPS-2403 did not arise if the the queue was populated and given time to be distributed given: 'the queue is empty at the start' + registerSequenceOfCmHandlesWithManyModuleReferencesButDoNotWaitForReady(DMI1_URL, NO_MODULE_SET_TAG, PARALLEL_SYNC_SAMPLE_SIZE, 1) assert moduleSyncWorkQueue.isEmpty() when: 'attempt to populate the queue on the main (test) and another parallel thread a little later' objectUnderTest.populateWorkQueueIfNeeded() @@ -76,7 +128,12 @@ class ModuleSyncWatchdogIntegrationSpec extends CpsIntegrationSpecBase { and: 'wait a little (to give all threads time to complete their task)' Thread.sleep(50) then: 'the queue size is exactly the sample size' - assert moduleSyncWorkQueue.size() == SYNC_SAMPLE_SIZE + assert moduleSyncWorkQueue.size() == PARALLEL_SYNC_SAMPLE_SIZE + } + + def logInstrumentation(timer, description) { + System.out.println('*** CPS-2478, ' + description + ' : ' + timer.count()+ ' times, total ' + timer.totalTime(TimeUnit.MILLISECONDS) + ' ms') + return true } def populateQueueWithoutDelay = () -> { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy index 56d4bfaee4..f897393a53 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/ncmp/PolicyExecutorIntegrationSpec.groovy @@ -43,9 +43,7 @@ class PolicyExecutorIntegrationSpec extends CpsIntegrationSpecBase { } def cleanup() { - deregisterCmHandle(DMI1_URL, 'ch-1') - deregisterCmHandle(DMI1_URL, 'ch-2') - deregisterCmHandle(DMI1_URL, 'ch-3') + deregisterSequenceOfCmHandles(DMI1_URL, 3, 1) } def 'Policy Executor create request with #scenario.'() { diff --git a/integration-test/src/test/java/org/onap/cps/integration/MicroMeterTestConfig.java b/integration-test/src/test/java/org/onap/cps/integration/MicroMeterTestConfig.java new file mode 100644 index 0000000000..3b26f42c8a --- /dev/null +++ b/integration-test/src/test/java/org/onap/cps/integration/MicroMeterTestConfig.java @@ -0,0 +1,41 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.integration; + +import io.micrometer.core.aop.TimedAspect; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MicroMeterTestConfig { + @Bean + public MeterRegistry meterRegistry() { + return new SimpleMeterRegistry(); // Use a simple in-memory registry for testing + } + + @Bean + public TimedAspect timedAspect(final MeterRegistry meterRegistry) { + return new TimedAspect(meterRegistry); + } +} + diff --git a/integration-test/src/test/resources/application.yml b/integration-test/src/test/resources/application.yml index b786a3d4c5..30598dfb90 100644 --- a/integration-test/src/test/resources/application.yml +++ b/integration-test/src/test/resources/application.yml @@ -191,7 +191,7 @@ ncmp: modules-sync-watchdog: async-executor: - parallelism-level: 1 + parallelism-level: 2 model-loader: maximum-attempt-count: 20 |