diff options
Diffstat (limited to 'catalog-model/src/main/java/org/openecomp/sdc/be/model/cache')
12 files changed, 1647 insertions, 1635 deletions
diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationCache.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationCache.java index bb8a1b0129..e9b7f7a2e5 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationCache.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationCache.java @@ -20,16 +20,15 @@ package org.openecomp.sdc.be.model.cache; -import java.util.Map; - +import fj.data.Either; import org.openecomp.sdc.be.dao.titan.TitanOperationStatus; -import fj.data.Either; +import java.util.Map; public interface ApplicationCache<T> { - public abstract Either<Map<String, T>, TitanOperationStatus> getAll(); + public abstract Either<Map<String, T>, TitanOperationStatus> getAll(); - public abstract Either<T, TitanOperationStatus> get(String uniqueId); + public abstract Either<T, TitanOperationStatus> get(String uniqueId); } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java index 4c8d239704..94992111d3 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java @@ -20,23 +20,7 @@ package org.openecomp.sdc.be.model.cache; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import javax.annotation.Resource; - +import fj.data.Either; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.commons.lang3.tuple.ImmutablePair; import org.openecomp.sdc.be.config.BeEcompErrorManager; @@ -45,290 +29,300 @@ import org.openecomp.sdc.be.config.Configuration.ApplicationL1CacheConfig; import org.openecomp.sdc.be.config.Configuration.ApplicationL1CacheInfo; import org.openecomp.sdc.be.config.ConfigurationManager; import org.openecomp.sdc.be.dao.titan.TitanOperationStatus; +import org.openecomp.sdc.be.datatypes.elements.DataTypeDataDefinition; import org.openecomp.sdc.be.model.DataTypeDefinition; import org.openecomp.sdc.be.model.operations.impl.PropertyOperation; import org.openecomp.sdc.be.resources.data.DataTypeData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.openecomp.sdc.common.log.wrappers.Logger; import org.springframework.stereotype.Component; -import fj.data.Either; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; @Component("application-datatype-cache") public class ApplicationDataTypeCache implements ApplicationCache<DataTypeDefinition>, Runnable { + private static final String APPLICATION_DATA_TYPES_CACHE = "ApplicationDataTypesCache"; private final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); - private final Lock r = rwl.readLock(); - private final Lock w = rwl.writeLock(); - - private Map<String, DataTypeDefinition> data = new HashMap<>(); - - private ScheduledExecutorService scheduledPollingService = Executors.newScheduledThreadPool(1, - new BasicThreadFactory.Builder().namingPattern("ApplicationDataTypeCacheThread-%d").build()); - ScheduledFuture<?> scheduledFuture = null; - - private static Logger log = LoggerFactory.getLogger(ApplicationDataTypeCache.class.getName()); - - private int firstRunDelayInSec = 30; - private int pollingIntervalInSec = 60; - - @Resource - private PropertyOperation propertyOperation; - - @PostConstruct - public void init() { - - ApplicationL1CacheConfig applicationL1CacheConfig = ConfigurationManager.getConfigurationManager() - .getConfiguration().getApplicationL1Cache(); - if (applicationL1CacheConfig != null) { - if (applicationL1CacheConfig.getDatatypes() != null) { - ApplicationL1CacheInfo datatypesInfo = applicationL1CacheConfig.getDatatypes(); - if (datatypesInfo.getEnabled()) { - Integer intervalInSec = datatypesInfo.getPollIntervalInSec(); - if (intervalInSec != null) { - pollingIntervalInSec = intervalInSec; - } - Integer firstRunDelay = datatypesInfo.getFirstRunDelay(); - if (firstRunDelay != null) { - firstRunDelayInSec = firstRunDelay; - } - log.trace("ApplicationDataTypesCache polling interval is {} seconds.", pollingIntervalInSec); - if (scheduledPollingService != null) { - log.debug("Start ApplicationDataTypeCache polling task. polling interval {} seconds", - pollingIntervalInSec); - scheduledFuture = scheduledPollingService.scheduleAtFixedRate(this, firstRunDelayInSec, - pollingIntervalInSec, TimeUnit.SECONDS); - } - - } - } else { - BeEcompErrorManager.getInstance().logInternalFlowError("ApplicationDataTypesCache", "Cache is disabled", - ErrorSeverity.INFO); - } - } else { - BeEcompErrorManager.getInstance().logInternalFlowError("ApplicationDataTypesCache", "Cache is disabled", - ErrorSeverity.INFO); - } - - } - - @PreDestroy - void destroy() { - - if (scheduledFuture != null) { - boolean result = scheduledFuture.cancel(true); - log.debug("Stop polling task. result = {}", result); - - scheduledFuture = null; - } - shutdownExecutor(); - } - - private void shutdownExecutor() { - if (scheduledPollingService == null) - return; - - scheduledPollingService.shutdown(); // Disable new tasks from being - // submitted - try { - // Wait a while for existing tasks to terminate - if (!scheduledPollingService.awaitTermination(60, TimeUnit.SECONDS)) { - scheduledPollingService.shutdownNow(); // Cancel currently - // executing tasks - // Wait a while for tasks to respond to being cancelled - if (!scheduledPollingService.awaitTermination(60, TimeUnit.SECONDS)) - log.debug("Pool did not terminate"); - } - } catch (InterruptedException ie) { - // (Re-)Cancel if current thread also interrupted - scheduledPollingService.shutdownNow(); - // Preserve interrupt status - Thread.currentThread().interrupt(); - } - } - - private Either<Map<String, DataTypeDefinition>, TitanOperationStatus> getAllDataTypesFromGraph() { - - Either<Map<String, DataTypeDefinition>, TitanOperationStatus> allDataTypes = propertyOperation - .getAllDataTypes(); - - return allDataTypes; - - } - - @Override - public Either<Map<String, DataTypeDefinition>, TitanOperationStatus> getAll() { - - try { - - r.lock(); - if (data == null || data.isEmpty()) { - return getAllDataTypesFromGraph(); - } - - return Either.left(data); - - } finally { - r.unlock(); - } - } - - @Override - public Either<DataTypeDefinition, TitanOperationStatus> get(String uniqueId) { - - try { - r.lock(); - - if (data == null || data.isEmpty()) { - Either<DataTypeDefinition, TitanOperationStatus> dataTypeByUid = propertyOperation - .getDataTypeByUid(uniqueId); - return dataTypeByUid; - } else { - DataTypeDefinition dataTypeDefinition = data.values().stream() - .filter(p -> p.getUniqueId().equals(uniqueId)).findFirst().orElse(null); - if (dataTypeDefinition == null) { - Either<DataTypeDefinition, TitanOperationStatus> dataTypeByUid = propertyOperation - .getDataTypeByUid(uniqueId); - return dataTypeByUid; - } else { - return Either.left(dataTypeDefinition); - } - } - } finally { - r.unlock(); - } - } - - @Override - public void run() { - log.trace("run() method. polling db to fetch data types"); - - try { - - Long start = System.currentTimeMillis(); - log.trace("Start fetching all data types from db"); - Either<List<DataTypeData>, TitanOperationStatus> allDataTypeNodes = propertyOperation.getAllDataTypeNodes(); - Long end = System.currentTimeMillis(); - log.trace("Finish fetching all data types from db. Took {} Milliseconds", (end - start)); - if (allDataTypeNodes.isRight()) { - TitanOperationStatus status = allDataTypeNodes.right().value(); - if (status != TitanOperationStatus.OK) { - log.debug("ApplicationDataTypesCache - Failed to fetch all data types nodes"); - BeEcompErrorManager.getInstance().logInternalConnectionError("FetchDataTypes", - "Failed to fetch data types from graph(cache)", ErrorSeverity.INFO); - } - } else { - - List<DataTypeData> list = allDataTypeNodes.left().value(); - if (list != null) { - - Map<String, ImmutablePair<Long, Long>> dataTypeNameToModificationTime = list.stream() - .collect(Collectors.toMap(p -> p.getDataTypeDataDefinition().getName(), - p -> new ImmutablePair<Long, Long>(p.getDataTypeDataDefinition().getCreationTime(), - p.getDataTypeDataDefinition().getModificationTime()))); - - Map<String, ImmutablePair<Long, Long>> currentDataTypeToModificationTime = new HashMap<>(); - try { - r.lock(); - if (data != null) { - currentDataTypeToModificationTime = data.values().stream().collect(Collectors.toMap( - p -> p.getName(), - p -> new ImmutablePair<Long, Long>(p.getCreationTime(), p.getModificationTime()))); - - } - } finally { - r.unlock(); - } - - boolean isChanged = compareDataTypes(dataTypeNameToModificationTime, - currentDataTypeToModificationTime); - if (isChanged) { - replaceAllData(); - } - - } - } - - } catch (Exception e) { - log.debug("unexpected error occured", e); - - BeEcompErrorManager.getInstance().logInternalUnexpectedError("ApplicationDataTypesCache", - "Failed to run refresh data types job", ErrorSeverity.INFO); - } finally { - try { - propertyOperation.getTitanGenericDao().commit(); - } catch (Exception e) { - log.trace("Failed to commit ApplicationDataTypeCache", e); - } - } - - } - - private boolean compareDataTypes(Map<String, ImmutablePair<Long, Long>> dataTypeNameToModificationTime, - Map<String, ImmutablePair<Long, Long>> currentDataTypeToModificationTime) { - if (dataTypeNameToModificationTime.size() != currentDataTypeToModificationTime.size()) { - return true; - } else { - - Set<String> currentkeySet = currentDataTypeToModificationTime.keySet(); - Set<String> keySet = dataTypeNameToModificationTime.keySet(); - - if (currentkeySet.containsAll(keySet)) { - - for (Entry<String, ImmutablePair<Long, Long>> entry : dataTypeNameToModificationTime.entrySet()) { - String dataTypeName = entry.getKey(); - ImmutablePair<Long, Long> creationAndModificationTimes = entry.getValue(); - long creationTime = creationAndModificationTimes.getLeft() == null ? 0 - : creationAndModificationTimes.getLeft().longValue(); - long modificationTime = creationAndModificationTimes.getRight() == null ? 0 - : creationAndModificationTimes.getRight().longValue(); - - ImmutablePair<Long, Long> currentEntry = currentDataTypeToModificationTime.get(dataTypeName); - long currentCreationTime = currentEntry.getLeft() == null ? 0 : currentEntry.getLeft().longValue(); - long currentModificationTime = currentEntry.getRight() == null ? 0 - : currentEntry.getRight().longValue(); - - if (creationTime > currentCreationTime || modificationTime > currentModificationTime) { - log.debug("Datatype {} was updated. Creation Time {} vs {}. Modification Time {} vs {}", - dataTypeName, currentCreationTime, creationTime, currentModificationTime, - modificationTime); - return true; - } - } - } else { - return true; - } - - } - - return false; - } - - private void replaceAllData() { - - Either<Map<String, DataTypeDefinition>, TitanOperationStatus> allDataTypes = propertyOperation - .getAllDataTypes(); - - if (allDataTypes.isRight()) { - TitanOperationStatus status = allDataTypes.right().value(); - log.debug("Failed to fetch all data types from db. Status is {}", status); - } else { - - try { - w.lock(); - - Map<String, DataTypeDefinition> newDataTypes = allDataTypes.left().value(); - data = newDataTypes; - - BeEcompErrorManager.getInstance().logInternalFlowError("ReplaceDataTypesCache", - "Succeed to replace the data types cache", ErrorSeverity.INFO); - - } finally { - w.unlock(); - } - - } - - } + private final Lock r = rwl.readLock(); + private final Lock w = rwl.writeLock(); + + private Map<String, DataTypeDefinition> data = new HashMap<>(); + + private ScheduledExecutorService scheduledPollingService = Executors.newScheduledThreadPool(1, + new BasicThreadFactory.Builder().namingPattern("ApplicationDataTypeCacheThread-%d").build()); + ScheduledFuture<?> scheduledFuture = null; + + private static final Logger log = Logger.getLogger(ApplicationDataTypeCache.class.getName()); + + private int firstRunDelayInSec = 30; + private int pollingIntervalInSec = 60; + + @Resource + private PropertyOperation propertyOperation; + + @PostConstruct + public void init() { + + ApplicationL1CacheConfig applicationL1CacheConfig = ConfigurationManager.getConfigurationManager() + .getConfiguration().getApplicationL1Cache(); + if (applicationL1CacheConfig != null) { + if (applicationL1CacheConfig.getDatatypes() != null) { + ApplicationL1CacheInfo datatypesInfo = applicationL1CacheConfig.getDatatypes(); + if (datatypesInfo.getEnabled()) { + Integer intervalInSec = datatypesInfo.getPollIntervalInSec(); + if (intervalInSec != null) { + pollingIntervalInSec = intervalInSec; + } + Integer firstRunDelay = datatypesInfo.getFirstRunDelay(); + if (firstRunDelay != null) { + firstRunDelayInSec = firstRunDelay; + } + log.trace("ApplicationDataTypesCache polling interval is {} seconds.", pollingIntervalInSec); + if (scheduledPollingService != null) { + log.debug("Start ApplicationDataTypeCache polling task. polling interval {} seconds", + pollingIntervalInSec); + scheduledFuture = scheduledPollingService.scheduleAtFixedRate(this, firstRunDelayInSec, + pollingIntervalInSec, TimeUnit.SECONDS); + } + + } + } else { + BeEcompErrorManager.getInstance().logInternalFlowError(APPLICATION_DATA_TYPES_CACHE, "Cache is disabled", + ErrorSeverity.INFO); + } + } else { + BeEcompErrorManager.getInstance().logInternalFlowError(APPLICATION_DATA_TYPES_CACHE, "Cache is disabled", + ErrorSeverity.INFO); + } + + } + + @PreDestroy + void destroy() { + + if (scheduledFuture != null) { + boolean result = scheduledFuture.cancel(true); + log.debug("Stop polling task. result = {}", result); + + scheduledFuture = null; + } + shutdownExecutor(); + } + + private void shutdownExecutor() { + if (scheduledPollingService == null) + return; + + scheduledPollingService.shutdown(); // Disable new tasks from being + // submitted + try { + // Wait a while for existing tasks to terminate + if (!scheduledPollingService.awaitTermination(60, TimeUnit.SECONDS)) { + scheduledPollingService.shutdownNow(); // Cancel currently + // executing tasks + // Wait a while for tasks to respond to being cancelled + if (!scheduledPollingService.awaitTermination(60, TimeUnit.SECONDS)) + log.debug("Pool did not terminate"); + } + } catch (InterruptedException ie) { + // (Re-)Cancel if current thread also interrupted + scheduledPollingService.shutdownNow(); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + } + + private Either<Map<String, DataTypeDefinition>, TitanOperationStatus> getAllDataTypesFromGraph() { + + return propertyOperation + .getAllDataTypes(); + + } + + @Override + public Either<Map<String, DataTypeDefinition>, TitanOperationStatus> getAll() { + + try { + + r.lock(); + if (data == null || data.isEmpty()) { + return getAllDataTypesFromGraph(); + } + + return Either.left(data); + + } finally { + r.unlock(); + } + } + + @Override + public Either<DataTypeDefinition, TitanOperationStatus> get(String uniqueId) { + + try { + r.lock(); + + if (data == null || data.isEmpty()) { + return propertyOperation + .getDataTypeByUid(uniqueId); + } else { + DataTypeDefinition dataTypeDefinition = data.values().stream() + .filter(p -> p.getUniqueId().equals(uniqueId)).findFirst().orElse(null); + if (dataTypeDefinition == null) { + return propertyOperation + .getDataTypeByUid(uniqueId); + } else { + return Either.left(dataTypeDefinition); + } + } + } finally { + r.unlock(); + } + } + + @Override + public void run() { + log.trace("run() method. polling db to fetch data types"); + + try { + + Long start = System.currentTimeMillis(); + log.trace("Start fetching all data types from db"); + Either<List<DataTypeData>, TitanOperationStatus> allDataTypeNodes = propertyOperation.getAllDataTypeNodes(); + Long end = System.currentTimeMillis(); + log.trace("Finish fetching all data types from db. Took {} Milliseconds", (end - start)); + if (allDataTypeNodes.isRight()) { + TitanOperationStatus status = allDataTypeNodes.right().value(); + if (status != TitanOperationStatus.OK) { + log.debug("ApplicationDataTypesCache - Failed to fetch all data types nodes"); + BeEcompErrorManager.getInstance().logInternalConnectionError("FetchDataTypes", + "Failed to fetch data types from graph(cache)", ErrorSeverity.INFO); + } + } else { + + List<DataTypeData> list = allDataTypeNodes.left().value(); + if (list != null) { + + Map<String, ImmutablePair<Long, Long>> dataTypeNameToModificationTime = list.stream() + .collect(Collectors.toMap(p -> p.getDataTypeDataDefinition().getName(), + p -> new ImmutablePair<>(p.getDataTypeDataDefinition().getCreationTime(), + p.getDataTypeDataDefinition().getModificationTime()))); + + Map<String, ImmutablePair<Long, Long>> currentDataTypeToModificationTime = new HashMap<>(); + try { + r.lock(); + if (data != null) { + currentDataTypeToModificationTime = data.values().stream().collect(Collectors.toMap( + DataTypeDataDefinition::getName, + p -> new ImmutablePair<>(p.getCreationTime(), p.getModificationTime()))); + + } + } finally { + r.unlock(); + } + + boolean isChanged = compareDataTypes(dataTypeNameToModificationTime, + currentDataTypeToModificationTime); + if (isChanged) { + replaceAllData(); + } + + } + } + + } catch (Exception e) { + log.debug("unexpected error occured", e); + + BeEcompErrorManager.getInstance().logInternalUnexpectedError(APPLICATION_DATA_TYPES_CACHE, + "Failed to run refresh data types job", ErrorSeverity.INFO); + } finally { + try { + propertyOperation.getTitanGenericDao().commit(); + } catch (Exception e) { + log.trace("Failed to commit ApplicationDataTypeCache", e); + } + } + + } + + private boolean compareDataTypes(Map<String, ImmutablePair<Long, Long>> dataTypeNameToModificationTime, + Map<String, ImmutablePair<Long, Long>> currentDataTypeToModificationTime) { + if (dataTypeNameToModificationTime.size() != currentDataTypeToModificationTime.size()) { + return true; + } else { + + Set<String> currentkeySet = currentDataTypeToModificationTime.keySet(); + Set<String> keySet = dataTypeNameToModificationTime.keySet(); + + if (currentkeySet.containsAll(keySet)) { + + for (Entry<String, ImmutablePair<Long, Long>> entry : dataTypeNameToModificationTime.entrySet()) { + String dataTypeName = entry.getKey(); + ImmutablePair<Long, Long> creationAndModificationTimes = entry.getValue(); + long creationTime = creationAndModificationTimes.getLeft() == null ? 0 + : creationAndModificationTimes.getLeft().longValue(); + long modificationTime = creationAndModificationTimes.getRight() == null ? 0 + : creationAndModificationTimes.getRight().longValue(); + + ImmutablePair<Long, Long> currentEntry = currentDataTypeToModificationTime.get(dataTypeName); + long currentCreationTime = currentEntry.getLeft() == null ? 0 : currentEntry.getLeft().longValue(); + long currentModificationTime = currentEntry.getRight() == null ? 0 + : currentEntry.getRight().longValue(); + + if (creationTime > currentCreationTime || modificationTime > currentModificationTime) { + log.debug("Datatype {} was updated. Creation Time {} vs {}. Modification Time {} vs {}", + dataTypeName, currentCreationTime, creationTime, currentModificationTime, + modificationTime); + return true; + } + } + } else { + return true; + } + + } + + return false; + } + + private void replaceAllData() { + + Either<Map<String, DataTypeDefinition>, TitanOperationStatus> allDataTypes = propertyOperation + .getAllDataTypes(); + + if (allDataTypes.isRight()) { + TitanOperationStatus status = allDataTypes.right().value(); + log.debug("Failed to fetch all data types from db. Status is {}", status); + } else { + + try { + w.lock(); + + data = allDataTypes.left().value(); + + BeEcompErrorManager.getInstance().logInternalFlowError("ReplaceDataTypesCache", + "Succeed to replace the data types cache", ErrorSeverity.INFO); + + } finally { + w.unlock(); + } + + } + + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ComponentCache.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ComponentCache.java index 4fa41e5bee..b17a66434a 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ComponentCache.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ComponentCache.java @@ -36,27 +36,21 @@ import org.openecomp.sdc.be.datatypes.components.ResourceMetadataDataDefinition; import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum; import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum; import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum; -import org.openecomp.sdc.be.model.Component; -import org.openecomp.sdc.be.model.LifecycleStateEnum; -import org.openecomp.sdc.be.model.Product; -import org.openecomp.sdc.be.model.Resource; -import org.openecomp.sdc.be.model.Service; +import org.openecomp.sdc.be.model.*; import org.openecomp.sdc.be.model.jsontitan.operations.ToscaOperationFacade; -import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus; import org.openecomp.sdc.be.resources.data.ComponentCacheData; +import org.openecomp.sdc.common.log.wrappers.Logger; import org.openecomp.sdc.common.util.SerializationUtils; import org.openecomp.sdc.common.util.ZipUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import javax.annotation.PostConstruct; import java.io.IOException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; +import java.util.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.List; import java.util.Map; import java.util.Set; @@ -68,817 +62,853 @@ import java.util.stream.Collectors; @org.springframework.stereotype.Component("component-cache") public class ComponentCache { - private static Logger logger = LoggerFactory.getLogger(ComponentCache.class.getName()); - - @Autowired - ComponentCassandraDao componentCassandraDao; - - @Autowired - ToscaOperationFacade toscaOperationFacade; - - private Map<ComponentTypeEnum, Map<String, Component>> catalogInMemoryCache = new HashMap<>(); - private final ReentrantReadWriteLock rwCatalogLock = new ReentrantReadWriteLock(); - private final Lock rCatalogLock = rwCatalogLock.readLock(); - private final Lock wCatalogLock = rwCatalogLock.writeLock(); - - boolean enabled = true; - int catalogInMemorySizePerResource = 300; - int catalogInMemorySizePerService = 200; - int catalogInMemorySizePerProduct = 100; - boolean catalogInMemoryEnabled = true; - Map<ComponentTypeEnum, Integer> limitMemoryCatalogSizePerType = new HashMap<>(); - - @PostConstruct - public void init() { - - Configuration configuration = ConfigurationManager.getConfigurationManager().getConfiguration(); - if (configuration != null) { - ApplicationL2CacheConfig applicationL2Cache = configuration.getApplicationL2Cache(); - if (applicationL2Cache != null) { - boolean isEnabled = applicationL2Cache.isEnabled(); - this.enabled = isEnabled; - - ApplicationL1CacheCatalogInfo catalog = applicationL2Cache.getCatalogL1Cache(); - if (catalog != null) { - catalogInMemoryEnabled = catalog.getEnabled(); - catalogInMemorySizePerResource = catalog.getResourcesSizeInCache(); - catalogInMemorySizePerService = catalog.getServicesSizeInCache(); - catalogInMemorySizePerProduct = catalog.getProductsSizeInCache(); - } - } - } - - ComponentTypeEnum[] typesForCache = { ComponentTypeEnum.RESOURCE, ComponentTypeEnum.SERVICE, - ComponentTypeEnum.PRODUCT }; - for (ComponentTypeEnum typeEnum : typesForCache) { - Map<String, Component> map = new HashMap<>(); - catalogInMemoryCache.put(typeEnum, map); - } - - limitMemoryCatalogSizePerType.put(ComponentTypeEnum.RESOURCE, catalogInMemorySizePerResource); - limitMemoryCatalogSizePerType.put(ComponentTypeEnum.SERVICE, catalogInMemorySizePerService); - limitMemoryCatalogSizePerType.put(ComponentTypeEnum.PRODUCT, catalogInMemorySizePerProduct); - } - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public Either<Component, ActionStatus> getComponent(String componentUid, Long lastModificationTime, - Function<Component, Component> filterFieldsFunc) { - - Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache( - componentUid, lastModificationTime, filterFieldsFunc); - - if (componentFromCache.isRight()) { - return Either.right(componentFromCache.right().value()); - } - - return Either.left(componentFromCache.left().value().left); - - } - - public Either<List<ComponentCacheData>, ActionStatus> getAllComponentIdTimeAndType() { - if (false == isEnabled()) { - return Either.right(ActionStatus.NOT_ALLOWED); - } - - Either<List<ComponentCacheData>, ActionStatus> componentRes = componentCassandraDao - .getAllComponentIdTimeAndType(); - - return componentRes; - - } - - /** - * @param foundComponents - * @param componentTypeEnum - */ - private void updateCatalogInMemoryCacheWithCertified(List<Component> foundComponents, - ComponentTypeEnum componentTypeEnum) { - - try { - wCatalogLock.lock(); - - long start = System.currentTimeMillis(); - Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum); - int mapSizeBefore = map.size(); - map.clear(); - Map<String, Component> collect = foundComponents.stream() - .filter(p -> p.getLifecycleState() == LifecycleStateEnum.CERTIFIED) - .limit(limitMemoryCatalogSizePerType.get(componentTypeEnum)) - .collect(Collectors.toMap(p -> p.getUniqueId(), p -> p)); - map.putAll(collect); - logger.debug( - "Size of in memory cache for catalog {}(certified only): Before {}, After {}. Replacement Time is {} ms.", - componentTypeEnum.name().toLowerCase(), mapSizeBefore, map.size(), - System.currentTimeMillis() - start); - } finally { - wCatalogLock.unlock(); - } - - } - - private List<Component> getDataFromInMemoryCache(Set<String> components, ComponentTypeEnum componentTypeEnum) { - List<Component> foundComponents = new ArrayList<>(); - - try { - - rCatalogLock.lock(); - - Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum); - for (String compUid : components) { - Component component = map.get(compUid); - if (component != null) { - foundComponents.add(component); - } - } - - } finally { - rCatalogLock.unlock(); - } - - return foundComponents; - } - - /** - * - * get full components from cassandra. On each component apply filter function - * in order to remove unused members - * - * @param components - * @param filterFieldsFunc - * @return <found components, found dirty components, not found components list> - * or Error - */ - public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponents( - Set<String> components, Function<List<Component>, List<Component>> filterFieldsFunc) { - - if (false == isEnabled()) { - logger.debug("Component Cache is disabled"); - return Either.right(ActionStatus.NOT_ALLOWED); - } - - Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull( - components); - - if (componentsFull.isRight()) { - return Either.right(componentsFull.right().value()); - } - - ImmutableTriple<List<Component>, List<Component>, Set<String>> immutableTriple = componentsFull.left().value(); - List<Component> foundResources = immutableTriple.left; - List<Component> foundDirtyResources = immutableTriple.middle; - Set<String> notFoundResources = immutableTriple.right; - - List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources); - List<Component> filterdFoundDirtyResources = filterFieldsFunc.apply(foundDirtyResources); - - ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<List<Component>, List<Component>, Set<String>>( - filterdFoundResources, filterdFoundDirtyResources, notFoundResources); - - return Either.left(result); - - } - - public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsForLeftPanel( - ComponentTypeEnum componentTypeEnum, String internalComponentType, Set<String> filteredResources) { - - logger.debug("In getComponentsForLeftPanel componentTypeEnum = {}, internalComponentType = {}", - componentTypeEnum, internalComponentType); - - Function<List<Component>, List<Component>> filterFieldsFunc = x -> filterForLeftPanel(x); - - return getComponents(filteredResources, filterFieldsFunc); - - } - - private List<Component> filterForLeftPanel(List<Component> components) { - - List<Component> result = new ArrayList<>(); - if (components != null) { - components.forEach(p -> result.add(filterFieldsForLeftPanel(p))); - } - - return result; - } - - private List<Component> filterForCatalog(List<Component> components) { - - List<Component> result = new ArrayList<>(); - if (components != null) { - components.forEach(p -> result.add(filterFieldsForCatalog(p))); - } - - return result; - } - - private Component filterFieldsForLeftPanel(Component component) { - - Component result = null; - ComponentTypeEnum componentTypeEnum = component.getComponentType(); - switch (componentTypeEnum) { - case RESOURCE: - result = new Resource(); - copyFieldsForLeftPanel(component, result); - break; - case SERVICE: - result = new Service(); - copyFieldsForLeftPanel(component, result); - break; - default: - break; - } - - return result; - } - - private Component filterFieldsForCatalog(Component component) { - - Component result = null; - ComponentTypeEnum componentTypeEnum = component.getComponentType(); - switch (componentTypeEnum) { - case RESOURCE: - result = new Resource(); - copyFieldsForCatalog(component, result); - break; - case SERVICE: - result = new Service(); - copyFieldsForCatalog(component, result); - break; - case PRODUCT: - result = new Product(); - copyFieldsForCatalog(component, result); - default: - break; - } - - return result; - } - - /** - * Copy relevant fields to the filtered component for left panel - * - * @param component - * @param filteredComponent - */ - private void copyFieldsForLeftPanel(Component component, Component filteredComponent) { - - ComponentTypeEnum componentTypeEnum = component.getComponentType(); - filteredComponent.setCategories(component.getCategories()); - filteredComponent.setComponentType(component.getComponentType()); - if (ComponentTypeEnum.RESOURCE.equals(component.getComponentType()) - && ResourceTypeEnum.VL.equals(((ResourceMetadataDataDefinition) component - .getComponentMetadataDefinition().getMetadataDataDefinition()).getResourceType())) { - filteredComponent.setCapabilities(component.getCapabilities()); - filteredComponent.setRequirements(component.getRequirements()); - } - filteredComponent.setVersion(component.getVersion()); - filteredComponent.setDescription(component.getDescription()); - filteredComponent.setUniqueId(component.getUniqueId()); - filteredComponent.setIcon(component.getIcon()); - filteredComponent.setTags(component.getTags()); - filteredComponent.setLifecycleState(component.getLifecycleState()); - filteredComponent.setInvariantUUID(component.getInvariantUUID()); - filteredComponent.setUUID(component.getUUID()); - filteredComponent.setSystemName(component.getSystemName()); - filteredComponent.setName(component.getName()); - - if (componentTypeEnum == ComponentTypeEnum.RESOURCE) { - Resource resource = (Resource) component; - Resource filteredResource = (Resource) filteredComponent; - filteredResource.setToscaResourceName(resource.getToscaResourceName()); - filteredResource.setResourceType(resource.getResourceType()); - } - } - - private void copyFieldsForCatalog(Component component, Component filteredComponent) { - - ComponentTypeEnum componentTypeEnum = component.getComponentType(); - filteredComponent.setCategories(component.getCategories()); - filteredComponent.setComponentType(component.getComponentType()); - filteredComponent.setVersion(component.getVersion()); - filteredComponent.setDescription(component.getDescription()); - filteredComponent.setUniqueId(component.getUniqueId()); - filteredComponent.setIcon(component.getIcon()); - filteredComponent.setTags(component.getTags()); - filteredComponent.setLifecycleState(component.getLifecycleState()); - filteredComponent.setSystemName(component.getSystemName()); - filteredComponent.setName(component.getName()); - filteredComponent.setLastUpdateDate(component.getLastUpdateDate()); - - if (componentTypeEnum == ComponentTypeEnum.RESOURCE) { - Resource resource = (Resource) component; - Resource filteredResource = (Resource) filteredComponent; - filteredResource.setToscaResourceName(resource.getToscaResourceName()); - filteredResource.setResourceType(resource.getResourceType()); - } else if (componentTypeEnum == ComponentTypeEnum.SERVICE) { - Service service = (Service) component; - Service filteredService = (Service) filteredComponent; - filteredService.setDistributionStatus(service.getDistributionStatus()); - } - } - - /** - * get components from cache of a given list ou unique ids. - * - * for each component data from cassandra, unzip the data if needed and - * deserialize the unzipped data to java object(Component). - * - * @param filteredResources - * @return ImmutableTripple or ActionStatus. | |-- components |-- dirty - * components - components with dirty flag = true. |-- set of non cached - * components - * - */ - private Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsFull( - Set<String> filteredResources) { - - if (false == isEnabled()) { - logger.debug("Component Cache is disabled"); - return Either.right(ActionStatus.NOT_ALLOWED); - } - - List<Component> foundResources = new LinkedList<>(); - List<Component> foundDirtyResources = new LinkedList<>(); - Set<String> notFoundResources = new HashSet<>(); - ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<List<Component>, List<Component>, Set<String>>( - foundResources, foundDirtyResources, notFoundResources); - - long cassandraFetchStart = System.currentTimeMillis(); - List<String> uidsList = new ArrayList<>(); - uidsList.addAll(filteredResources); - Either<List<ComponentCacheData>, ActionStatus> componentsFromCache = componentCassandraDao - .getComponents(uidsList); - - long cassandraFetchEnd = System.currentTimeMillis(); - logger.debug("Fetch time from cassandara of all components took {} ms", - (cassandraFetchEnd - cassandraFetchStart)); - if (componentsFromCache.isRight()) { - BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache", - "Failed to fetch components from cache", ErrorSeverity.ERROR); - return Either.right(componentsFromCache.right().value()); - } - - List<ComponentCacheData> list = componentsFromCache.left().value(); - logger.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size())); - if (list != null && false == list.isEmpty()) { - - List<ComponentCacheData> filteredData = list.stream().filter(p -> filteredResources.contains(p.getId())) - .collect(Collectors.toList()); - logger.debug("Number of components filterd is {}", filteredData == null ? 0 : filteredData.size()); - - if (filteredData != null) { - long desStart = System.currentTimeMillis(); - - for (ComponentCacheData componentCacheData : filteredData) { - - logger.debug("Process uid {} from cache", componentCacheData.getId()); - - String compUid = componentCacheData.getId(); - - Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent( - componentCacheData); - - if (deserializeExt.isLeft()) { - Component component = deserializeExt.left().value(); - if (false == componentCacheData.getIsDirty()) { - foundResources.add(component); - } else { - foundDirtyResources.add(component); - } - } else { - notFoundResources.add(compUid); - } - - } - long desEnd = System.currentTimeMillis(); - logger.debug("Deserialization and unzip of {} components took {} ms", filteredData.size(), - (desEnd - desStart)); - } - } - List<String> foundResourcesUid = foundResources.stream().map(p -> p.getUniqueId()).collect(Collectors.toList()); - List<String> foundDirtyResourcesUid = foundDirtyResources.stream().map(p -> p.getUniqueId()) - .collect(Collectors.toList()); - logger.debug("Number of processed components from cache is {}", - (foundResourcesUid.size() + foundDirtyResourcesUid.size())); - Set<String> notCachedResources = filteredResources.stream() - .filter(p -> false == foundResourcesUid.contains(p) && false == foundDirtyResourcesUid.contains(p)) - .collect(Collectors.toSet()); - notFoundResources.addAll(notCachedResources); - - logger.debug("Number of components fetched is {}", foundResources.size()); - logger.debug("Number of components fetched dirty is {}", foundDirtyResources.size()); - logger.debug("Number of components non cached is {}", notCachedResources.size()); - - return Either.left(result); - } - - private Either<? extends Component, Boolean> convertComponentCacheToComponent( - ComponentCacheData componentCacheData) { - - String compUid = componentCacheData.getId(); - - byte[] dataAsArray = componentCacheData.getDataAsArray(); - - if (true == componentCacheData.getIsZipped()) { - long startUnzip = System.nanoTime(); - dataAsArray = ZipUtil.unzip(dataAsArray); - long endUnzip = System.nanoTime(); - logger.trace("Unzip component {} took {} microsecond", compUid, (endUnzip - startUnzip) / 1000); - } - - long startDes = System.nanoTime(); - - Either<? extends Component, Boolean> deserializeExt = deserializeComponent(componentCacheData, dataAsArray); - - long endDes = System.nanoTime(); - logger.trace("Deserialize component {} took {} microsecond", compUid, (endDes - startDes) / 1000); - return deserializeExt; - } - - private Either<? extends Component, Boolean> deserializeComponent(ComponentCacheData componentCacheData, - byte[] dataAsArray) { - String type = componentCacheData.getType(); - NodeTypeEnum typeEnum = NodeTypeEnum.getByNameIgnoreCase(type); - - Either<? extends Component, Boolean> deserializeExt = Either.right(false); - switch (typeEnum) { - case Resource: - deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Resource.class, componentCacheData.getId()); - break; - case Service: - deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Service.class, componentCacheData.getId()); - break; - case Product: - deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Product.class, componentCacheData.getId()); - break; - default: - break; - } - return deserializeExt; - } - - public Either<Component, ActionStatus> getComponent(String componentUid) { - - return getComponent(componentUid, null, Function.identity()); - } - - public Either<Component, ActionStatus> getComponent(String componentUid, Long lastModificationTime) { - - return getComponent(componentUid, lastModificationTime, Function.identity()); - - } - - public boolean setComponent(String componentUid, Long lastModificationTime, NodeTypeEnum nodeTypeEnum) { - - boolean result = false; - - if (false == isEnabled()) { - logger.debug("Component Cache is disabled"); - return false; - } - - Either<Component, StorageOperationStatus> either = toscaOperationFacade.getToscaElement(componentUid); - if (either.isLeft()) { - Component component = either.left().value(); - result = saveComponent(componentUid, lastModificationTime, nodeTypeEnum, component); - } else { - logger.debug("Failed to get component {} of type {} from graph. Status is {}", componentUid, - nodeTypeEnum.name().toLowerCase(), either.right().value()); - } - - return result; - - } - - private boolean saveComponent(String componentUid, Long lastModificationTime, NodeTypeEnum nodeTypeEnum, - Component component) { - - logger.trace("Going to save component {} of type {} in cache", componentUid, nodeTypeEnum.name().toLowerCase()); - - boolean result = false; - - Either<byte[], Boolean> serializeExt = SerializationUtils.serializeExt(component); - if (serializeExt.isLeft()) { - byte[] serializedData = serializeExt.left().value(); - byte[] zipBytes; - try { - zipBytes = ZipUtil.zipBytes(serializedData); - ComponentCacheData componentCacheData = new ComponentCacheData(); - componentCacheData.setDataAsArray(zipBytes); - componentCacheData.setIsZipped(true); - componentCacheData.setId(componentUid); - componentCacheData.setModificationTime(new Date(lastModificationTime)); - componentCacheData.setType(component.getComponentType().name().toLowerCase()); - - CassandraOperationStatus status = componentCassandraDao.saveComponent(componentCacheData); - - if (status == CassandraOperationStatus.OK) { - result = true; - } - - } catch (IOException e) { - logger.debug("Failed to prepare component {} of type {} for cache", componentUid, - nodeTypeEnum.name().toLowerCase()); - } - } else { - logger.debug("Failed to serialize component {} of type {} for cache", componentUid, - nodeTypeEnum.name().toLowerCase()); - } - return result; - } - - public boolean setComponent(Component component, NodeTypeEnum nodeTypeEnum) { - - boolean result = false; - - if (false == isEnabled()) { - logger.debug("Component Cache is disabled"); - return false; - } - - String componentUid = component.getUniqueId(); - Long lastUpdateDate = component.getLastUpdateDate(); - - result = saveComponent(componentUid, lastUpdateDate, nodeTypeEnum, component); - - return result; - - } - - /** - * get components from cache of a given list ou unique ids. - * - * for each component data from cassandra, unzip the data if needed and - * deserialize the unzipped data to java object(Component). - * - * @param filteredResources - * @return ImmutableTripple or ActionStatus. | |-- components |-- set of non - * cached components - * - */ - private Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsFull( - Map<String, Long> filteredResources) { - - if (false == isEnabled()) { - logger.debug("Component Cache is disabled"); - return Either.right(ActionStatus.NOT_ALLOWED); - } - - List<Component> foundResources = new LinkedList<>(); - Set<String> notFoundResources = new HashSet<>(); - ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<List<Component>, Set<String>>( - foundResources, notFoundResources); - - long cassandraFetchStart = System.currentTimeMillis(); - - Either<ImmutablePair<List<ComponentCacheData>, Set<String>>, ActionStatus> componentsFromCache = componentCassandraDao - .getComponents(filteredResources); - - long cassandraFetchEnd = System.currentTimeMillis(); - logger.debug("Fetch time from cassandara of all components took {} ms", - (cassandraFetchEnd - cassandraFetchStart)); - if (componentsFromCache.isRight()) { - BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache", - "Failed to fetch components from cache", ErrorSeverity.ERROR); - return Either.right(componentsFromCache.right().value()); - } - - ImmutablePair<List<ComponentCacheData>, Set<String>> immutablePair = componentsFromCache.left().value(); - List<ComponentCacheData> list = immutablePair.getLeft(); - logger.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size())); - if (list != null && false == list.isEmpty()) { - - // List<ComponentCacheData> filteredData = list.stream().filter(p -> - // filteredResources.contains(p.getId())).collect(Collectors.toList()); - logger.debug("Number of components filterd is {}", list == null ? 0 : list.size()); - - if (list != null) { - long desStart = System.currentTimeMillis(); - - for (ComponentCacheData componentCacheData : list) { - - logger.debug("Process uid {} from cache", componentCacheData.getId()); - - String compUid = componentCacheData.getId(); - - Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent( - componentCacheData); - - if (deserializeExt.isLeft()) { - Component component = deserializeExt.left().value(); - foundResources.add(component); - } else { - notFoundResources.add(compUid); - } - - } - long desEnd = System.currentTimeMillis(); - logger.debug("Deserialization and unzip of {} components took {} ms", list.size(), (desEnd - desStart)); - } - } - logger.debug("Number of processed components from cache is {}", foundResources.size()); - - Set<String> notFoundInCache = immutablePair.getRight(); - notFoundResources.addAll(notFoundInCache); - - logger.debug("Number of components fetched is {}", foundResources.size()); - logger.debug("Number of components non cached is {}", notFoundResources.size()); - - return Either.left(result); - } - - /** - * get components for catalog - * - * @param components - * @param componentTypeEnum - * @return - */ - public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsForCatalog( - Map<String, Long> components, ComponentTypeEnum componentTypeEnum) { - - if (false == isEnabled()) { - logger.debug("In getComponentsForCatalog for type {}. Cache is disabled.", - componentTypeEnum.name().toLowerCase()); - return Either.right(ActionStatus.NOT_ALLOWED); - } - logger.debug("In getComponentsForCatalog for type {}", componentTypeEnum.name().toLowerCase()); - - Function<List<Component>, List<Component>> filterFieldsFunc = x -> filterForCatalog(x); - - Map<String, Long> leftComponentsForSearch = new HashMap<>(); - leftComponentsForSearch.putAll(components); - - // get components from inmemory cache - List<Component> componentsFromMemory = null; - if (true == catalogInMemoryEnabled) { - componentsFromMemory = getDataFromInMemoryCache(components.keySet(), componentTypeEnum); - logger.debug("The number of components of type {} fetched from memory is {}", - componentTypeEnum.name().toLowerCase(), - componentsFromMemory == null ? 0 : componentsFromMemory.size()); - if (componentsFromMemory != null) { - List<String> ignoredComponents = new ArrayList<>(); - for (Component componentFromMem : componentsFromMemory) { - if (componentFromMem.getLastUpdateDate().longValue() != components - .get(componentFromMem.getUniqueId()).longValue()) { - // Ignore the component from memory - ignoredComponents.add(componentFromMem.getUniqueId()); - } - } - - logger.debug("Number of components from type {} ignored from memory cache is {}", - componentTypeEnum.name().toLowerCase(), ignoredComponents.size()); - // remove from memory result the components which are not valid - componentsFromMemory = componentsFromMemory.stream() - .filter(p -> false == ignoredComponents.contains(p.getUniqueId())).collect(Collectors.toList()); - // Remove from leftComponentsForSearch the valid components from - // memory - componentsFromMemory.forEach(p -> leftComponentsForSearch.remove(p.getUniqueId())); - - } - } else { - logger.debug("Catalog InMemory cache is disabled"); - } - - logger.debug("Number of components from type {} needed to fetch is {}", componentTypeEnum.name().toLowerCase(), - leftComponentsForSearch.size()); - - // get components from cassandra cache and filter each component - Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> result = getComponents( - leftComponentsForSearch, filterFieldsFunc); - - if (result.isLeft()) { - // add inmemory components to the valid components(not dirty) - List<Component> foundComponents = result.left().value().getLeft(); - if (componentsFromMemory != null) { - foundComponents.addAll(componentsFromMemory); - } - if (true == catalogInMemoryEnabled) { - updateCatalogInMemoryCacheWithCertified(foundComponents, componentTypeEnum); - } - } - - return result; - } - - /** - * @param components - * - Map of <componentUniqueId, last update date> - * @param filterFieldsFunc - * @return - */ - public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponents(Map<String, Long> components, - Function<List<Component>, List<Component>> filterFieldsFunc) { - - if (false == isEnabled()) { - logger.debug("Component Cache is disabled"); - return Either.right(ActionStatus.NOT_ALLOWED); - } - - Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull( - components); - - if (componentsFull.isRight()) { - return Either.right(componentsFull.right().value()); - } - - ImmutablePair<List<Component>, Set<String>> immutablePair = componentsFull.left().value(); - List<Component> foundResources = immutablePair.left; - Set<String> notFoundResources = immutablePair.right; - - List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources); - - ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<List<Component>, Set<String>>( - filterdFoundResources, notFoundResources); - - return Either.left(result); - - } - - /** - * get the component and its modification time from cache - * - * @param componentUid - * @param filterFieldsFunc - * @return - */ - public Either<ImmutablePair<Component, Long>, ActionStatus> getComponentAndTime(String componentUid, - Function<Component, Component> filterFieldsFunc) { - - Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache( - componentUid, null, filterFieldsFunc); - - if (componentFromCache.isRight()) { - return Either.right(componentFromCache.right().value()); - } - - ImmutablePair<Component, ComponentCacheData> immutablePair = componentFromCache.left().value(); - - ImmutablePair<Component, Long> result = new ImmutablePair<Component, Long>(immutablePair.left, - immutablePair.right.getModificationTime().getTime()); - - return Either.left(result); - } - - private Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> getComponentFromCache( - String componentUid, Long lastModificationTime, Function<Component, Component> filterFieldsFunc) { - if (false == isEnabled()) { - return Either.right(ActionStatus.NOT_ALLOWED); - } - - Either<ComponentCacheData, ActionStatus> componentRes = componentCassandraDao.getComponent(componentUid); - - if (componentRes.isRight()) { - return Either.right(componentRes.right().value()); - } - - ComponentCacheData componentCacheData = componentRes.left().value(); - - if (lastModificationTime != null) { - long cacheCompModificationTime = componentCacheData.getModificationTime().getTime(); - if (lastModificationTime != cacheCompModificationTime) { - logger.debug( - "Component {} found in cache but its modification time {} does not match to the timestamp in cache {}.", - componentUid, lastModificationTime, cacheCompModificationTime); - return Either.right(ActionStatus.INVALID_CONTENT); - } - } - - Either<? extends Component, Boolean> convertRes = convertComponentCacheToComponent(componentCacheData); - if (convertRes.isRight()) { - return Either.right(ActionStatus.CONVERT_COMPONENT_ERROR); - } - - Component component = convertRes.left().value(); - - Component filteredComponent = component; - if (filterFieldsFunc != null) { - filteredComponent = filterFieldsFunc.apply(component); - } - - ImmutablePair<Component, ComponentCacheData> result = new ImmutablePair<Component, ComponentCacheData>( - filteredComponent, componentCacheData); - - return Either.left(result); - } - - public ActionStatus deleteComponentFromCache(String id) { - if (false == isEnabled()) { - return ActionStatus.NOT_ALLOWED; - } - CassandraOperationStatus status = componentCassandraDao.deleteComponent(id); - if (CassandraOperationStatus.OK.equals(status)) { - return ActionStatus.OK; - } else { - logger.debug("delete component failed with error {}", status); - return ActionStatus.GENERAL_ERROR; - } - } + private static final String COMPONENT_CACHE_IS_DISABLED = "Component Cache is disabled"; + + private static final Logger log = Logger.getLogger(ComponentCache.class); + + @Autowired + ComponentCassandraDao componentCassandraDao; + + @Autowired + ToscaOperationFacade toscaOperationFacade; + + private Map<ComponentTypeEnum, Map<String, Component>> catalogInMemoryCache = new HashMap<>(); + private final ReentrantReadWriteLock rwCatalogLock = new ReentrantReadWriteLock(); + private final Lock rCatalogLock = rwCatalogLock.readLock(); + private final Lock wCatalogLock = rwCatalogLock.writeLock(); + + boolean enabled = true; + int catalogInMemorySizePerResource = 300; + int catalogInMemorySizePerService = 200; + int catalogInMemorySizePerProduct = 100; + boolean catalogInMemoryEnabled = true; + Map<ComponentTypeEnum, Integer> limitMemoryCatalogSizePerType = new HashMap<>(); + + @PostConstruct + public void init() { + + Configuration configuration = ConfigurationManager.getConfigurationManager().getConfiguration(); + if (configuration != null) { + ApplicationL2CacheConfig applicationL2Cache = configuration.getApplicationL2Cache(); + if (applicationL2Cache != null) { + this.enabled = applicationL2Cache.isEnabled(); + + ApplicationL1CacheCatalogInfo catalog = applicationL2Cache.getCatalogL1Cache(); + if (catalog != null) { + catalogInMemoryEnabled = catalog.getEnabled(); + catalogInMemorySizePerResource = catalog.getResourcesSizeInCache(); + catalogInMemorySizePerService = catalog.getServicesSizeInCache(); + catalogInMemorySizePerProduct = catalog.getProductsSizeInCache(); + } + } + } + + ComponentTypeEnum[] typesForCache = { ComponentTypeEnum.RESOURCE, ComponentTypeEnum.SERVICE, + ComponentTypeEnum.PRODUCT }; + for (ComponentTypeEnum typeEnum : typesForCache) { + Map<String, Component> map = new HashMap<>(); + catalogInMemoryCache.put(typeEnum, map); + } + + limitMemoryCatalogSizePerType.put(ComponentTypeEnum.RESOURCE, catalogInMemorySizePerResource); + limitMemoryCatalogSizePerType.put(ComponentTypeEnum.SERVICE, catalogInMemorySizePerService); + limitMemoryCatalogSizePerType.put(ComponentTypeEnum.PRODUCT, catalogInMemorySizePerProduct); + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public Either<Component, ActionStatus> getComponent(String componentUid, Long lastModificationTime, + Function<Component, Component> filterFieldsFunc) { + + Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache( + componentUid, lastModificationTime, filterFieldsFunc); + + if (componentFromCache.isRight()) { + return Either.right(componentFromCache.right().value()); + } + + return Either.left(componentFromCache.left().value().left); + + } + + public Either<List<ComponentCacheData>, ActionStatus> getAllComponentIdTimeAndType() { + if (!isEnabled()) { + return Either.right(ActionStatus.NOT_ALLOWED); + } + + return componentCassandraDao + .getAllComponentIdTimeAndType(); + + } + + /** + * get components for catalog + * + * @param components + * @param componentTypeEnum + * @return + */ + @Deprecated + public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsForCatalog( + Set<String> components, ComponentTypeEnum componentTypeEnum) { + + if (!isEnabled()) { + log.debug("In getComponentsForCatalog for type {}. Cache is disabled.", + componentTypeEnum.name().toLowerCase()); + return Either.right(ActionStatus.NOT_ALLOWED); + } + log.debug("In getComponentsForCatalog for type {}", componentTypeEnum.name().toLowerCase()); + + Function<List<Component>, List<Component>> filterFieldsFunc = this::filterForCatalog; + + Set<String> leftComponentsForSearch = new HashSet<>(); + leftComponentsForSearch.addAll(components); + + // get components from inmemory cache + List<Component> componentsFromMemory = null; + if (catalogInMemoryEnabled) { + componentsFromMemory = getDataFromInMemoryCache(components, componentTypeEnum); + log.debug("The number of components of type {} fetched from memory is {}", + componentTypeEnum.name().toLowerCase(), + componentsFromMemory == null ? 0 : componentsFromMemory.size()); + if (componentsFromMemory != null) { + componentsFromMemory.forEach(p -> leftComponentsForSearch.remove(p.getUniqueId())); + } + } else { + log.debug("Catalog InMemory cache is disabled"); + } + + log.debug("Number of components from type {} needed to fetch is {}", componentTypeEnum.name().toLowerCase(), + leftComponentsForSearch.size()); + + // get components from cassandra cache and filter each component + Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> result = getComponents( + leftComponentsForSearch, filterFieldsFunc); + + if (result.isLeft()) { + // add inmemory components to the valid components(not dirty) + List<Component> foundComponents = result.left().value().getLeft(); + if (componentsFromMemory != null) { + foundComponents.addAll(componentsFromMemory); + } + if (catalogInMemoryEnabled) { + updateCatalogInMemoryCacheWithCertified(foundComponents, componentTypeEnum); + } + } + + return result; + } + + /** + * @param foundComponents + * @param componentTypeEnum + */ + private void updateCatalogInMemoryCacheWithCertified(List<Component> foundComponents, + ComponentTypeEnum componentTypeEnum) { + + try { + wCatalogLock.lock(); + + long start = System.currentTimeMillis(); + Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum); + int mapSizeBefore = map.size(); + map.clear(); + Map<String, Component> collect = foundComponents.stream() + .filter(p -> p.getLifecycleState() == LifecycleStateEnum.CERTIFIED) + .limit(limitMemoryCatalogSizePerType.get(componentTypeEnum)) + .collect(Collectors.toMap(Component::getUniqueId, p -> p)); + map.putAll(collect); + log.debug( + "Size of in memory cache for catalog {}(certified only): Before {}, After {}. Replacement Time is {} ms.", + componentTypeEnum.name().toLowerCase(), mapSizeBefore, map.size(), + System.currentTimeMillis() - start); + } finally { + wCatalogLock.unlock(); + } + + } + + private List<Component> getDataFromInMemoryCache(Set<String> components, ComponentTypeEnum componentTypeEnum) { + List<Component> foundComponents = new ArrayList<>(); + + try { + + rCatalogLock.lock(); + + Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum); + for (String compUid : components) { + Component component = map.get(compUid); + if (component != null) { + foundComponents.add(component); + } + } + + } finally { + rCatalogLock.unlock(); + } + + return foundComponents; + } + + /** + * + * get full components from cassandra. On each component apply filter + * function in order to remove unused members + * + * @param components + * @param filterFieldsFunc + * @return <found components, found dirty components, not found components + * list> or Error + */ + public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponents( + Set<String> components, Function<List<Component>, List<Component>> filterFieldsFunc) { + + if (!isEnabled()) { + log.debug(COMPONENT_CACHE_IS_DISABLED); + return Either.right(ActionStatus.NOT_ALLOWED); + } + + Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull( + components); + + if (componentsFull.isRight()) { + return Either.right(componentsFull.right().value()); + } + + ImmutableTriple<List<Component>, List<Component>, Set<String>> immutableTriple = componentsFull.left().value(); + List<Component> foundResources = immutableTriple.left; + List<Component> foundDirtyResources = immutableTriple.middle; + Set<String> notFoundResources = immutableTriple.right; + + List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources); + List<Component> filterdFoundDirtyResources = filterFieldsFunc.apply(foundDirtyResources); + + ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<>( + filterdFoundResources, filterdFoundDirtyResources, notFoundResources); + + return Either.left(result); + + } + + public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsForLeftPanel( + ComponentTypeEnum componentTypeEnum, String internalComponentType, Set<String> filteredResources) { + + log.debug("In getComponentsForLeftPanel componentTypeEnum = {}, internalComponentType = {}", + componentTypeEnum, internalComponentType); + + Function<List<Component>, List<Component>> filterFieldsFunc = this::filterForLeftPanel; + + return getComponents(filteredResources, filterFieldsFunc); + + } + + private List<Component> filterForLeftPanel(List<Component> components) { + + List<Component> result = new ArrayList<>(); + if (components != null) { + components.forEach(p -> result.add(filterFieldsForLeftPanel(p))); + } + + return result; + } + + private List<Component> filterForCatalog(List<Component> components) { + + List<Component> result = new ArrayList<>(); + if (components != null) { + components.forEach(p -> result.add(filterFieldsForCatalog(p))); + } + + return result; + } + + private Component filterFieldsForLeftPanel(Component component) { + + Component result = null; + ComponentTypeEnum componentTypeEnum = component.getComponentType(); + switch (componentTypeEnum) { + case RESOURCE: + result = new Resource(); + copyFieldsForLeftPanel(component, result); + break; + case SERVICE: + result = new Service(); + copyFieldsForLeftPanel(component, result); + break; + default: + break; + } + + return result; + } + + private Component filterFieldsForCatalog(Component component) { + + Component result = null; + ComponentTypeEnum componentTypeEnum = component.getComponentType(); + switch (componentTypeEnum) { + case RESOURCE: + result = new Resource(); + copyFieldsForCatalog(component, result); + break; + case SERVICE: + result = new Service(); + copyFieldsForCatalog(component, result); + break; + case PRODUCT: + result = new Product(); + copyFieldsForCatalog(component, result); + break; + default: + break; + } + + return result; + } + + /** + * Copy relevant fields to the filtered component for left panel + * + * @param component + * @param filteredComponent + */ + private void copyFieldsForLeftPanel(Component component, Component filteredComponent) { + + ComponentTypeEnum componentTypeEnum = component.getComponentType(); + filteredComponent.setCategories(component.getCategories()); + filteredComponent.setComponentType(component.getComponentType()); + if (ComponentTypeEnum.RESOURCE.equals(component.getComponentType()) + && ResourceTypeEnum.VL.equals(((ResourceMetadataDataDefinition) component + .getComponentMetadataDefinition().getMetadataDataDefinition()).getResourceType())) { + filteredComponent.setCapabilities(component.getCapabilities()); + filteredComponent.setRequirements(component.getRequirements()); + } + filteredComponent.setVersion(component.getVersion()); + filteredComponent.setDescription(component.getDescription()); + filteredComponent.setUniqueId(component.getUniqueId()); + filteredComponent.setIcon(component.getIcon()); + filteredComponent.setTags(component.getTags()); + filteredComponent.setLifecycleState(component.getLifecycleState()); + filteredComponent.setInvariantUUID(component.getInvariantUUID()); + filteredComponent.setUUID(component.getUUID()); + filteredComponent.setSystemName(component.getSystemName()); + filteredComponent.setName(component.getName()); + + if (componentTypeEnum == ComponentTypeEnum.RESOURCE) { + Resource resource = (Resource) component; + Resource filteredResource = (Resource) filteredComponent; + filteredResource.setToscaResourceName(resource.getToscaResourceName()); + filteredResource.setResourceType(resource.getResourceType()); + } + } + + private void copyFieldsForCatalog(Component component, Component filteredComponent) { + + ComponentTypeEnum componentTypeEnum = component.getComponentType(); + filteredComponent.setCategories(component.getCategories()); + filteredComponent.setComponentType(component.getComponentType()); + filteredComponent.setVersion(component.getVersion()); + filteredComponent.setDescription(component.getDescription()); + filteredComponent.setUniqueId(component.getUniqueId()); + filteredComponent.setIcon(component.getIcon()); + filteredComponent.setTags(component.getTags()); + filteredComponent.setLifecycleState(component.getLifecycleState()); + filteredComponent.setSystemName(component.getSystemName()); + filteredComponent.setName(component.getName()); + filteredComponent.setLastUpdateDate(component.getLastUpdateDate()); + + if (componentTypeEnum == ComponentTypeEnum.RESOURCE) { + Resource resource = (Resource) component; + Resource filteredResource = (Resource) filteredComponent; + filteredResource.setToscaResourceName(resource.getToscaResourceName()); + filteredResource.setResourceType(resource.getResourceType()); + } else if (componentTypeEnum == ComponentTypeEnum.SERVICE) { + Service service = (Service) component; + Service filteredService = (Service) filteredComponent; + filteredService.setDistributionStatus(service.getDistributionStatus()); + } + } + + /** + * get components from cache of a given list ou unique ids. + * + * for each component data from cassandra, unzip the data if needed and + * deserialize the unzipped data to java object(Component). + * + * @param filteredResources + * @return ImmutableTripple or ActionStatus. | |-- components |-- dirty + * components - components with dirty flag = true. |-- set of non + * cached components + * + */ + private Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsFull( + Set<String> filteredResources) { + + if (!isEnabled()) { + log.debug(COMPONENT_CACHE_IS_DISABLED); + return Either.right(ActionStatus.NOT_ALLOWED); + } + + List<Component> foundResources = new LinkedList<>(); + List<Component> foundDirtyResources = new LinkedList<>(); + Set<String> notFoundResources = new HashSet<>(); + ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<>( + foundResources, foundDirtyResources, notFoundResources); + + long cassandraFetchStart = System.currentTimeMillis(); + List<String> uidsList = new ArrayList<>(); + uidsList.addAll(filteredResources); + Either<List<ComponentCacheData>, ActionStatus> componentsFromCache = componentCassandraDao + .getComponents(uidsList); + + long cassandraFetchEnd = System.currentTimeMillis(); + log.debug("Fetch time from cassandara of all components took {} ms", + (cassandraFetchEnd - cassandraFetchStart)); + if (componentsFromCache.isRight()) { + BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache", + "Failed to fetch components from cache", ErrorSeverity.ERROR); + return Either.right(componentsFromCache.right().value()); + } + + List<ComponentCacheData> list = componentsFromCache.left().value(); + log.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size())); + if (list != null && !list.isEmpty()) { + + List<ComponentCacheData> filteredData = list.stream().filter(p -> filteredResources.contains(p.getId())) + .collect(Collectors.toList()); + log.debug("Number of components filterd is {}", filteredData == null ? 0 : filteredData.size()); + + if (filteredData != null) { + long desStart = System.currentTimeMillis(); + + for (ComponentCacheData componentCacheData : filteredData) { + + log.debug("Process uid {} from cache", componentCacheData.getId()); + + String compUid = componentCacheData.getId(); + + Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent( + componentCacheData); + + if (deserializeExt.isLeft()) { + Component component = deserializeExt.left().value(); + if (!componentCacheData.getIsDirty()) { + foundResources.add(component); + } else { + foundDirtyResources.add(component); + } + } else { + notFoundResources.add(compUid); + } + + } + long desEnd = System.currentTimeMillis(); + log.debug("Deserialization and unzip of {} components took {} ms", filteredData.size(), + (desEnd - desStart)); + } + } + List<String> foundResourcesUid = foundResources.stream().map(Component::getUniqueId).collect(Collectors.toList()); + List<String> foundDirtyResourcesUid = foundDirtyResources.stream().map(Component::getUniqueId) + .collect(Collectors.toList()); + log.debug("Number of processed components from cache is {}", + (foundResourcesUid.size() + foundDirtyResourcesUid.size())); + Set<String> notCachedResources = filteredResources.stream() + .filter(p -> !foundResourcesUid.contains(p) && !foundDirtyResourcesUid.contains(p)) + .collect(Collectors.toSet()); + notFoundResources.addAll(notCachedResources); + + if (log.isDebugEnabled()) { + log.debug("Number of components fetched is {}", foundResources.size()); + log.debug("Number of components fetched dirty is {}", foundDirtyResources.size()); + log.debug("Number of components non cached is {}", notCachedResources.size()); + } + + return Either.left(result); + } + + private Either<? extends Component, Boolean> convertComponentCacheToComponent( + ComponentCacheData componentCacheData) { + + String compUid = componentCacheData.getId(); + + byte[] dataAsArray = componentCacheData.getDataAsArray(); + + if (componentCacheData.getIsZipped()) { + long startUnzip = System.nanoTime(); + dataAsArray = ZipUtil.unzip(dataAsArray); + long endUnzip = System.nanoTime(); + log.trace("Unzip component {} took {} microsecond", compUid, (endUnzip - startUnzip) / 1000); + } + + long startDes = System.nanoTime(); + + Either<? extends Component, Boolean> deserializeExt = deserializeComponent(componentCacheData, dataAsArray); + + long endDes = System.nanoTime(); + log.trace("Deserialize component {} took {} microsecond", compUid, (endDes - startDes) / 1000); + return deserializeExt; + } + + private Either<? extends Component, Boolean> deserializeComponent(ComponentCacheData componentCacheData, + byte[] dataAsArray) { + String type = componentCacheData.getType(); + NodeTypeEnum typeEnum = NodeTypeEnum.getByNameIgnoreCase(type); + + Either<? extends Component, Boolean> deserializeExt = Either.right(false); + switch (typeEnum) { + case Resource: + deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Resource.class, componentCacheData.getId()); + break; + case Service: + deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Service.class, componentCacheData.getId()); + break; + case Product: + deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Product.class, componentCacheData.getId()); + break; + default: + break; + } + return deserializeExt; + } + + public Either<Component, ActionStatus> getComponent(String componentUid) { + + return getComponent(componentUid, null, Function.identity()); + + } + + private boolean saveComponent(String componentUid, Long lastModificationTime, NodeTypeEnum nodeTypeEnum, + Component component) { + + log.trace("Going to save component {} of type {} in cache", componentUid, nodeTypeEnum.name().toLowerCase()); + + boolean result = false; + + Either<byte[], Boolean> serializeExt = SerializationUtils.serializeExt(component); + if (serializeExt.isLeft()) { + byte[] serializedData = serializeExt.left().value(); + byte[] zipBytes; + try { + zipBytes = ZipUtil.zipBytes(serializedData); + ComponentCacheData componentCacheData = new ComponentCacheData(); + componentCacheData.setDataAsArray(zipBytes); + componentCacheData.setIsZipped(true); + componentCacheData.setId(componentUid); + componentCacheData.setModificationTime(new Date(lastModificationTime)); + componentCacheData.setType(component.getComponentType().name().toLowerCase()); + + CassandraOperationStatus status = componentCassandraDao.saveComponent(componentCacheData); + + if (status == CassandraOperationStatus.OK) { + result = true; + } + + } catch (IOException e) { + log.debug("Failed to prepare component {} of type {} for cache", componentUid, + nodeTypeEnum.name().toLowerCase()); + if (log.isTraceEnabled()) { + log.trace("Failed to prepare component {} of type {} for cache",componentUid,nodeTypeEnum.name().toLowerCase()); + } + } + } else { + log.debug("Failed to serialize component {} of type {} for cache", componentUid, + nodeTypeEnum.name().toLowerCase()); + } + return result; + } + + public boolean setComponent(Component component, NodeTypeEnum nodeTypeEnum) { + + boolean result = false; + + if (!isEnabled()) { + log.debug(COMPONENT_CACHE_IS_DISABLED); + return false; + } + + String componentUid = component.getUniqueId(); + Long lastUpdateDate = component.getLastUpdateDate(); + + result = saveComponent(componentUid, lastUpdateDate, nodeTypeEnum, component); + + return result; + + } + + /** + * get components from cache of a given list ou unique ids. + * + * for each component data from cassandra, unzip the data if needed and + * deserialize the unzipped data to java object(Component). + * + * @param filteredResources + * @return ImmutableTripple or ActionStatus. | |-- components |-- set of non + * cached components + * + */ + private Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsFull( + Map<String, Long> filteredResources) { + + if (!isEnabled()) { + log.debug(COMPONENT_CACHE_IS_DISABLED); + return Either.right(ActionStatus.NOT_ALLOWED); + } + + List<Component> foundResources = new LinkedList<>(); + Set<String> notFoundResources = new HashSet<>(); + ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<>( + foundResources, notFoundResources); + + long cassandraFetchStart = System.currentTimeMillis(); + + Either<ImmutablePair<List<ComponentCacheData>, Set<String>>, ActionStatus> componentsFromCache = componentCassandraDao + .getComponents(filteredResources); + + long cassandraFetchEnd = System.currentTimeMillis(); + log.debug("Fetch time from cassandara of all components took {} ms", + (cassandraFetchEnd - cassandraFetchStart)); + if (componentsFromCache.isRight()) { + BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache", + "Failed to fetch components from cache", ErrorSeverity.ERROR); + return Either.right(componentsFromCache.right().value()); + } + + ImmutablePair<List<ComponentCacheData>, Set<String>> immutablePair = componentsFromCache.left().value(); + List<ComponentCacheData> list = immutablePair.getLeft(); + log.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size())); + if (list != null && !list.isEmpty()) { + + log.debug("Number of components filterd is {}", list == null ? 0 : list.size()); + + if (list != null) { + long desStart = System.currentTimeMillis(); + + for (ComponentCacheData componentCacheData : list) { + + log.debug("Process uid {} from cache", componentCacheData.getId()); + + String compUid = componentCacheData.getId(); + + Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent( + componentCacheData); + + if (deserializeExt.isLeft()) { + Component component = deserializeExt.left().value(); + foundResources.add(component); + } else { + notFoundResources.add(compUid); + } + + } + long desEnd = System.currentTimeMillis(); + log.debug("Deserialization and unzip of {} components took {} ms", list.size(), (desEnd - desStart)); + } + } + log.debug("Number of processed components from cache is {}", foundResources.size()); + + Set<String> notFoundInCache = immutablePair.getRight(); + notFoundResources.addAll(notFoundInCache); + + if (log.isDebugEnabled()) { + log.debug("Number of components fetched is {}", foundResources.size()); + log.debug("Number of components non cached is {}", notFoundResources.size()); + } + + return Either.left(result); + } + + /** + * get components for catalog + * + * @param components + * @param componentTypeEnum + * @return + */ + public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsForCatalog( + Map<String, Long> components, ComponentTypeEnum componentTypeEnum) { + + if (!isEnabled()) { + log.debug("In getComponentsForCatalog for type {}. Cache is disabled.", + componentTypeEnum.name().toLowerCase()); + return Either.right(ActionStatus.NOT_ALLOWED); + } + log.debug("In getComponentsForCatalog for type {}", componentTypeEnum.name().toLowerCase()); + + Function<List<Component>, List<Component>> filterFieldsFunc = this::filterForCatalog; + + Map<String, Long> leftComponentsForSearch = new HashMap<>(); + leftComponentsForSearch.putAll(components); + + // get components from inmemory cache + List<Component> componentsFromMemory = null; + if (catalogInMemoryEnabled) { + componentsFromMemory = getDataFromInMemoryCache(components.keySet(), componentTypeEnum); + log.debug("The number of components of type {} fetched from memory is {}", + componentTypeEnum.name().toLowerCase(), + componentsFromMemory == null ? 0 : componentsFromMemory.size()); + if (componentsFromMemory != null) { + List<String> ignoredComponents = new ArrayList<>(); + for (Component componentFromMem : componentsFromMemory) { + if (componentFromMem.getLastUpdateDate().longValue() != components + .get(componentFromMem.getUniqueId()).longValue()) { + // Ignore the component from memory + ignoredComponents.add(componentFromMem.getUniqueId()); + } + } + + log.debug("Number of components from type {} ignored from memory cache is {}", + componentTypeEnum.name().toLowerCase(), ignoredComponents.size()); + // remove from memory result the components which are not valid + componentsFromMemory = componentsFromMemory.stream() + .filter(p -> !ignoredComponents.contains(p.getUniqueId())).collect(Collectors.toList()); + // Remove from leftComponentsForSearch the valid components from + // memory + componentsFromMemory.forEach(p -> leftComponentsForSearch.remove(p.getUniqueId())); + + } + } else { + log.debug("Catalog InMemory cache is disabled"); + } + + log.debug("Number of components from type {} needed to fetch is {}", componentTypeEnum.name().toLowerCase(), + leftComponentsForSearch.size()); + + // get components from cassandra cache and filter each component + Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> result = getComponents( + leftComponentsForSearch, filterFieldsFunc); + + if (result.isLeft()) { + // add inmemory components to the valid components(not dirty) + List<Component> foundComponents = result.left().value().getLeft(); + if (componentsFromMemory != null) { + foundComponents.addAll(componentsFromMemory); + } + if (catalogInMemoryEnabled) { + updateCatalogInMemoryCacheWithCertified(foundComponents, componentTypeEnum); + } + } + + return result; + } + + /** + * @param components + * - Map of <componentUniqueId, last update date> + * @param filterFieldsFunc + * @return + */ + public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponents(Map<String, Long> components, + Function<List<Component>, List<Component>> filterFieldsFunc) { + + if (!isEnabled()) { + log.debug(COMPONENT_CACHE_IS_DISABLED); + return Either.right(ActionStatus.NOT_ALLOWED); + } + + Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull( + components); + + if (componentsFull.isRight()) { + return Either.right(componentsFull.right().value()); + } + + ImmutablePair<List<Component>, Set<String>> immutablePair = componentsFull.left().value(); + List<Component> foundResources = immutablePair.left; + Set<String> notFoundResources = immutablePair.right; + + List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources); + + ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<>( + filterdFoundResources, notFoundResources); + + return Either.left(result); + + } + + /** + * get the component and its modification time from cache + * + * @param componentUid + * @param filterFieldsFunc + * @return + */ + public Either<ImmutablePair<Component, Long>, ActionStatus> getComponentAndTime(String componentUid, + Function<Component, Component> filterFieldsFunc) { + + Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache( + componentUid, null, filterFieldsFunc); + + if (componentFromCache.isRight()) { + return Either.right(componentFromCache.right().value()); + } + + ImmutablePair<Component, ComponentCacheData> immutablePair = componentFromCache.left().value(); + + ImmutablePair<Component, Long> result = new ImmutablePair<>(immutablePair.left, + immutablePair.right.getModificationTime().getTime()); + + return Either.left(result); + } + + private Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> getComponentFromCache( + String componentUid, Long lastModificationTime, Function<Component, Component> filterFieldsFunc) { + if (!isEnabled()) { + return Either.right(ActionStatus.NOT_ALLOWED); + } + + Either<ComponentCacheData, ActionStatus> componentRes = componentCassandraDao.getComponent(componentUid); + + if (componentRes.isRight()) { + return Either.right(componentRes.right().value()); + } + + ComponentCacheData componentCacheData = componentRes.left().value(); + + if (lastModificationTime != null) { + long cacheCompModificationTime = componentCacheData.getModificationTime().getTime(); + if (lastModificationTime != cacheCompModificationTime) { + log.debug( + "Component {} found in cache but its modification time {} does not match to the timestamp in cache {}.", + componentUid, lastModificationTime, cacheCompModificationTime); + return Either.right(ActionStatus.INVALID_CONTENT); + } + } + + Either<? extends Component, Boolean> convertRes = convertComponentCacheToComponent(componentCacheData); + if (convertRes.isRight()) { + return Either.right(ActionStatus.CONVERT_COMPONENT_ERROR); + } + + Component component = convertRes.left().value(); + + Component filteredComponent = component; + if (filterFieldsFunc != null) { + filteredComponent = filterFieldsFunc.apply(component); + } + + ImmutablePair<Component, ComponentCacheData> result = new ImmutablePair<>( + filteredComponent, componentCacheData); + + return Either.left(result); + } + + public ActionStatus deleteComponentFromCache(String id) { + if (!isEnabled()) { + return ActionStatus.NOT_ALLOWED; + } + CassandraOperationStatus status = this.componentCassandraDao.deleteComponent(id); + if (CassandraOperationStatus.OK.equals(status)) { + return ActionStatus.OK; + } else { + log.debug("delete component failed with error {}", status); + return ActionStatus.GENERAL_ERROR; + } + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/DaoInfo.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/DaoInfo.java index c7bcc54d3d..4880afee81 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/DaoInfo.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/DaoInfo.java @@ -23,19 +23,19 @@ package org.openecomp.sdc.be.model.cache; import org.openecomp.sdc.be.model.jsontitan.operations.ToscaOperationFacade; public class DaoInfo { - private ToscaOperationFacade toscaOperationFacade; - private ComponentCache ComponentCache; + private ToscaOperationFacade toscaOperationFacade; + private ComponentCache ComponentCache; - public DaoInfo(ToscaOperationFacade toscaOperationFacade, org.openecomp.sdc.be.model.cache.ComponentCache componentCache) { - this.toscaOperationFacade = toscaOperationFacade; - ComponentCache = componentCache; - } + public DaoInfo(ToscaOperationFacade toscaOperationFacade, org.openecomp.sdc.be.model.cache.ComponentCache componentCache) { + this.toscaOperationFacade = toscaOperationFacade; + ComponentCache = componentCache; + } - public ToscaOperationFacade getToscaOperationFacade() { - return toscaOperationFacade; - } + public ToscaOperationFacade getToscaOperationFacade() { + return toscaOperationFacade; + } - public org.openecomp.sdc.be.model.cache.ComponentCache getComponentCache() { - return ComponentCache; - } + public org.openecomp.sdc.be.model.cache.ComponentCache getComponentCache() { + return ComponentCache; + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/CheckAndUpdateJob.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/CheckAndUpdateJob.java index ff1c21c7bd..daf14b0b79 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/CheckAndUpdateJob.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/CheckAndUpdateJob.java @@ -20,8 +20,7 @@ package org.openecomp.sdc.be.model.cache.jobs; -import java.util.function.Function; - +import fj.data.Either; import org.apache.commons.lang3.tuple.ImmutablePair; import org.openecomp.sdc.be.dao.api.ActionStatus; import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum; @@ -29,103 +28,102 @@ import org.openecomp.sdc.be.model.Component; import org.openecomp.sdc.be.model.cache.DaoInfo; import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus; import org.openecomp.sdc.be.resources.data.ComponentMetadataData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.openecomp.sdc.common.log.wrappers.Logger; -import fj.data.Either; +import java.util.function.Function; /** * Created by mlando on 9/7/2016. */ public class CheckAndUpdateJob extends Job { - private static Logger log = LoggerFactory.getLogger(CheckAndUpdateJob.class.getName()); + private static final Logger log = Logger.getLogger(CheckAndUpdateJob.class.getName()); - public CheckAndUpdateJob(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { - super(daoInfo, componentId, nodeTypeEnum, timestamp); - } + public CheckAndUpdateJob(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { + super(daoInfo, componentId, nodeTypeEnum, timestamp); + } - @Override - public Object doWork() { - log.trace("starting work on job."); - log.trace("update cache for componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, nodeTypeEnum, - timestamp); + @Override + public Object doWork() { + log.trace("starting work on job."); + log.trace("update cache for componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, nodeTypeEnum, + timestamp); - try { + try { - // get from cache - Either<ImmutablePair<Component, Long>, ActionStatus> cacheResult = daoInfo.getComponentCache() - .getComponentAndTime(componentId, Function.identity()); - // if error while getting from cache abort and update - if (cacheResult.isRight()) { - // genral error - if (!ActionStatus.RESOURCE_NOT_FOUND.equals(cacheResult.right().value()) - && !ActionStatus.INVALID_CONTENT.equals(cacheResult.right().value())) { - log.debug("failed to get component:{} from cache error:{}", componentId, - cacheResult.right().value()); - return false; - } - // component not in cache put there - else { - return updateCache(componentId, nodeTypeEnum, timestamp); - } - } - ImmutablePair<Component, Long> recored = cacheResult.left().value(); - // the cache has allready been updated exit - if (this.timestamp < recored.getRight()) { - log.debug("job timestemp:{} is smaller then the cache timestamp:{} no update is needed.", - this.timestamp, recored.getRight()); - return false; - } - return updateCache(componentId, nodeTypeEnum, timestamp); + // get from cache + Either<ImmutablePair<Component, Long>, ActionStatus> cacheResult = daoInfo.getComponentCache() + .getComponentAndTime(componentId, Function.identity()); + // if error while getting from cache abort and update + if (cacheResult.isRight()) { + // genral error + if (!ActionStatus.RESOURCE_NOT_FOUND.equals(cacheResult.right().value()) + && !ActionStatus.INVALID_CONTENT.equals(cacheResult.right().value())) { + log.debug("failed to get component:{} from cache error:{}", componentId, + cacheResult.right().value()); + return false; + } + // component not in cache put there + else { + return updateCache(componentId, nodeTypeEnum, timestamp); + } + } + ImmutablePair<Component, Long> recored = cacheResult.left().value(); + // the cache has allready been updated exit + if (this.timestamp < recored.getRight()) { + log.debug("job timestemp:{} is smaller then the cache timestamp:{} no update is needed.", + this.timestamp, recored.getRight()); + return false; + } + return updateCache(componentId, nodeTypeEnum, timestamp); - } catch (Exception e) { - log.debug("an exception was encountered during CheckAndUpdateJob", e); - } finally { - daoInfo.getToscaOperationFacade().commit(); - } - return false; - } + } catch (Exception e) { + log.debug("an exception was encountered during CheckAndUpdateJob", e); + } finally { + daoInfo.getToscaOperationFacade().commit(); + } + return false; + } - /** - * @param componentId - * @param nodeTypeEnum - * @return - */ - private boolean updateCache(String componentId, NodeTypeEnum nodeTypeEnum, Long timestamp) { - // get component from cache - Either<ComponentMetadataData, StorageOperationStatus> metaDataRes = getComponentMetaData(componentId, - nodeTypeEnum); - if (metaDataRes.isRight()) { - return false; - } - ComponentMetadataData metaData = metaDataRes.left().value(); - // the job time is older then the one on graph nothing to do there is a - // job that will handle this. - Long graphTimestamp = metaData.getMetadataDataDefinition().getLastUpdateDate(); - if (timestamp < graphTimestamp) { - log.debug( - "the job timestamp:{} is smaller then the graph timestamp:{}. exiting because another job will update the cache.", - timestamp, graphTimestamp); - return false; - } else { - // update cache - // get component from grath - Either<Component, StorageOperationStatus> componentRes = daoInfo.getToscaOperationFacade().getToscaElement(componentId); - if (componentRes.isRight()) { - log.debug("failed to get full component:{} from graph status:{}", componentId, - componentRes.right().value()); - return false; - } - Component component = componentRes.left().value(); - // store in cache - if (!this.daoInfo.getComponentCache().setComponent(component, nodeTypeEnum)) { - log.debug("failed to store componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); - return false; - } - } - log.debug("cache successfully updated for componentId:{} nodeTypeEnum:{} timestemp:{}.", componentId, - nodeTypeEnum, timestamp); - return true; - } + /** + * @param componentId + * @param nodeTypeEnum + * @return + */ + private boolean updateCache(String componentId, NodeTypeEnum nodeTypeEnum, Long timestamp) { + // get component from cache + Either<ComponentMetadataData, StorageOperationStatus> metaDataRes = getComponentMetaData(componentId, + nodeTypeEnum); + if (metaDataRes.isRight()) { + return false; + } + ComponentMetadataData metaData = metaDataRes.left().value(); + // the job time is older then the one on graph nothing to do there is a + // job that will handle this. + Long graphTimestamp = metaData.getMetadataDataDefinition().getLastUpdateDate(); + if (timestamp < graphTimestamp) { + log.debug( + "the job timestamp:{} is smaller then the graph timestamp:{}. exiting because another job will update the cache.", + timestamp, graphTimestamp); + return false; + } else { + // update cache + // get component from grath + Either<Component, StorageOperationStatus> componentRes = daoInfo.getToscaOperationFacade().getToscaElement(componentId); + if (componentRes.isRight()) { + log.debug("failed to get full component:{} from graph status:{}", componentId, + componentRes.right().value()); + return false; + } + Component component = componentRes.left().value(); + // store in cache + if (!this.daoInfo.getComponentCache().setComponent(component, nodeTypeEnum)) { + log.debug("failed to store componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); + return false; + } + } + log.debug("cache successfully updated for componentId:{} nodeTypeEnum:{} timestemp:{}.", componentId, + nodeTypeEnum, timestamp); + return true; + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/DeleteJob.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/DeleteJob.java index 20dd4141f9..0d883a7877 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/DeleteJob.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/DeleteJob.java @@ -23,38 +23,37 @@ package org.openecomp.sdc.be.model.cache.jobs; import org.openecomp.sdc.be.dao.api.ActionStatus; import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum; import org.openecomp.sdc.be.model.cache.DaoInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.openecomp.sdc.common.log.wrappers.Logger; /** * Created by mlando on 9/20/2016. */ public class DeleteJob extends Job { - private static Logger log = LoggerFactory.getLogger(DeleteJob.class.getName()); + private static final Logger log = Logger.getLogger(DeleteJob.class.getName()); - public DeleteJob(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { - super(daoInfo, componentId, nodeTypeEnum, timestamp); + public DeleteJob(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { + super(daoInfo, componentId, nodeTypeEnum, timestamp); - } + } - @Override - public Object doWork() { - try { - log.trace("starting work on job."); - log.trace("delete component in cache, componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, - nodeTypeEnum, timestamp); - ActionStatus status = this.daoInfo.getComponentCache().deleteComponentFromCache(componentId); - if (!ActionStatus.OK.equals(status)) { - log.debug("failed to delete componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); - return false; - } - log.trace("cache successfully deleted componentId:{} nodeTypeEnum:{} timestamp:{}.", componentId, - nodeTypeEnum, timestamp); - return true; - } catch (Exception e) { - log.debug("an exception was encountered durring deletejob", e); - } - return false; + @Override + public Object doWork() { + try { + log.trace("starting work on job."); + log.trace("delete component in cache, componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, + nodeTypeEnum, timestamp); + ActionStatus status = this.daoInfo.getComponentCache().deleteComponentFromCache(componentId); + if (!ActionStatus.OK.equals(status)) { + log.debug("failed to delete componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); + return false; + } + log.trace("cache successfully deleted componentId:{} nodeTypeEnum:{} timestamp:{}.", componentId, + nodeTypeEnum, timestamp); + return true; + } catch (Exception e) { + log.debug("an exception was encountered durring deletejob", e); + } + return false; - } + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/Job.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/Job.java index f701aaf661..593df2b434 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/Job.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/Job.java @@ -20,70 +20,68 @@ package org.openecomp.sdc.be.model.cache.jobs; +import fj.data.Either; import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum; import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum; import org.openecomp.sdc.be.model.Component; import org.openecomp.sdc.be.model.cache.DaoInfo; import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus; import org.openecomp.sdc.be.resources.data.ComponentMetadataData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import fj.data.Either; +import org.openecomp.sdc.common.log.wrappers.Logger; public abstract class Job<E> { - private static Logger log = LoggerFactory.getLogger(Job.class.getName()); - protected DaoInfo daoInfo; - protected String componentId; - protected long timestamp; - protected NodeTypeEnum nodeTypeEnum; + private static final Logger log = Logger.getLogger(Job.class.getName()); + protected DaoInfo daoInfo; + protected String componentId; + protected long timestamp; + protected NodeTypeEnum nodeTypeEnum; - protected Job(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { - this.daoInfo = daoInfo; - this.componentId = componentId; - this.timestamp = timestamp; - this.nodeTypeEnum = nodeTypeEnum; - } + protected Job(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { + this.daoInfo = daoInfo; + this.componentId = componentId; + this.timestamp = timestamp; + this.nodeTypeEnum = nodeTypeEnum; + } - protected Job(DaoInfo daoInfo, Component component, NodeTypeEnum nodeTypeEnum) { - this.daoInfo = daoInfo; - this.componentId = component.getUniqueId(); - this.timestamp = component.getLastUpdateDate(); - this.nodeTypeEnum = nodeTypeEnum; - } + protected Job(DaoInfo daoInfo, Component component, NodeTypeEnum nodeTypeEnum) { + this.daoInfo = daoInfo; + this.componentId = component.getUniqueId(); + this.timestamp = component.getLastUpdateDate(); + this.nodeTypeEnum = nodeTypeEnum; + } - public abstract E doWork(); + public abstract E doWork(); - protected Either<ComponentMetadataData, StorageOperationStatus> getComponentMetaData(String componentId, - NodeTypeEnum nodeTypeEnum) { - Either<ComponentMetadataData, StorageOperationStatus> metaDataRes = daoInfo.getToscaOperationFacade().getComponentMetadata(componentId); - if (metaDataRes.isRight()) { - // in case we cant find the component on graph exit - if (StorageOperationStatus.NOT_FOUND.equals(metaDataRes.right().value())) { - log.debug("failed to locate component:{} on graph status:{}", componentId, metaDataRes.right().value()); - } else { - log.debug("failed to get component:{} from graph status:{}", componentId, metaDataRes.right().value()); - } - } - return metaDataRes; - } + protected Either<ComponentMetadataData, StorageOperationStatus> getComponentMetaData(String componentId, + NodeTypeEnum nodeTypeEnum) { + Either<ComponentMetadataData, StorageOperationStatus> metaDataRes = daoInfo.getToscaOperationFacade().getComponentMetadata(componentId); + if (metaDataRes.isRight()) { + // in case we cant find the component on graph exit + if (StorageOperationStatus.NOT_FOUND.equals(metaDataRes.right().value())) { + log.debug("failed to locate component:{} on graph status:{}", componentId, metaDataRes.right().value()); + } else { + log.debug("failed to get component:{} from graph status:{}", componentId, metaDataRes.right().value()); + } + } + return metaDataRes; + } - protected NodeTypeEnum getNodeTypeFromComponentType(ComponentTypeEnum type) { - NodeTypeEnum result = null; - switch (type) { - case PRODUCT: - result = NodeTypeEnum.Product; - break; - case RESOURCE: - result = NodeTypeEnum.Resource; - break; - case SERVICE: - result = NodeTypeEnum.Service; - break; - default: + protected NodeTypeEnum getNodeTypeFromComponentType(ComponentTypeEnum type) { + NodeTypeEnum result = null; + switch (type) { + case PRODUCT: + result = NodeTypeEnum.Product; + break; + case RESOURCE: + result = NodeTypeEnum.Resource; + break; + case SERVICE: + result = NodeTypeEnum.Service; + break; + default: - } - return result; + } + return result; - } + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/OverrideJob.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/OverrideJob.java index 641d125122..cb33a02076 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/OverrideJob.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/OverrideJob.java @@ -20,54 +20,52 @@ package org.openecomp.sdc.be.model.cache.jobs; +import fj.data.Either; import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum; import org.openecomp.sdc.be.model.Component; import org.openecomp.sdc.be.model.cache.DaoInfo; import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import fj.data.Either; +import org.openecomp.sdc.common.log.wrappers.Logger; /** * Created by mlando on 9/20/2016. */ public class OverrideJob extends Job { - private static Logger log = LoggerFactory.getLogger(OverrideJob.class.getName()); + private static final Logger log = Logger.getLogger(OverrideJob.class.getName()); - public OverrideJob(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { - super(daoInfo, componentId, nodeTypeEnum, timestamp); + public OverrideJob(DaoInfo daoInfo, String componentId, NodeTypeEnum nodeTypeEnum, long timestamp) { + super(daoInfo, componentId, nodeTypeEnum, timestamp); - } + } - @Override - public Object doWork() { - try { - log.trace("starting work on job."); - log.trace("override component in cache, componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, - nodeTypeEnum, timestamp); - // get component from grath - Either<Component, StorageOperationStatus> componentRes = daoInfo.getToscaOperationFacade().getToscaElement(componentId); - if (componentRes.isRight()) { - log.debug("failed to get full component:{} from graph status:{}", componentId, - componentRes.right().value()); - return false; - } - Component component = componentRes.left().value(); - // store in cache - if (!this.daoInfo.getComponentCache().setComponent(component, nodeTypeEnum)) { - log.debug("failed to store componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); - return false; - } - log.debug("cache successfully overrided componentId:{} nodeTypeEnum:{} timestemp:{}.", componentId, - nodeTypeEnum, timestamp); - return true; - } catch (Exception e) { - log.debug("an exception was encountered during OverrideJob", e); - } finally { - this.daoInfo.getToscaOperationFacade().commit(); - } - return false; + @Override + public Object doWork() { + try { + log.trace("starting work on job."); + log.trace("override component in cache, componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, + nodeTypeEnum, timestamp); + // get component from grath + Either<Component, StorageOperationStatus> componentRes = daoInfo.getToscaOperationFacade().getToscaElement(componentId); + if (componentRes.isRight()) { + log.debug("failed to get full component:{} from graph status:{}", componentId, + componentRes.right().value()); + return false; + } + Component component = componentRes.left().value(); + // store in cache + if (!this.daoInfo.getComponentCache().setComponent(component, nodeTypeEnum)) { + log.debug("failed to store componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); + return false; + } + log.debug("cache successfully overrided componentId:{} nodeTypeEnum:{} timestemp:{}.", componentId, + nodeTypeEnum, timestamp); + return true; + } catch (Exception e) { + log.debug("an exception was encountered during OverrideJob", e); + } finally { + this.daoInfo.getToscaOperationFacade().commit(); + } + return false; - } + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/StoreJob.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/StoreJob.java index c282e5dca5..c8baafe114 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/StoreJob.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/jobs/StoreJob.java @@ -23,37 +23,36 @@ package org.openecomp.sdc.be.model.cache.jobs; import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum; import org.openecomp.sdc.be.model.Component; import org.openecomp.sdc.be.model.cache.DaoInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.openecomp.sdc.common.log.wrappers.Logger; /** * Created by mlando on 9/11/2016. */ public class StoreJob extends Job { - private static Logger log = LoggerFactory.getLogger(StoreJob.class.getName()); - private Component component; + private static final Logger log = Logger.getLogger(StoreJob.class.getName()); + private Component component; - public StoreJob(DaoInfo daoInfo, Component component, NodeTypeEnum nodeTypeEnum) { - super(daoInfo, component, nodeTypeEnum); - this.component = component; - } + public StoreJob(DaoInfo daoInfo, Component component, NodeTypeEnum nodeTypeEnum) { + super(daoInfo, component, nodeTypeEnum); + this.component = component; + } - @Override - public Object doWork() { - try { - log.trace("starting work on job."); - log.trace("store component in cache, componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, nodeTypeEnum, timestamp); - if (!this.daoInfo.getComponentCache().setComponent(component, nodeTypeEnum)) { - log.debug("failed to store componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); - return false; - } - log.debug("cache successfully updated for componentId:{} nodeTypeEnum:{} timestemp:{}.", componentId, nodeTypeEnum, timestamp); - return true; + @Override + public Object doWork() { + try { + log.trace("starting work on job."); + log.trace("store component in cache, componentId:{} of nodeTypeEnum:{} with timestamp:{}.", componentId, nodeTypeEnum, timestamp); + if (!this.daoInfo.getComponentCache().setComponent(component, nodeTypeEnum)) { + log.debug("failed to store componentId:{} nodeTypeEnum:", componentId, nodeTypeEnum); + return false; + } + log.debug("cache successfully updated for componentId:{} nodeTypeEnum:{} timestemp:{}.", componentId, nodeTypeEnum, timestamp); + return true; - } catch (Exception e) { - log.debug("an exception was encountered during StoreJob", e); - } - return false; - } + } catch (Exception e) { + log.debug("an exception was encountered during StoreJob", e); + } + return false; + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/CacheWorker.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/CacheWorker.java index d29ade9c95..e70916f170 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/CacheWorker.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/CacheWorker.java @@ -20,13 +20,12 @@ package org.openecomp.sdc.be.model.cache.workers; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - import org.openecomp.sdc.be.model.cache.jobs.Job; import org.openecomp.sdc.be.workers.Worker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.openecomp.sdc.common.log.wrappers.Logger; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; /** * Created by mlando on 9/6/2016. the class represents a worker the pull job @@ -34,59 +33,59 @@ import org.slf4j.LoggerFactory; * */ public class CacheWorker implements Runnable, IWorker { - private String workerName; - private static Logger log = LoggerFactory.getLogger(Worker.class.getName()); - private LinkedBlockingQueue<Job> jobQueue; - private volatile boolean shutdown = false; + private String workerName; + private static final Logger log = Logger.getLogger(Worker.class.getName()); + private LinkedBlockingQueue<Job> jobQueue; + private volatile boolean shutdown = false; - /** - * constructor - * - * @param workerName - * the name of the given worker - * @param jobQueue - * the queue the worker will block on. - */ - public CacheWorker(String workerName, LinkedBlockingQueue<Job> jobQueue) { - this.workerName = workerName; - this.jobQueue = jobQueue; - } + /** + * constructor + * + * @param workerName + * the name of the given worker + * @param jobQueue + * the queue the worker will block on. + */ + public CacheWorker(String workerName, LinkedBlockingQueue<Job> jobQueue) { + this.workerName = workerName; + this.jobQueue = jobQueue; + } - /** - * the method will try to get a job if one is avilable it will be retrived - * and handled. if no jobs are available the worker will block for 500 - * milliseconds and then it wil check if it needs to shutdown. if not it - * will block again and so on until sutdown or a new job is available - */ - @Override - public void run() { - while (true) { - log.trace("CacheWorker:{} doing work", workerName); - try { - Job job = jobQueue.poll(500, TimeUnit.MILLISECONDS); - if (job != null) { - job.doWork(); - log.trace("worker:{} done with work", workerName); - } - } catch (Throwable e) { - log.debug("worker {} failed during job execution.", workerName); - log.debug("exception", e); - } - if (shutdown) { - log.debug("worker:{} nothing to do stoping", workerName); - break; - } - } + /** + * the method will try to get a job if one is avilable it will be retrived + * and handled. if no jobs are available the worker will block for 500 + * milliseconds and then it wil check if it needs to shutdown. if not it + * will block again and so on until sutdown or a new job is available + */ + @Override + public void run() { + while (true) { + log.trace("CacheWorker:{} doing work", workerName); + try { + Job job = jobQueue.poll(500, TimeUnit.MILLISECONDS); + if (job != null) { + job.doWork(); + log.trace("worker:{} done with work", workerName); + } + } catch (Exception e) { + log.debug("worker {} failed during job execution.", workerName); + log.debug("exception", e); + } + if (shutdown) { + log.debug("worker:{} nothing to do stoping", workerName); + break; + } + } - } + } - /** - * the method sets the shutdown flag, when set the worker will stop it's - * execution as soon as possible with out completing its work - */ - @Override - public void shutDown() { - this.shutdown = true; - } + /** + * the method sets the shutdown flag, when set the worker will stop it's + * execution as soon as possible with out completing its work + */ + @Override + public void shutDown() { + this.shutdown = true; + } } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/IWorker.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/IWorker.java index fcdf9f4148..fa508a4923 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/IWorker.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/IWorker.java @@ -24,5 +24,5 @@ package org.openecomp.sdc.be.model.cache.workers; * Created by mlando on 9/6/2016. */ public interface IWorker { - void shutDown(); + void shutDown(); } diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/SyncWorker.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/SyncWorker.java index 64cc1c8eea..169f857b2b 100644 --- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/SyncWorker.java +++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/workers/SyncWorker.java @@ -20,12 +20,7 @@ package org.openecomp.sdc.be.model.cache.workers; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - +import fj.data.Either; import org.openecomp.sdc.be.dao.api.ActionStatus; import org.openecomp.sdc.be.dao.titan.TitanOperationStatus; import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum; @@ -34,10 +29,13 @@ import org.openecomp.sdc.be.model.operations.impl.CacheMangerOperation; import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder; import org.openecomp.sdc.be.resources.data.ComponentCacheData; import org.openecomp.sdc.be.resources.data.ComponentMetadataData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.openecomp.sdc.common.log.wrappers.Logger; -import fj.data.Either; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** * the class creates a worker that is used to update cache date, in case of @@ -45,225 +43,225 @@ import fj.data.Either; */ public class SyncWorker implements Runnable, IWorker { - private static Logger log = LoggerFactory.getLogger(SyncWorker.class.getName()); - private final CacheMangerOperation cacheMangerOperation; - private final String workerName; - private volatile boolean shutdown = false; - private Map<String, ComponentCacheData> cacheIdAndTimeMap; - private long updateDelayInMilliseconds = 60 * 60 * 1000; + private static final Logger log = Logger.getLogger(SyncWorker.class.getName()); + private final CacheMangerOperation cacheMangerOperation; + private final String workerName; + private volatile boolean shutdown = false; + private Map<String, ComponentCacheData> cacheIdAndTimeMap; + private long updateDelayInMilliseconds = 60 * 60 * 1000L; - /** - * creates the sync worker - * - * @param workerName - * the name of the worker - * @param cacheMangerOperation - * responsible for all persistence's operations to graph and the - * cache - */ - public SyncWorker(String workerName, CacheMangerOperation cacheMangerOperation) { - this.workerName = workerName; - this.cacheMangerOperation = cacheMangerOperation; - } + /** + * creates the sync worker + * + * @param workerName + * the name of the worker + * @param cacheMangerOperation + * responsible for all persistence's operations to graph and the + * cache + */ + public SyncWorker(String workerName, CacheMangerOperation cacheMangerOperation) { + this.workerName = workerName; + this.cacheMangerOperation = cacheMangerOperation; + } - /** - * the method collects all the resources/services/products from graph and - * checks if the component representing them in the cache is valid logic: if - * the record is present in the graph but not in cache -> create a job that - * will update the record oin cache if the timestamp of the record in cache - * is older than the timestamp on the graph -> create a job that will update - * the record oin cache otherwise no update is required - */ - @Override - public void run() { - try { - collectAllCacheRecords(); - syncCacheByComponentType(NodeTypeEnum.Resource); - syncCacheByComponentType(NodeTypeEnum.Service); - syncCacheByComponentType(NodeTypeEnum.Product); - clearCacheRecords(); + /** + * the method collects all the resources/services/products from graph and + * checks if the component representing them in the cache is valid logic: if + * the record is present in the graph but not in cache -> create a job that + * will update the record oin cache if the timestamp of the record in cache + * is older than the timestamp on the graph -> create a job that will update + * the record oin cache otherwise no update is required + */ + @Override + public void run() { + try { + collectAllCacheRecords(); + syncCacheByComponentType(NodeTypeEnum.Resource); + syncCacheByComponentType(NodeTypeEnum.Service); + syncCacheByComponentType(NodeTypeEnum.Product); + clearCacheRecords(); - } catch (Exception e) { - log.debug("sync worker:{} encounered an exception", workerName); - log.debug("exception", e); - } finally { - this.cacheMangerOperation.getTitanGenericDao().commit(); - } - } + } catch (Exception e) { + log.debug("sync worker:{} encounered an exception", workerName); + log.debug("exception", e); + } finally { + this.cacheMangerOperation.getTitanGenericDao().commit(); + } + } - /** - * the method checks for each component in the cache except the ones that - * were update during the sync, if they exist on the graph if not a job to - * remove them is created - */ - private void clearCacheRecords() { - cacheIdAndTimeMap.forEach((k, v) -> { - try { - Either<ComponentMetadataData, TitanOperationStatus> componentFromGraphRes = getComponentMetaData(k, - NodeTypeEnum.getByName(v.getType())); - if (componentFromGraphRes.isRight()) { - TitanOperationStatus error = componentFromGraphRes.right().value(); - if (TitanOperationStatus.NOT_FOUND.equals(error)) { - long delay = System.currentTimeMillis() - v.getModificationTime().getTime(); - if (delay > updateDelayInMilliseconds) { - this.cacheMangerOperation.deleteComponentInCache(k, v.getModificationTime().getTime(), - NodeTypeEnum.getByName(v.getType())); - } else { - log.trace( - "no delete done because an hour did not pass since the delete was done timeSinceUpdate {} < updateDelayInMilliseconds {} ", - delay, updateDelayInMilliseconds); - } - } else { - log.debug("failed to get metadata for id:{} from graph error:{}", k, error); - } - } else { - log.trace("id {} is in graph nothing to do"); - } - } catch (Exception e) { - log.debug("during clean cache records an exception was thrown", e); - } - }); - } + /** + * the method checks for each component in the cache except the ones that + * were update during the sync, if they exist on the graph if not a job to + * remove them is created + */ + private void clearCacheRecords() { + cacheIdAndTimeMap.forEach((k, v) -> { + try { + Either<ComponentMetadataData, TitanOperationStatus> componentFromGraphRes = getComponentMetaData(k, + NodeTypeEnum.getByName(v.getType())); + if (componentFromGraphRes.isRight()) { + TitanOperationStatus error = componentFromGraphRes.right().value(); + if (TitanOperationStatus.NOT_FOUND.equals(error)) { + long delay = System.currentTimeMillis() - v.getModificationTime().getTime(); + if (delay > updateDelayInMilliseconds) { + this.cacheMangerOperation.deleteComponentInCache(k, v.getModificationTime().getTime(), + NodeTypeEnum.getByName(v.getType())); + } else { + log.trace( + "no delete done because an hour did not pass since the delete was done timeSinceUpdate {} < updateDelayInMilliseconds {} ", + delay, updateDelayInMilliseconds); + } + } else { + log.debug("failed to get metadata for id:{} from graph error:{}", k, error); + } + } else { + log.trace("id {} is in graph nothing to do", k); + } + } catch (Exception e) { + log.debug("during clean cache records an exception was thrown", e); + } + }); + } - /** - * the method collects all the records from cache except the component - * itself - */ - public void collectAllCacheRecords() { - Either<List<ComponentCacheData>, ActionStatus> getAllRes = this.cacheMangerOperation.getComponentCache() - .getAllComponentIdTimeAndType(); - if (getAllRes.isRight()) { - log.debug("error while trying to get all records from cache error:{}", getAllRes.right().value()); - cacheIdAndTimeMap = new HashMap<>(); - } else { - cacheIdAndTimeMap = getAllRes.left().value().stream().collect(Collectors.toMap(e -> e.getId(), e -> e)); - } - } + /** + * the method collects all the records from cache except the component + * itself + */ + public void collectAllCacheRecords() { + Either<List<ComponentCacheData>, ActionStatus> getAllRes = this.cacheMangerOperation.getComponentCache() + .getAllComponentIdTimeAndType(); + if (getAllRes.isRight()) { + log.debug("error while trying to get all records from cache error:{}", getAllRes.right().value()); + cacheIdAndTimeMap = new HashMap<>(); + } else { + cacheIdAndTimeMap = getAllRes.left().value().stream().collect(Collectors.toMap(ComponentCacheData::getId, e -> e)); + } + } - /** - * the method checks that the records ot the given type are sync between the - * cache and the graph - * - * @param nodeTypeEnum - * the type of components we want to sync - */ - private void syncCacheByComponentType(NodeTypeEnum nodeTypeEnum) { - if (!this.shutdown) { - log.trace("syncCache records of type:{} .", nodeTypeEnum); - Either<List<ComponentMetadataData>, TitanOperationStatus> getAllResult = getAllComponentsMetaData( - nodeTypeEnum); - List<ComponentMetadataData> componentList = new ArrayList<>(); - if (getAllResult.isRight() && !TitanOperationStatus.NOT_FOUND.equals(getAllResult.right().value())) { - log.debug("error while trying to get all components of type:{} TitanOperationStatus:{}.", nodeTypeEnum, - getAllResult.right().value()); - return; - } - if (getAllResult.isLeft()) { - componentList = getAllResult.left().value(); - log.trace("get all components of type:{} returned:{} components.", nodeTypeEnum, componentList.size()); - } - componentList.forEach(this::checkAndUpdateCacheComponent); - log.trace("syncCache records of type:{} was successful.", nodeTypeEnum); - } - } + /** + * the method checks that the records ot the given type are sync between the + * cache and the graph + * + * @param nodeTypeEnum + * the type of components we want to sync + */ + private void syncCacheByComponentType(NodeTypeEnum nodeTypeEnum) { + if (!this.shutdown) { + log.trace("syncCache records of type:{} .", nodeTypeEnum); + Either<List<ComponentMetadataData>, TitanOperationStatus> getAllResult = getAllComponentsMetaData( + nodeTypeEnum); + List<ComponentMetadataData> componentList = new ArrayList<>(); + if (getAllResult.isRight() && !TitanOperationStatus.NOT_FOUND.equals(getAllResult.right().value())) { + log.debug("error while trying to get all components of type:{} TitanOperationStatus:{}.", nodeTypeEnum, + getAllResult.right().value()); + return; + } + if (getAllResult.isLeft()) { + componentList = getAllResult.left().value(); + log.trace("get all components of type:{} returned:{} components.", nodeTypeEnum, componentList.size()); + } + componentList.forEach(this::checkAndUpdateCacheComponent); + log.trace("syncCache records of type:{} was successful.", nodeTypeEnum); + } + } - /** - * the method compares the given component to the record in the cache if the - * record is not in the cache a job to update the cache for this record will - * be created. if the record is present in the graph but not in cache -> - * create a job that will update the record oin cache if the timestamp of - * the record in cache is older than the timestamp on the graph -> create a - * job that will update the record oin cache if the retried component from - * cache fails to be deserialized -> create job to override it otherwise no - * update is required - * - * @param metadataData - * the date of the node we want to compare to the value in the - * cache - */ - private void checkAndUpdateCacheComponent(ComponentMetadataData metadataData) { - long timeSinceUpdate = System.currentTimeMillis() - - metadataData.getMetadataDataDefinition().getLastUpdateDate(); - if (timeSinceUpdate >= updateDelayInMilliseconds) { - String uid = metadataData.getMetadataDataDefinition().getUniqueId(); - log.trace("checking cache if record for uid:{} needs to be updated.", uid); - Either<Component, ActionStatus> cacheResult = this.cacheMangerOperation.getComponentCache() - .getComponent(uid); - if (cacheResult.isRight()) { - ActionStatus actionStatus = cacheResult.right().value(); - if (ActionStatus.RESOURCE_NOT_FOUND.equals(actionStatus)) { - log.trace("record for uid:{} not found in cache. creating an update job.", uid); - this.cacheMangerOperation.updateComponentInCache(uid, - metadataData.getMetadataDataDefinition().getLastUpdateDate(), - NodeTypeEnum.getByName(metadataData.getLabel())); - } else if (ActionStatus.CONVERT_COMPONENT_ERROR.equals(actionStatus)) { - log.trace("uid:{} found in cache but we failed deserializing it. creating an override job .", uid); - this.cacheMangerOperation.overideComponentInCache(uid, - metadataData.getMetadataDataDefinition().getLastUpdateDate(), - NodeTypeEnum.getByName(metadataData.getLabel())); - } else { - log.debug("during lookup for uid:{} an error accords status:{} .", uid, actionStatus); - } - } else { - log.trace("uid:{} found in cache.", uid); - this.cacheIdAndTimeMap.remove(uid); - Component cacheComponent = cacheResult.left().value(); - Long cacheTimestamp = cacheComponent.getLastUpdateDate(); - Long graphTimestamp = metadataData.getMetadataDataDefinition().getLastUpdateDate(); - if (cacheTimestamp < graphTimestamp) { - log.trace("uid:{} found in cache. cache Timestamp {} < graph timestamp , creating an update job .", - uid, cacheTimestamp, graphTimestamp); - this.cacheMangerOperation.updateComponentInCache(uid, graphTimestamp, - NodeTypeEnum.getByName(metadataData.getLabel())); - } else { - log.trace("uid:{} found in cache. cache Timestamp {} => graph timestamp , no update is needed .", - uid, cacheTimestamp, graphTimestamp); - } - } - } else { - log.trace( - "no update done because an hour did not pass since the update was done timeSinceUpdate {} < updateDelayInMilliseconds {} ", - timeSinceUpdate, updateDelayInMilliseconds); - } - } + /** + * the method compares the given component to the record in the cache if the + * record is not in the cache a job to update the cache for this record will + * be created. if the record is present in the graph but not in cache -> + * create a job that will update the record oin cache if the timestamp of + * the record in cache is older than the timestamp on the graph -> create a + * job that will update the record oin cache if the retried component from + * cache fails to be deserialized -> create job to override it otherwise no + * update is required + * + * @param metadataData + * the date of the node we want to compare to the value in the + * cache + */ + private void checkAndUpdateCacheComponent(ComponentMetadataData metadataData) { + long timeSinceUpdate = System.currentTimeMillis() + - metadataData.getMetadataDataDefinition().getLastUpdateDate(); + if (timeSinceUpdate >= updateDelayInMilliseconds) { + String uid = metadataData.getMetadataDataDefinition().getUniqueId(); + log.trace("checking cache if record for uid:{} needs to be updated.", uid); + Either<Component, ActionStatus> cacheResult = this.cacheMangerOperation.getComponentCache() + .getComponent(uid); + if (cacheResult.isRight()) { + ActionStatus actionStatus = cacheResult.right().value(); + if (ActionStatus.RESOURCE_NOT_FOUND.equals(actionStatus)) { + log.trace("record for uid:{} not found in cache. creating an update job.", uid); + this.cacheMangerOperation.updateComponentInCache(uid, + metadataData.getMetadataDataDefinition().getLastUpdateDate(), + NodeTypeEnum.getByName(metadataData.getLabel())); + } else if (ActionStatus.CONVERT_COMPONENT_ERROR.equals(actionStatus)) { + log.trace("uid:{} found in cache but we failed deserializing it. creating an override job .", uid); + this.cacheMangerOperation.overideComponentInCache(uid, + metadataData.getMetadataDataDefinition().getLastUpdateDate(), + NodeTypeEnum.getByName(metadataData.getLabel())); + } else { + log.debug("during lookup for uid:{} an error accords status:{} .", uid, actionStatus); + } + } else { + log.trace("uid:{} found in cache.", uid); + this.cacheIdAndTimeMap.remove(uid); + Component cacheComponent = cacheResult.left().value(); + Long cacheTimestamp = cacheComponent.getLastUpdateDate(); + Long graphTimestamp = metadataData.getMetadataDataDefinition().getLastUpdateDate(); + if (cacheTimestamp < graphTimestamp) { + log.trace("uid:{} found in cache. cache Timestamp {} < graph timestamp {} , creating an update job .", + uid, cacheTimestamp, graphTimestamp); + this.cacheMangerOperation.updateComponentInCache(uid, graphTimestamp, + NodeTypeEnum.getByName(metadataData.getLabel())); + } else { + log.trace("uid:{} found in cache. cache Timestamp {} => graph timestamp {}, no update is needed .", + uid, cacheTimestamp, graphTimestamp); + } + } + } else { + log.trace( + "no update done because an hour did not pass since the update was done timeSinceUpdate {} < updateDelayInMilliseconds {} ", + timeSinceUpdate, updateDelayInMilliseconds); + } + } - /** - * the method sets the shutdown flag, when set the worker will stop it's - * execution as soon as possible with out completing its work - */ - @Override - public void shutDown() { - log.debug("syncWorker {} shuting down.", workerName); - this.shutdown = true; - } + /** + * the method sets the shutdown flag, when set the worker will stop it's + * execution as soon as possible with out completing its work + */ + @Override + public void shutDown() { + log.debug("syncWorker {} shuting down.", workerName); + this.shutdown = true; + } - /** - * the method retrives all nodes matching the given node type from the graph - * - * @param nodeTypeEnum - * node type we want to lookup on the graph - * @return a list of retrieved nodes matching the given type or not found in - * case no nodes were found or error in case of failure - */ - private Either<List<ComponentMetadataData>, TitanOperationStatus> getAllComponentsMetaData( - NodeTypeEnum nodeTypeEnum) { - return this.cacheMangerOperation.getTitanGenericDao().getByCriteria(nodeTypeEnum, null, - ComponentMetadataData.class); - } + /** + * the method retrives all nodes matching the given node type from the graph + * + * @param nodeTypeEnum + * node type we want to lookup on the graph + * @return a list of retrieved nodes matching the given type or not found in + * case no nodes were found or error in case of failure + */ + private Either<List<ComponentMetadataData>, TitanOperationStatus> getAllComponentsMetaData( + NodeTypeEnum nodeTypeEnum) { + return this.cacheMangerOperation.getTitanGenericDao().getByCriteria(nodeTypeEnum, null, + ComponentMetadataData.class); + } - /** - * the method retrieves the metadata from graph for the given id - * - * @param uid - * the unique id of the component we want to retrieve - * @param nodeTypeEnum - * the type of the recored we want to retrieve - * @return the meta dat of the component or the error encountered during the - * get - */ - private Either<ComponentMetadataData, TitanOperationStatus> getComponentMetaData(String uid, - NodeTypeEnum nodeTypeEnum) { - return this.cacheMangerOperation.getTitanGenericDao().getNode(UniqueIdBuilder.getKeyByNodeType(nodeTypeEnum), - uid, ComponentMetadataData.class); - } + /** + * the method retrieves the metadata from graph for the given id + * + * @param uid + * the unique id of the component we want to retrieve + * @param nodeTypeEnum + * the type of the recored we want to retrieve + * @return the meta dat of the component or the error encountered during the + * get + */ + private Either<ComponentMetadataData, TitanOperationStatus> getComponentMetaData(String uid, + NodeTypeEnum nodeTypeEnum) { + return this.cacheMangerOperation.getTitanGenericDao().getNode(UniqueIdBuilder.getKeyByNodeType(nodeTypeEnum), + uid, ComponentMetadataData.class); + } } |