diff options
Diffstat (limited to 'sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync')
39 files changed, 5477 insertions, 0 deletions
diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/AbstractEntitySynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/AbstractEntitySynchronizer.java new file mode 100644 index 0000000..a2b750b --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/AbstractEntitySynchronizer.java @@ -0,0 +1,524 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.util.EnumSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.mdc.MdcContext; +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ActiveInventoryAdapter; +import org.onap.aai.sparky.dal.ElasticSearchAdapter; +import org.onap.aai.sparky.dal.NetworkTransaction; +import org.onap.aai.sparky.dal.aai.ActiveInventoryEntityStatistics; +import org.onap.aai.sparky.dal.aai.ActiveInventoryProcessingExceptionStatistics; +import org.onap.aai.sparky.dal.elasticsearch.ElasticSearchEntityStatistics; +import org.onap.aai.sparky.dal.rest.HttpMethod; +import org.onap.aai.sparky.dal.rest.RestOperationalStatistics; +import org.onap.aai.sparky.logging.AaiUiMsgs; +import org.onap.aai.sparky.sync.config.NetworkStatisticsConfig; +import org.onap.aai.sparky.util.NodeUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * The Class AbstractEntitySynchronizer. + * + * @author davea. + */ +public abstract class AbstractEntitySynchronizer { + + protected static final int VERSION_CONFLICT_EXCEPTION_CODE = 409; + protected static final Integer RETRY_COUNT_PER_ENTITY_LIMIT = new Integer(3); + + protected final Logger logger; + protected ObjectMapper mapper; + protected long syncDurationInMs; + + /** + * The Enum StatFlag. + */ + protected enum StatFlag { + AAI_REST_STATS, AAI_ENTITY_STATS, AAI_PROCESSING_EXCEPTION_STATS, + AAI_TASK_PROCESSING_STATS, ES_REST_STATS, ES_ENTITY_STATS, ES_TASK_PROCESSING_STATS + } + + protected EnumSet<StatFlag> enabledStatFlags; + + protected ElasticSearchAdapter elasticSearchAdapter; + protected ActiveInventoryAdapter aaiAdapter; + + protected ExecutorService synchronizerExecutor; + protected ExecutorService aaiExecutor; + protected ExecutorService esExecutor; + + private RestOperationalStatistics esRestStats; + protected ElasticSearchEntityStatistics esEntityStats; + + private RestOperationalStatistics aaiRestStats; + protected ActiveInventoryEntityStatistics aaiEntityStats; + private ActiveInventoryProcessingExceptionStatistics aaiProcessingExceptionStats; + + private TaskProcessingStats aaiTaskProcessingStats; + private TaskProcessingStats esTaskProcessingStats; + + private TransactionRateMonitor aaiTransactionRateController; + private TransactionRateMonitor esTransactionRateController; + + protected AtomicInteger aaiWorkOnHand; + protected AtomicInteger esWorkOnHand; + protected String synchronizerName; + + protected abstract boolean isSyncDone(); + protected boolean shouldSkipSync; + + public String getActiveInventoryStatisticsReport() { + + StringBuilder sb = new StringBuilder(128); + + if (enabledStatFlags.contains(StatFlag.AAI_REST_STATS)) { + sb.append("\n\n ").append("REST Operational Stats:"); + sb.append(aaiRestStats.getStatisticsReport()); + } + + if (enabledStatFlags.contains(StatFlag.AAI_ENTITY_STATS)) { + sb.append("\n\n ").append("Entity Stats:"); + sb.append(aaiEntityStats.getStatisticsReport()); + } + + if (enabledStatFlags.contains(StatFlag.AAI_PROCESSING_EXCEPTION_STATS)) { + sb.append("\n\n ").append("Processing Exception Stats:"); + sb.append(aaiProcessingExceptionStats.getStatisticsReport()); + } + + return sb.toString(); + + } + + public String getElasticSearchStatisticsReport() { + + StringBuilder sb = new StringBuilder(128); + + if (enabledStatFlags.contains(StatFlag.ES_REST_STATS)) { + sb.append("\n\n ").append("REST Operational Stats:"); + sb.append(esRestStats.getStatisticsReport()); + } + + if (enabledStatFlags.contains(StatFlag.ES_ENTITY_STATS)) { + sb.append("\n\n ").append("Entity Stats:"); + sb.append(esEntityStats.getStatisticsReport()); + } + + return sb.toString(); + + } + + /** + * Adds the active inventory stat report. + * + * @param sb the sb + */ + private void addActiveInventoryStatReport(StringBuilder sb) { + + if (sb == null) { + return; + } + + sb.append("\n\n AAI"); + sb.append(getActiveInventoryStatisticsReport()); + + double currentTps = 0; + if (enabledStatFlags.contains(StatFlag.AAI_TASK_PROCESSING_STATS)) { + sb.append("\n\n ").append("Task Processor Stats:"); + sb.append(aaiTaskProcessingStats.getStatisticsReport(false, " ")); + + currentTps = aaiTransactionRateController.getCurrentTps(); + + sb.append("\n ").append("Current TPS: ").append(currentTps); + } + + sb.append("\n ").append("Current WOH: ").append(aaiWorkOnHand.get()); + + if (enabledStatFlags.contains(StatFlag.AAI_TASK_PROCESSING_STATS)) { + if (currentTps > 0) { + double numMillisecondsToCompletion = (aaiWorkOnHand.get() / currentTps) * 1000; + sb.append("\n ").append("SyncDurationRemaining=") + .append(NodeUtils.getDurationBreakdown((long) numMillisecondsToCompletion)); + } + } + + } + + /** + * Adds the elastic stat report. + * + * @param sb the sb + */ + private void addElasticStatReport(StringBuilder sb) { + + if (sb == null) { + return; + } + + sb.append("\n\n ELASTIC"); + sb.append(getElasticSearchStatisticsReport()); + + double currentTps = 0; + + if (enabledStatFlags.contains(StatFlag.ES_TASK_PROCESSING_STATS)) { + sb.append("\n\n ").append("Task Processor Stats:"); + sb.append(esTaskProcessingStats.getStatisticsReport(false, " ")); + + currentTps = esTransactionRateController.getCurrentTps(); + + sb.append("\n ").append("Current TPS: ").append(currentTps); + } + + sb.append("\n ").append("Current WOH: ").append(esWorkOnHand.get()); + + if (enabledStatFlags.contains(StatFlag.ES_TASK_PROCESSING_STATS)) { + if (currentTps > 0) { + double numMillisecondsToCompletion = (esWorkOnHand.get() / currentTps) * 1000; + sb.append("\n ").append("SyncDurationRemaining=") + .append(NodeUtils.getDurationBreakdown((long) numMillisecondsToCompletion)); + } + } + + + } + + /** + * Gets the stat report. + * + * @param syncOpTimeInMs the sync op time in ms + * @param showFinalReport the show final report + * @return the stat report + */ + protected String getStatReport(long syncOpTimeInMs, boolean showFinalReport) { + + StringBuilder sb = new StringBuilder(128); + + sb.append("\n").append(synchronizerName + " Statistics: ( Sync Operation Duration = " + + NodeUtils.getDurationBreakdown(syncOpTimeInMs) + " )"); + + addActiveInventoryStatReport(sb); + addElasticStatReport(sb); + + if (showFinalReport) { + sb.append("\n\n ").append("Sync Completed!\n"); + } else { + sb.append("\n\n ").append("Sync in Progress...\n"); + } + + return sb.toString(); + + } + + protected String indexName; + protected long syncStartedTimeStampInMs; + + /** + * Instantiates a new abstract entity synchronizer. + * + * @param logger the logger + * @param syncName the sync name + * @param numSyncWorkers the num sync workers + * @param numActiveInventoryWorkers the num active inventory workers + * @param numElasticsearchWorkers the num elasticsearch workers + * @param indexName the index name + * @throws Exception the exception + */ + protected AbstractEntitySynchronizer(Logger logger, String syncName, int numSyncWorkers, + int numActiveInventoryWorkers, int numElasticsearchWorkers, String indexName, + NetworkStatisticsConfig aaiStatConfig, NetworkStatisticsConfig esStatConfig) + throws Exception { + this.logger = logger; + this.synchronizerExecutor = + NodeUtils.createNamedExecutor(syncName + "-INTERNAL", numSyncWorkers, logger); + this.aaiExecutor = + NodeUtils.createNamedExecutor(syncName + "-AAI", numActiveInventoryWorkers, logger); + this.esExecutor = + NodeUtils.createNamedExecutor(syncName + "-ES", numElasticsearchWorkers, logger); + this.mapper = new ObjectMapper(); + this.indexName = indexName; + this.esRestStats = new RestOperationalStatistics(); + this.esEntityStats = new ElasticSearchEntityStatistics(); + this.aaiRestStats = new RestOperationalStatistics(); + this.aaiEntityStats = new ActiveInventoryEntityStatistics(); + this.aaiProcessingExceptionStats = new ActiveInventoryProcessingExceptionStatistics(); + this.aaiTaskProcessingStats = + new TaskProcessingStats(aaiStatConfig); + this.esTaskProcessingStats = + new TaskProcessingStats(esStatConfig); + + this.aaiTransactionRateController = + new TransactionRateMonitor(numActiveInventoryWorkers, aaiStatConfig); + this.esTransactionRateController = + new TransactionRateMonitor(numElasticsearchWorkers, esStatConfig); + + this.aaiWorkOnHand = new AtomicInteger(0); + this.esWorkOnHand = new AtomicInteger(0); + + enabledStatFlags = EnumSet.allOf(StatFlag.class); + + this.synchronizerName = "Abstact Entity Synchronizer"; + + String txnID = NodeUtils.getRandomTxnId(); + MdcContext.initialize(txnID, "AbstractEntitySynchronizer", "", "Sync", ""); + + this.shouldSkipSync = false; + this.syncStartedTimeStampInMs = System.currentTimeMillis(); + this.syncDurationInMs = -1; + } + + public boolean shouldSkipSync() { + return shouldSkipSync; + } + + public void setShouldSkipSync(boolean shouldSkipSync) { + this.shouldSkipSync = shouldSkipSync; + } + + /** + * Inc active inventory work on hand counter. + */ + protected void incActiveInventoryWorkOnHandCounter() { + aaiWorkOnHand.incrementAndGet(); + } + + /** + * Dec active inventory work on hand counter. + */ + protected void decActiveInventoryWorkOnHandCounter() { + aaiWorkOnHand.decrementAndGet(); + } + + /** + * Inc elastic search work on hand counter. + */ + protected void incElasticSearchWorkOnHandCounter() { + esWorkOnHand.incrementAndGet(); + } + + /** + * Dec elastic search work on hand counter. + */ + protected void decElasticSearchWorkOnHandCounter() { + esWorkOnHand.decrementAndGet(); + } + + /** + * Shutdown executors. + */ + protected void shutdownExecutors() { + try { + + if (synchronizerExecutor != null) { + synchronizerExecutor.shutdown(); + } + + if (aaiExecutor != null) { + aaiExecutor.shutdown(); + } + + if (esExecutor != null) { + esExecutor.shutdown(); + } + + } catch (Exception exc) { + logger.error(AaiUiMsgs.ERROR_SHUTDOWN_EXECUTORS, exc ); + } + } + + /** + * Clear cache. + */ + public void clearCache() {} + + public ElasticSearchAdapter getElasticSearchAdapter() { + return elasticSearchAdapter; + } + + public void setElasticSearchAdapter(ElasticSearchAdapter elasticSearchAdapter) { + this.elasticSearchAdapter = elasticSearchAdapter; + } + + public ActiveInventoryAdapter getAaiAdapter() { + return aaiAdapter; + } + + public void setAaiAdapter(ActiveInventoryAdapter aaiAdapter) { + this.aaiAdapter = aaiAdapter; + } + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + + /** + * Gets the response length. + * + * @param txn the txn + * @return the response length + */ + private long getResponseLength(NetworkTransaction txn) { + + if (txn == null) { + return -1; + } + + OperationResult result = txn.getOperationResult(); + + if (result == null) { + return -1; + } + + if (result.getResult() != null) { + return result.getResult().length(); + } + + return -1; + } + + /** + * Update elastic search counters. + * + * @param method the method + * @param entityType the entity type + * @param or the or + */ + protected void updateElasticSearchCounters(HttpMethod method, String entityType, + OperationResult or) { + updateElasticSearchCounters(new NetworkTransaction(method, entityType, or)); + } + + /** + * Update elastic search counters. + * + * @param txn the txn + */ + protected void updateElasticSearchCounters(NetworkTransaction txn) { + + if (enabledStatFlags.contains(StatFlag.ES_REST_STATS)) { + esRestStats.updateCounters(txn); + } + + if (enabledStatFlags.contains(StatFlag.ES_ENTITY_STATS)) { + esEntityStats.updateCounters(txn); + } + + if (enabledStatFlags.contains(StatFlag.ES_TASK_PROCESSING_STATS)) { + + esTransactionRateController.trackResponseTime(txn.getOpTimeInMs()); + + esTaskProcessingStats + .updateTaskResponseStatsHistogram(txn.getOpTimeInMs()); + esTaskProcessingStats.updateTaskAgeStatsHistogram(txn.getTaskAgeInMs()); + + // don't know the cost of the lengh calc, we'll see if it causes a + // problem + + long responsePayloadSizeInBytes = getResponseLength(txn); + if (responsePayloadSizeInBytes >= 0) { + esTaskProcessingStats.updateResponseSizeInBytesHistogram(responsePayloadSizeInBytes); + } + + esTaskProcessingStats + .updateTransactionsPerSecondHistogram((long) esTransactionRateController.getCurrentTps()); + } + } + + /** + * Update active inventory counters. + * + * @param method the method + * @param entityType the entity type + * @param or the or + */ + protected void updateActiveInventoryCounters(HttpMethod method, String entityType, + OperationResult or) { + updateActiveInventoryCounters(new NetworkTransaction(method, entityType, or)); + } + + /** + * Update active inventory counters. + * + * @param txn the txn + */ + protected void updateActiveInventoryCounters(NetworkTransaction txn) { + + if (enabledStatFlags.contains(StatFlag.AAI_REST_STATS)) { + aaiRestStats.updateCounters(txn); + } + + if (enabledStatFlags.contains(StatFlag.AAI_ENTITY_STATS)) { + aaiEntityStats.updateCounters(txn); + } + + if (enabledStatFlags.contains(StatFlag.AAI_PROCESSING_EXCEPTION_STATS)) { + aaiProcessingExceptionStats.updateCounters(txn); + } + + if (enabledStatFlags.contains(StatFlag.AAI_TASK_PROCESSING_STATS)) { + aaiTransactionRateController + .trackResponseTime(txn.getOpTimeInMs()); + + aaiTaskProcessingStats + .updateTaskResponseStatsHistogram(txn.getOpTimeInMs()); + aaiTaskProcessingStats.updateTaskAgeStatsHistogram(txn.getTaskAgeInMs()); + + // don't know the cost of the lengh calc, we'll see if it causes a + // problem + + long responsePayloadSizeInBytes = getResponseLength(txn); + if (responsePayloadSizeInBytes >= 0) { + aaiTaskProcessingStats.updateResponseSizeInBytesHistogram(responsePayloadSizeInBytes); + } + + aaiTaskProcessingStats.updateTransactionsPerSecondHistogram( + (long) aaiTransactionRateController.getCurrentTps()); + } + } + + /** + * Reset counters. + */ + protected void resetCounters() { + aaiRestStats.reset(); + aaiEntityStats.reset(); + aaiProcessingExceptionStats.reset(); + + esRestStats.reset(); + esEntityStats.reset(); + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/ElasticSearchIndexCleaner.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/ElasticSearchIndexCleaner.java new file mode 100644 index 0000000..3255e39 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/ElasticSearchIndexCleaner.java @@ -0,0 +1,604 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import javax.ws.rs.core.MediaType; + +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ElasticSearchAdapter; +import org.onap.aai.sparky.logging.AaiUiMsgs; +import org.onap.aai.sparky.sync.config.ElasticSearchEndpointConfig; +import org.onap.aai.sparky.sync.config.ElasticSearchSchemaConfig; +import org.onap.aai.sparky.sync.entity.ObjectIdCollection; +import org.onap.aai.sparky.sync.entity.SearchableEntity; +import org.onap.aai.sparky.sync.enumeration.OperationState; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * The Class ElasticSearchIndexCleaner. + */ +public class ElasticSearchIndexCleaner implements IndexCleaner { + + private static final Logger LOG = + LoggerFactory.getInstance().getLogger(ElasticSearchIndexCleaner.class); + + private static final String BULK_OP_LINE_TEMPLATE = "%s\n"; + private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; + + private ObjectIdCollection before; + private ObjectIdCollection after; + + private ObjectMapper mapper; + private ElasticSearchAdapter esAdapter; + private ElasticSearchEndpointConfig endpointConfig; + private ElasticSearchSchemaConfig schemaConfig; + + /** + * Instantiates a new elastic search index cleaner. + * + * @param restDataProvider the rest data provider + * @param indexName the index name + * @param indexType the index type + * @param host the host + * @param port the port + * @param scrollContextTimeToLiveInMinutes the scroll context time to live in minutes + * @param numItemsToGetBulkRequest the num items to get bulk request + */ + public ElasticSearchIndexCleaner(ElasticSearchAdapter esAdapter, ElasticSearchEndpointConfig endpointConfig, ElasticSearchSchemaConfig schemaConfig) { + this.esAdapter = esAdapter; + this.before = null; + this.after = null; + this.endpointConfig = endpointConfig; + this.schemaConfig = schemaConfig; + this.mapper = new ObjectMapper(); + } + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.IndexCleaner#populatePreOperationCollection() + */ + @Override + public OperationState populatePreOperationCollection() { + + try { + before = retrieveAllDocumentIdentifiers(); + return OperationState.OK; + } catch (Exception exc) { + LOG.error(AaiUiMsgs.ES_PRE_SYNC_FAILURE, schemaConfig.getIndexName(), exc.getMessage()); + return OperationState.ERROR; + } + + } + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.IndexCleaner#populatePostOperationCollection() + */ + @Override + public OperationState populatePostOperationCollection() { + try { + after = retrieveAllDocumentIdentifiers(); + return OperationState.OK; + } catch (Exception exc) { + LOG.error(AaiUiMsgs.ES_PRE_SYNC_FAILURE, schemaConfig.getIndexName(), exc.getMessage()); + return OperationState.ERROR; + } + } + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.IndexCleaner#performCleanup() + */ + @Override + public OperationState performCleanup() { + // TODO Auto-generated method stub + LOG.info(AaiUiMsgs.ES_SYNC_CLEAN_UP, schemaConfig.getIndexName()); + + int sizeBefore = before.getSize(); + int sizeAfter = after.getSize(); + + LOG.info(AaiUiMsgs.ES_SYNC_CLEAN_UP_SIZE, String.valueOf(sizeBefore), + String.valueOf(sizeAfter)); + + /* + * If the processedImportIds size <= 0, then something has failed in the sync operation and we + * shouldn't do the selective delete right now. + */ + + if (sizeAfter > 0) { + + Collection<String> presyncIds = before.getImportedObjectIds(); + presyncIds.removeAll(after.getImportedObjectIds()); + + try { + LOG.info(AaiUiMsgs.ES_SYNC_SELECTIVE_DELETE, schemaConfig.getIndexName(), schemaConfig.getIndexDocType(), + String.valueOf(presyncIds.size())); + + ObjectIdCollection bulkIds = new ObjectIdCollection(); + + Iterator<String> it = presyncIds.iterator(); + int numItemsInBulkRequest = 0; + int numItemsRemainingToBeDeleted = presyncIds.size(); + + while (it.hasNext()) { + + bulkIds.addObjectId(it.next()); + numItemsInBulkRequest++; + + if (numItemsInBulkRequest >= endpointConfig.getScrollContextBatchRequestSize()) { + LOG.info(AaiUiMsgs.ES_BULK_DELETE, schemaConfig.getIndexName(), String.valueOf(bulkIds.getSize())); + bulkDelete(bulkIds.getImportedObjectIds()); + numItemsRemainingToBeDeleted -= numItemsInBulkRequest; + numItemsInBulkRequest = 0; + bulkIds.clear(); + } + } + + if (numItemsRemainingToBeDeleted > 0) { + LOG.info(AaiUiMsgs.ES_BULK_DELETE, schemaConfig.getIndexName(), String.valueOf(bulkIds.getSize())); + bulkDelete(bulkIds.getImportedObjectIds()); + } + + + } catch (Exception exc) { + LOG.error(AaiUiMsgs.ES_BULK_DELETE_ERROR, schemaConfig.getIndexName(), exc.getLocalizedMessage()); + + } + } + + return OperationState.OK; + } + + @Override + public String getIndexName() { + return schemaConfig.getIndexName(); + } + + /** + * Builds the initial scroll request payload. + * + * @param numItemsToGetPerRequest the num items to get per request + * @param fieldList the field list + * @return the string + * @throws JsonProcessingException the json processing exception + */ + protected String buildInitialScrollRequestPayload(int numItemsToGetPerRequest, + List<String> fieldList) throws JsonProcessingException { + + ObjectNode rootNode = mapper.createObjectNode(); + rootNode.put("size", numItemsToGetPerRequest); + + ArrayNode fields = mapper.createArrayNode(); + + for (String f : fieldList) { + fields.add(f); + } + + rootNode.set("fields", fields); + + ObjectNode queryNode = mapper.createObjectNode(); + queryNode.set("match_all", mapper.createObjectNode()); + + rootNode.set("query", queryNode); + + return mapper.writeValueAsString(rootNode); + + } + + /** + * Builds the subsequent scroll context request payload. + * + * @param scrollId the scroll id + * @param contextTimeToLiveInMinutes the context time to live in minutes + * @return the string + * @throws JsonProcessingException the json processing exception + */ + protected String buildSubsequentScrollContextRequestPayload(String scrollId, + int contextTimeToLiveInMinutes) throws JsonProcessingException { + + ObjectNode rootNode = mapper.createObjectNode(); + + rootNode.put("scroll", contextTimeToLiveInMinutes + "m"); + rootNode.put("scroll_id", scrollId); + + return mapper.writeValueAsString(rootNode); + + } + + /** + * Parses the elastic search result. + * + * @param jsonResult the json result + * @return the json node + * @throws JsonProcessingException the json processing exception + * @throws IOException Signals that an I/O exception has occurred. + */ + protected JsonNode parseElasticSearchResult(String jsonResult) + throws JsonProcessingException, IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readTree(jsonResult); + } + + /** + * Lookup index doc. + * + * @param ids the ids + * @param docs the docs + * @return the array list + */ + protected ArrayList<SearchableEntity> lookupIndexDoc(ArrayList<String> ids, + List<SearchableEntity> docs) { + ArrayList<SearchableEntity> objs = new ArrayList<SearchableEntity>(); + + if (ids != null && docs != null) { + for (SearchableEntity d : docs) { + if (ids.contains(d.getId())) { + objs.add(d); + } + } + } + + return objs; + } + + /** + * Builds the delete data object. + * + * @param index the index + * @param type the type + * @param id the id + * @return the object node + */ + protected ObjectNode buildDeleteDataObject(String index, String type, String id) { + + ObjectNode indexDocProperties = mapper.createObjectNode(); + + indexDocProperties.put("_index", index); + indexDocProperties.put("_type", type); + indexDocProperties.put("_id", id); + + ObjectNode rootNode = mapper.createObjectNode(); + rootNode.set("delete", indexDocProperties); + + return rootNode; + } + + /** + * This method might appear to be a little strange, and is simply an optimization to take an + * elipsed JsonNode key path and retrieve the node at the end of the path, if it exists. + * + * @param startNode the start node + * @param fieldPath the field path + * @return the node path + */ + protected JsonNode getNodePath(JsonNode startNode, String... fieldPath) { + + JsonNode jsonNode = null; + + for (String field : fieldPath) { + if (jsonNode == null) { + jsonNode = startNode.get(field); + } else { + jsonNode = jsonNode.get(field); + } + + /* + * This is our safety net in case any intermediate path returns a null + */ + + if (jsonNode == null) { + return null; + } + + } + + return jsonNode; + } + + /** + * Gets the full url. + * + * @param resourceUrl the resource url + * @return the full url + */ + private String getFullUrl(String resourceUrl) { + return String.format("http://%s:%s%s", endpointConfig.getEsIpAddress(), + endpointConfig.getEsServerPort(), resourceUrl); + } + + /** + * Retrieve all document identifiers. + * + * @return the object id collection + * @throws IOException Signals that an I/O exception has occurred. + */ + public ObjectIdCollection retrieveAllDocumentIdentifiers() throws IOException { + + ObjectIdCollection currentDocumentIds = new ObjectIdCollection(); + + long opStartTimeInMs = System.currentTimeMillis(); + + List<String> fields = new ArrayList<String>(); + fields.add("_id"); + // fields.add("entityType"); + + String scrollRequestPayload = + buildInitialScrollRequestPayload(endpointConfig.getScrollContextBatchRequestSize(), fields); + + final String fullUrlStr = getFullUrl("/" + schemaConfig.getIndexName()+ "/" + schemaConfig.getIndexDocType() + "/_search?scroll=" + + endpointConfig.getScrollContextTimeToLiveInMinutes() + "m"); + + OperationResult result = + esAdapter.doPost(fullUrlStr, scrollRequestPayload, MediaType.APPLICATION_JSON_TYPE); + + if (result.wasSuccessful()) { + + JsonNode rootNode = parseElasticSearchResult(result.getResult()); + + /* + * Check the result for success / failure, and enumerate all the index ids that resulted in + * success, and ignore the ones that failed or log them so we have a record of the failure. + */ + int totalRecordsAvailable = 0; + String scrollId = null; + int numRecordsFetched = 0; + + if (rootNode != null) { + + scrollId = getFieldValue(rootNode, "_scroll_id"); + final String tookStr = getFieldValue(rootNode, "took"); + int tookInMs = (tookStr == null) ? 0 : Integer.parseInt(tookStr); + boolean timedOut = Boolean.parseBoolean(getFieldValue(rootNode, "timed_out")); + + if (timedOut) { + LOG.error(AaiUiMsgs.COLLECT_TIME_WITH_ERROR, "all document Identifiers", + String.valueOf(tookInMs)); + } else { + LOG.info(AaiUiMsgs.COLLECT_TIME_WITH_SUCCESS, "all document Identifiers", + String.valueOf(tookInMs)); + } + + JsonNode hitsNode = rootNode.get("hits"); + totalRecordsAvailable = Integer.parseInt(hitsNode.get("total").asText()); + + LOG.info(AaiUiMsgs.COLLECT_TOTAL, "all document Identifiers", + String.valueOf(totalRecordsAvailable)); + + /* + * Collect all object ids + */ + + ArrayNode hitsArray = (ArrayNode) hitsNode.get("hits"); + + Iterator<JsonNode> nodeIterator = hitsArray.iterator(); + + String key = null; + String value = null; + JsonNode jsonNode = null; + + while (nodeIterator.hasNext()) { + + jsonNode = nodeIterator.next(); + + key = getFieldValue(jsonNode, "_id"); + + if (key != null) { + currentDocumentIds.addObjectId(key); + } + + } + + int totalRecordsRemainingToFetch = (totalRecordsAvailable - numRecordsFetched); + + int numRequiredAdditionalFetches = + (totalRecordsRemainingToFetch / endpointConfig.getScrollContextBatchRequestSize()); + + /* + * Do an additional fetch for the remaining items (if needed) + */ + + if (totalRecordsRemainingToFetch % endpointConfig.getScrollContextBatchRequestSize() != 0) { + numRequiredAdditionalFetches += 1; + } + + if (LOG.isDebugEnabled()) { + LOG.debug(AaiUiMsgs.SYNC_NUMBER_REQ_FETCHES, + String.valueOf(numRequiredAdditionalFetches)); + } + + + for (int x = 0; x < numRequiredAdditionalFetches; x++) { + + if (collectItemsFromScrollContext(scrollId, currentDocumentIds) != OperationState.OK) { + // abort the whole thing because now we can't reliably cleanup the orphans. + throw new IOException( + "Failed to collect pre-sync doc collection from index. Aborting operation"); + } + if (LOG.isDebugEnabled()) { + LOG.debug(AaiUiMsgs.SYNC_NUMBER_TOTAL_FETCHES, + String.valueOf(currentDocumentIds.getSize()), + String.valueOf(totalRecordsAvailable)); + } + + } + + } + + } else { + // scroll context get failed, nothing else to do + LOG.error(AaiUiMsgs.ERROR_GENERIC, result.toString()); + } + + LOG.info(AaiUiMsgs.COLLECT_TOTAL_TIME, "all document Identifiers", + String.valueOf((System.currentTimeMillis() - opStartTimeInMs))); + + return currentDocumentIds; + + } + + /** + * Collect items from scroll context. + * + * @param scrollId the scroll id + * @param objectIds the object ids + * @return the operation state + * @throws IOException Signals that an I/O exception has occurred. + */ + private OperationState collectItemsFromScrollContext(String scrollId, + ObjectIdCollection objectIds) throws IOException { + + String requestPayload = buildSubsequentScrollContextRequestPayload(scrollId, + endpointConfig.getScrollContextTimeToLiveInMinutes()); + + final String fullUrlStr = getFullUrl("/_search/scroll"); + + OperationResult opResult = + esAdapter.doPost(fullUrlStr, requestPayload, MediaType.APPLICATION_JSON_TYPE); + + if (opResult.getResultCode() >= 300) { + LOG.warn(AaiUiMsgs.ES_SCROLL_CONTEXT_ERROR, opResult.getResult()); + return OperationState.ERROR; + } + + JsonNode rootNode = parseElasticSearchResult(opResult.getResult()); + boolean timedOut = Boolean.parseBoolean(getFieldValue(rootNode, "timed_out")); + final String tookStr = getFieldValue(rootNode, "took"); + int tookInMs = (tookStr == null) ? 0 : Integer.parseInt(tookStr); + + JsonNode hitsNode = rootNode.get("hits"); + + /* + * Check the result for success / failure, and enumerate all the index ids that resulted in + * success, and ignore the ones that failed or log them so we have a record of the failure. + */ + + if (rootNode != null) { + + if (timedOut) { + LOG.info(AaiUiMsgs.COLLECT_TIME_WITH_ERROR, "Scroll Context", String.valueOf(tookInMs)); + } else { + LOG.info(AaiUiMsgs.COLLECT_TIME_WITH_SUCCESS, "Scroll Context", String.valueOf(tookInMs)); + } + + /* + * Collect all object ids + */ + + ArrayNode hitsArray = (ArrayNode) hitsNode.get("hits"); + String key = null; + String value = null; + JsonNode jsonNode = null; + + Iterator<JsonNode> nodeIterator = hitsArray.iterator(); + + while (nodeIterator.hasNext()) { + + jsonNode = nodeIterator.next(); + + key = getFieldValue(jsonNode, "_id"); + + if (key != null) { + objectIds.addObjectId(key); + + } + + } + } + + return OperationState.OK; + } + + /** + * Gets the field value. + * + * @param node the node + * @param fieldName the field name + * @return the field value + */ + protected String getFieldValue(JsonNode node, String fieldName) { + + JsonNode field = node.get(fieldName); + + if (field != null) { + return field.asText(); + } + + return null; + + } + + /** + * Bulk delete. + * + * @param docIds the doc ids + * @return the operation result + * @throws IOException Signals that an I/O exception has occurred. + */ + public OperationResult bulkDelete(Collection<String> docIds) throws IOException { + + if (docIds == null || docIds.size() == 0) { + LOG.info(AaiUiMsgs.ES_BULK_DELETE_SKIP); + return new OperationResult(500, + "Skipping bulkDelete(); operation because docs to delete list is empty"); + } + + LOG.info(AaiUiMsgs.ES_BULK_DELETE_START, String.valueOf(docIds.size())); + + StringBuilder sb = new StringBuilder(128); + + for (String id : docIds) { + sb.append(String.format(BULK_OP_LINE_TEMPLATE, + buildDeleteDataObject(schemaConfig.getIndexName(), schemaConfig.getIndexDocType(), id))); + } + + sb.append("\n"); + + final String fullUrlStr = getFullUrl("/_bulk"); + + return esAdapter.doPost(fullUrlStr, sb.toString(), MediaType.APPLICATION_FORM_URLENCODED_TYPE); + + } + + /* + + */ + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/ElasticSearchSchemaFactory.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/ElasticSearchSchemaFactory.java new file mode 100644 index 0000000..a2ff6cf --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/ElasticSearchSchemaFactory.java @@ -0,0 +1,97 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.io.IOException; + +import org.onap.aai.sparky.config.SparkyResourceLoader; +import org.onap.aai.sparky.dal.exception.ElasticSearchOperationException; +import org.onap.aai.sparky.sync.config.ElasticSearchSchemaConfig; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ElasticSearchSchemaFactory { + + private static final String SETTINGS = "settings"; + private static final String MAPPINGS = "mappings"; + + private static ObjectMapper mapper = new ObjectMapper(); + private SparkyResourceLoader resourceLoader; + + public String getIndexSchema(ElasticSearchSchemaConfig schemaConfig) + throws ElasticSearchOperationException { + + JsonNode esSettingsNode = null; + JsonNode esMappingsNodes = null; + + try { + + if (schemaConfig.getIndexSettingsFileName() != null) { + esSettingsNode = mapper + .readTree(resourceLoader.getResourceAsString(schemaConfig.getIndexSettingsFileName(),true)); + } + + if (schemaConfig.getIndexMappingsFileName() != null) { + esMappingsNodes = mapper + .readTree(resourceLoader.getResourceAsString(schemaConfig.getIndexMappingsFileName(),true)); + } + + } catch (IOException e1) { + + throw new ElasticSearchOperationException("Caught an exception building initial ES index. Error: " + e1.getMessage()); + } + + ObjectNode esConfig = null; + + ObjectNode mappings = + (ObjectNode) mapper.createObjectNode().set(schemaConfig.getIndexDocType(), esMappingsNodes); + + if (esSettingsNode == null) { + esConfig = (ObjectNode) mapper.createObjectNode().set(MAPPINGS, mappings); + } else { + esConfig = (ObjectNode) mapper.createObjectNode().set(SETTINGS, esSettingsNode); + esConfig.set(MAPPINGS, mappings); + } + + try { + return mapper.writeValueAsString(esConfig); + } catch (JsonProcessingException exc) { + throw new ElasticSearchOperationException("Error getting object node as string", exc); + } + + } + + public SparkyResourceLoader getResourceLoader() { + return resourceLoader; + } + + public void setResourceLoader(SparkyResourceLoader resourceLoader) { + this.resourceLoader = resourceLoader; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexCleaner.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexCleaner.java new file mode 100644 index 0000000..bc47fa8 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexCleaner.java @@ -0,0 +1,57 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import org.onap.aai.sparky.sync.enumeration.OperationState; + +/** + * The Interface IndexCleaner. + */ +public interface IndexCleaner { + + /** + * Populate pre operation collection. + * + * @return the operation state + */ + public OperationState populatePreOperationCollection(); + + /** + * Populate post operation collection. + * + * @return the operation state + */ + public OperationState populatePostOperationCollection(); + + /** + * Perform cleanup. + * + * @return the operation state + */ + public OperationState performCleanup(); + + public String getIndexName(); + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexIntegrityValidator.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexIntegrityValidator.java new file mode 100644 index 0000000..2c1bb16 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexIntegrityValidator.java @@ -0,0 +1,178 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import javax.ws.rs.core.MediaType; + +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ElasticSearchAdapter; +import org.onap.aai.sparky.logging.AaiUiMsgs; +import org.onap.aai.sparky.sync.config.ElasticSearchEndpointConfig; +import org.onap.aai.sparky.sync.config.ElasticSearchSchemaConfig; + +/** + * The Class IndexIntegrityValidator. + */ +public class IndexIntegrityValidator implements IndexValidator { + + private static final Logger LOG = + LoggerFactory.getInstance().getLogger(IndexIntegrityValidator.class); + + private ElasticSearchEndpointConfig endpointConfig; + private ElasticSearchSchemaConfig schemaConfig; + private String tableConfigJson; + + private final ElasticSearchAdapter esAdapter; + + /** + * Instantiates a new index integrity validator. + * + * @param restDataProvider the rest data provider + * @param indexName the index name + * @param indexType the index type + * @param host the host + * @param port the port + * @param tableConfigJson the table config json + */ + public IndexIntegrityValidator(ElasticSearchAdapter esAdapter, + ElasticSearchSchemaConfig esSchemaConfig, ElasticSearchEndpointConfig esEndpointConfig, + String tableConfigJson) { + + this.esAdapter = esAdapter; + this.schemaConfig = esSchemaConfig; + this.endpointConfig = esEndpointConfig; + this.tableConfigJson = tableConfigJson; + } + + public ElasticSearchEndpointConfig getEndpointConfig() { + return endpointConfig; + } + + public void setEndpointConfig(ElasticSearchEndpointConfig endpointConfig) { + this.endpointConfig = endpointConfig; + } + + public ElasticSearchSchemaConfig getSchemaConfig() { + return schemaConfig; + } + + public void setSchemaConfig(ElasticSearchSchemaConfig schemaConfig) { + this.schemaConfig = schemaConfig; + } + + public ElasticSearchAdapter getEsAdapter() { + return esAdapter; + } + + @Override + public String getIndexName() { + return schemaConfig.getIndexName(); + } + + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.IndexValidator#exists() + */ + @Override + public boolean exists() { + final String fullUrlStr = getFullUrl("/" + schemaConfig.getIndexName() + "/"); + OperationResult existsResult = esAdapter.doHead(fullUrlStr, MediaType.APPLICATION_JSON_TYPE); + + int rc = existsResult.getResultCode(); + + if (rc >= 200 && rc < 300) { + LOG.info(AaiUiMsgs.INDEX_EXISTS, schemaConfig.getIndexName()); + return true; + } else { + LOG.info(AaiUiMsgs.INDEX_NOT_EXIST, schemaConfig.getIndexName()); + return false; + } + } + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.IndexValidator#integrityValid() + */ + @Override + public boolean integrityValid() { + return true; + } + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.IndexValidator#createOrRepair() + */ + @Override + public void createOrRepair() { + + String message = + "IndexIntegrityValidator.createOrRepair() for indexName = " + schemaConfig.getIndexName(); + LOG.info(AaiUiMsgs.INFO_GENERIC, message); + + final String fullUrlStr = getFullUrl("/" + schemaConfig.getIndexName() + "/"); + OperationResult createResult = + esAdapter.doPut(fullUrlStr, tableConfigJson, MediaType.APPLICATION_JSON_TYPE); + + int rc = createResult.getResultCode(); + + if (rc >= 200 && rc < 300) { + LOG.info(AaiUiMsgs.INDEX_RECREATED, schemaConfig.getIndexName()); + } else if (rc == 400) { + LOG.info(AaiUiMsgs.INDEX_ALREADY_EXISTS, schemaConfig.getIndexName()); + } else { + LOG.warn(AaiUiMsgs.INDEX_INTEGRITY_CHECK_FAILED, schemaConfig.getIndexName(), + createResult.getResult()); + } + + } + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.IndexValidator#destroyIndex() + */ + @Override + public void destroyIndex() { + // we don't do this for now + } + + /** + * Gets the full url. + * + * @param resourceUrl the resource url + * @return the full url + */ + private String getFullUrl(String resourceUrl) { + return String.format("http://%s:%s%s", endpointConfig.getEsIpAddress(), + endpointConfig.getEsServerPort(), resourceUrl); + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexSynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexSynchronizer.java new file mode 100644 index 0000000..05c7a5e --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexSynchronizer.java @@ -0,0 +1,67 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import org.onap.aai.sparky.sync.enumeration.OperationState; +import org.onap.aai.sparky.sync.enumeration.SynchronizerState; + +/** + * The Interface IndexSynchronizer. + * + * @author davea. + */ +public interface IndexSynchronizer { + + /** + * Do sync. + * + * @return the operation state + */ + public OperationState doSync(); + + public SynchronizerState getState(); + + /** + * Gets the stat report. + * + * @param finalReport the final report + * @return the stat report + */ + public String getStatReport(boolean finalReport); + + /** + * Shutdown. + */ + public void shutdown(); + + public String getIndexName(); + + /** + * Clear cache. + */ + public void clearCache(); + + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexValidator.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexValidator.java new file mode 100644 index 0000000..05e7d91 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/IndexValidator.java @@ -0,0 +1,58 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +/** + * The Interface IndexValidator. + */ +public interface IndexValidator { + + /** + * Exists. + * + * @return true, if successful + */ + public boolean exists(); + + /** + * Integrity valid. + * + * @return true, if successful + */ + public boolean integrityValid(); + + /** + * Creates the or repair. + */ + public void createOrRepair(); + + /** + * Destroy index. + */ + public void destroyIndex(); + + public String getIndexName(); + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncController.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncController.java new file mode 100644 index 0000000..bdd009b --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncController.java @@ -0,0 +1,96 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.util.Calendar; +import java.util.Date; + +import org.onap.aai.sparky.sync.SyncControllerImpl.SyncActions; +import org.onap.aai.sparky.sync.enumeration.OperationState; +import org.onap.aai.sparky.sync.enumeration.SynchronizerState; + +public interface SyncController { + + String getControllerName(); + boolean isPeriodicSyncEnabled(); + boolean isRunOnceSyncEnabled(); + + /** + * Perform action. + * + * @param requestedAction the requested action + * @return + */ + OperationState performAction(SyncActions requestedAction); + + /** + * Register entity synchronizer. + * + * @param entitySynchronizer the entity synchronizer + */ + void registerEntitySynchronizer(IndexSynchronizer entitySynchronizer); + + /** + * Register index validator. + * + * @param indexValidator the index validator + */ + void registerIndexValidator(IndexValidator indexValidator); + + /** + * Register index cleaner. + * + * @param indexCleaner the index cleaner + */ + void registerIndexCleaner(IndexCleaner indexCleaner); + + /** + * Shutdown. + */ + void shutdown(); + + SynchronizerState getState(); + + long getDelayInMs(); + + void setDelayInMs(long delayInMs); + + long getSyncFrequencyInMs(); + + void setSyncFrequencyInMs(long syncFrequencyInMs); + + Date getSyncStartTime(); + + void setSyncStartTime(Date syncStartTime); + + Date getLastExecutionDate(); + + void setLastExecutionDate(Date lastExecutionDate); + + Calendar getCreationTime(); + + String getNextSyncTime(); + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerImpl.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerImpl.java new file mode 100644 index 0000000..0e0b193 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerImpl.java @@ -0,0 +1,682 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import static java.util.concurrent.CompletableFuture.supplyAsync; + +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.TimeZone; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.sparky.logging.AaiUiMsgs; +import org.onap.aai.sparky.sync.config.SyncControllerConfig; +import org.onap.aai.sparky.sync.enumeration.OperationState; +import org.onap.aai.sparky.sync.enumeration.SynchronizerState; +import org.onap.aai.sparky.util.NodeUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * The Class SyncController. + * + * @author davea. + */ +public class SyncControllerImpl implements SyncController { + private static final Logger LOG = LoggerFactory.getInstance().getLogger(SyncControllerImpl.class); + + /** + * The Enum InternalState. + */ + private enum InternalState { + IDLE, PRE_SYNC, SYNC_OPERATION, SELECTIVE_DELETE, ABORTING_SYNC, REPAIRING_INDEX, POST_SYNC, + TEST_INDEX_INTEGRITY, GENERATE_FINAL_REPORT + } + + /** + * The Enum SyncActions. + */ + public enum SyncActions { + SYNCHRONIZE, REPAIR_INDEX, INDEX_INTEGRITY_VALIDATION_COMPLETE, PRE_SYNC_COMPLETE, + SYNC_COMPLETE, SYNC_ABORTED, SYNC_FAILURE, POST_SYNC_COMPLETE, PURGE_COMPLETE, REPORT_COMPLETE + } + + private Collection<IndexSynchronizer> registeredSynchronizers; + private Collection<IndexValidator> registeredIndexValidators; + private Collection<IndexCleaner> registeredIndexCleaners; + private InternalState currentInternalState; + private ExecutorService syncControllerExecutor; + private ExecutorService statReporterExecutor; + + private long delayInMs; + private long syncFrequencyInMs; + private Date syncStartTime; + + private Date lastExecutionDate; + private AtomicInteger runCount; + private Semaphore performingActionGate; + private Calendar creationTime; + + private String syncStartTimeWithTimeZone; + private String controllerName; + + protected SyncControllerConfig syncControllerConfig; + + + + + /** + * Instantiates a new sync controller. + * + * @param name the name + * @throws Exception the exception + */ + public SyncControllerImpl(SyncControllerConfig syncControllerConfig) throws Exception { + this(syncControllerConfig,null); + } + + public SyncControllerImpl(SyncControllerConfig syncControllerConfig, String targetEntityType) + throws Exception { + + this.syncControllerConfig = syncControllerConfig; + + this.delayInMs = 0L; + this.syncFrequencyInMs = 86400000L; + this.syncStartTime = null; + this.lastExecutionDate = null; + this.runCount = new AtomicInteger(0); + this.performingActionGate = new Semaphore(1); + registeredSynchronizers = new LinkedHashSet<IndexSynchronizer>(); + registeredIndexValidators = new LinkedHashSet<IndexValidator>(); + registeredIndexCleaners = new LinkedHashSet<IndexCleaner>(); + + String controllerName = syncControllerConfig.getControllerName(); + + if (targetEntityType != null) { + controllerName += " (" + targetEntityType + ")"; + } + + this.controllerName = controllerName; + + this.syncControllerExecutor = NodeUtils.createNamedExecutor("SyncController-" + controllerName, + syncControllerConfig.getNumSyncControllerWorkers(), LOG); + this.statReporterExecutor = + NodeUtils.createNamedExecutor("StatReporter-" + controllerName, 1, LOG); + + this.currentInternalState = InternalState.IDLE; + + this.creationTime = + Calendar.getInstance(TimeZone.getTimeZone(syncControllerConfig.getTimeZoneOfSyncStartTimeStamp())); + + } + + + + + + + /** + * Change internal state. + * + * @param newState the new state + * @param causedByAction the caused by action + */ + private void changeInternalState(InternalState newState, SyncActions causedByAction) { + LOG.info(AaiUiMsgs.SYNC_INTERNAL_STATE_CHANGED, controllerName, + currentInternalState.toString(), newState.toString(), causedByAction.toString()); + + this.currentInternalState = newState; + + performStateAction(); + } + + + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#getDelayInMs() + */ + @Override + public long getDelayInMs() { + return delayInMs; + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#setDelayInMs(long) + */ + @Override + public void setDelayInMs(long delayInMs) { + this.delayInMs = delayInMs; + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#getSyncFrequencyInMs() + */ + @Override + public long getSyncFrequencyInMs() { + return syncFrequencyInMs; + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#setSyncFrequencyInMs(long) + */ + @Override + public void setSyncFrequencyInMs(long syncFrequencyInMs) { + this.syncFrequencyInMs = syncFrequencyInMs; + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#getSyncStartTime() + */ + @Override + public Date getSyncStartTime() { + return syncStartTime; + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#setSyncStartTime(java.util.Date) + */ + @Override + public void setSyncStartTime(Date syncStartTime) { + this.syncStartTime = syncStartTime; + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#getLastExecutionDate() + */ + @Override + public Date getLastExecutionDate() { + return lastExecutionDate; + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncController2#setLastExecutionDate(java.util.Date) + */ + @Override + public void setLastExecutionDate(Date lastExecutionDate) { + this.lastExecutionDate = lastExecutionDate; + } + + @Override + public String getControllerName() { + return controllerName; + } + + + + + @Override + public OperationState performAction(SyncActions requestedAction) { + + if (currentInternalState == InternalState.IDLE) { + + try { + + /* + * non-blocking semaphore acquire used to guarantee only 1 execution of the synchronization + * at a time. + */ + + switch (requestedAction) { + case SYNCHRONIZE: + + if (performingActionGate.tryAcquire()) { + try { + + long opStartTime = System.currentTimeMillis(); + + LOG.info(AaiUiMsgs.INFO_GENERIC, + getControllerName() + " started synchronization at " + + SynchronizerConstants.SIMPLE_DATE_FORMAT.format(opStartTime).replaceAll( + SynchronizerConstants.TIME_STD, SynchronizerConstants.TIME_CONFIG_STD)); + + runCount.incrementAndGet(); + + changeInternalState(InternalState.TEST_INDEX_INTEGRITY, requestedAction); + + long opEndTime = System.currentTimeMillis(); + + long opTime = (opEndTime - opStartTime); + + String durationMessage = + String.format(getControllerName() + " synchronization took '%d' ms.", opTime); + + LOG.info(AaiUiMsgs.SYNC_DURATION, durationMessage); + + if (syncControllerConfig.isPeriodicSyncEnabled()) { + + LOG.info(AaiUiMsgs.INFO_GENERIC, + getControllerName() + " next sync to begin at " + getNextSyncTime()); + + TimeZone tz = TimeZone.getTimeZone(syncControllerConfig.getTimeZoneOfSyncStartTimeStamp()); + + if (opTime > this.getSyncFrequencyInMs()) { + + String durationWasLongerMessage = String.format( + getControllerName() + " synchronization took '%d' ms which is larger than" + + " synchronization interval of '%d' ms.", + opTime, this.getSyncFrequencyInMs()); + + LOG.info(AaiUiMsgs.SYNC_DURATION, durationWasLongerMessage); + } + } + + } catch (Exception syncException) { + String message = "An error occurred while performing action = " + requestedAction + + ". Error = " + syncException.getMessage(); + LOG.error(AaiUiMsgs.ERROR_GENERIC, message); + } finally { + performingActionGate.release(); + } + } else { + return OperationState.IGNORED_SYNC_NOT_IDLE; + } + + break; + + default: + break; + } + + return OperationState.OK; + + } catch (Exception exc) { + String message = "An error occurred while performing action = " + requestedAction + + ". Error = " + exc.getMessage(); + LOG.error(AaiUiMsgs.ERROR_GENERIC, message); + return OperationState.ERROR; + } finally { + + } + } else { + LOG.error(AaiUiMsgs.SYNC_NOT_VALID_STATE_DURING_REQUEST, currentInternalState.toString()); + return OperationState.IGNORED_SYNC_NOT_IDLE; + } + } + + /** + * Perform state action. + */ + private void performStateAction() { + + try { + switch (currentInternalState) { + + case TEST_INDEX_INTEGRITY: + performIndexIntegrityValidation(); + break; + + case PRE_SYNC: + performPreSyncCleanupCollection(); + break; + + case SYNC_OPERATION: + performSynchronization(); + break; + + case POST_SYNC: + performIndexSyncPostCollection(); + changeInternalState(InternalState.SELECTIVE_DELETE, SyncActions.POST_SYNC_COMPLETE); + break; + + case SELECTIVE_DELETE: + performIndexCleanup(); + changeInternalState(InternalState.GENERATE_FINAL_REPORT, SyncActions.PURGE_COMPLETE); + break; + + case GENERATE_FINAL_REPORT: + + dumpStatReport(true); + clearCaches(); + changeInternalState(InternalState.IDLE, SyncActions.REPORT_COMPLETE); + break; + + case ABORTING_SYNC: + performSyncAbort(); + break; + + default: + break; + } + } catch (Exception exc) { + /* + * Perhaps we should abort the sync on an exception + */ + String message = "Caught an error which performing action. Error = " + exc.getMessage(); + LOG.error(AaiUiMsgs.ERROR_GENERIC, message); + } + } + + @Override + public void registerEntitySynchronizer(IndexSynchronizer entitySynchronizer) { + + String indexName = entitySynchronizer.getIndexName(); + + if (indexName != null) { + registeredSynchronizers.add(entitySynchronizer); + } else { + String message = "Failed to register entity synchronizer because index name is null"; + LOG.error(AaiUiMsgs.FAILED_TO_REGISTER_DUE_TO_NULL, message); + } + + } + + @Override + public void registerIndexValidator(IndexValidator indexValidator) { + + String indexName = indexValidator.getIndexName(); + + if (indexName != null) { + registeredIndexValidators.add(indexValidator); + } else { + String message = "Failed to register index validator because index name is null"; + LOG.error(AaiUiMsgs.FAILED_TO_REGISTER_DUE_TO_NULL, message); + } + + } + + @Override + public void registerIndexCleaner(IndexCleaner indexCleaner) { + + String indexName = indexCleaner.getIndexName(); + + if (indexName != null) { + registeredIndexCleaners.add(indexCleaner); + } else { + String message = "Failed to register index cleaner because index name is null"; + LOG.error(AaiUiMsgs.FAILED_TO_REGISTER_DUE_TO_NULL, message); + } + } + + /* + * State machine should drive our flow dosync just dispatches an action and the state machine + * determines what is in play and what is next + */ + + /** + * Dump stat report. + * + * @param showFinalReport the show final report + */ + private void dumpStatReport(boolean showFinalReport) { + + for (IndexSynchronizer synchronizer : registeredSynchronizers) { + + String statReport = synchronizer.getStatReport(showFinalReport); + + if (statReport != null) { + LOG.info(AaiUiMsgs.INFO_GENERIC, statReport); + } + } + } + + /** + * Clear caches. + */ + private void clearCaches() { + + /* + * Any entity caches that were built as part of the sync operation should be cleared to save + * memory. The original intent of the caching was to provide a short-lived cache to satisfy + * entity requests from multiple synchronizers yet minimizing interactions with the AAI. + */ + + for (IndexSynchronizer synchronizer : registeredSynchronizers) { + synchronizer.clearCache(); + } + } + + /** + * Perform pre sync cleanup collection. + */ + private void performPreSyncCleanupCollection() { + + /* + * ask the index cleaners to collect the their pre-sync object id collections + */ + + for (IndexCleaner cleaner : registeredIndexCleaners) { + cleaner.populatePreOperationCollection(); + } + + changeInternalState(InternalState.SYNC_OPERATION, SyncActions.PRE_SYNC_COMPLETE); + + } + + /** + * Perform index sync post collection. + */ + private void performIndexSyncPostCollection() { + + /* + * ask the entity purgers to collect the their pre-sync object id collections + */ + + for (IndexCleaner cleaner : registeredIndexCleaners) { + cleaner.populatePostOperationCollection(); + } + + } + + /** + * Perform index cleanup. + */ + private void performIndexCleanup() { + + /* + * ask the entity purgers to collect the their pre-sync object id collections + */ + + for (IndexCleaner cleaner : registeredIndexCleaners) { + cleaner.performCleanup(); + } + + } + + /** + * Perform sync abort. + */ + private void performSyncAbort() { + changeInternalState(InternalState.IDLE, SyncActions.SYNC_ABORTED); + } + + /** + * Perform index integrity validation. + */ + private void performIndexIntegrityValidation() { + + /* + * loop through registered index validators and test and fix, if needed + */ + + for (IndexValidator validator : registeredIndexValidators) { + try { + if (!validator.exists()) { + validator.createOrRepair(); + } + } catch (Exception exc) { + String message = "Index validator caused an error = " + exc.getMessage(); + LOG.error(AaiUiMsgs.ERROR_GENERIC, message); + } + } + + changeInternalState(InternalState.PRE_SYNC, SyncActions.INDEX_INTEGRITY_VALIDATION_COMPLETE); + + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncControllerInterface#shutdown() + */ + @Override + public void shutdown() { + + this.syncControllerExecutor.shutdown(); + for (IndexSynchronizer synchronizer : registeredSynchronizers) { + + try { + synchronizer.shutdown(); + } catch (Exception exc) { + LOG.error(AaiUiMsgs.ERROR_GENERIC, + "Synchronizer shutdown caused an error = " + exc.getMessage()); + } + + } + this.statReporterExecutor.shutdown(); + } + + /* + * Need some kind of task running that responds to a transient boolean to kill it or we just stop + * the executor that it is in? + */ + + + + /** + * Perform synchronization. + */ + private void performSynchronization() { + + /* + * Get all the synchronizers running in parallel + */ + + for (IndexSynchronizer synchronizer : registeredSynchronizers) { + supplyAsync(new Supplier<Void>() { + + @Override + public Void get() { + + synchronizer.doSync(); + return null; + } + + }, this.syncControllerExecutor).whenComplete((result, error) -> { + + /* + * We don't bother checking the result, because it will always be null as the doSync() is + * non-blocking. + */ + + if (error != null) { + LOG.error(AaiUiMsgs.ERROR_GENERIC, + "doSync operation failed with an error = " + error.getMessage()); + } + }); + } + + boolean allDone = false; + long nextReportTimeStampInMs = System.currentTimeMillis() + 30000L; + boolean dumpPeriodicStatReport = false; + + while (!allDone) { + int totalFinished = 0; + + for (IndexSynchronizer synchronizer : registeredSynchronizers) { + if (dumpPeriodicStatReport) { + if (synchronizer.getState() == SynchronizerState.PERFORMING_SYNCHRONIZATION) { + String statReport = synchronizer.getStatReport(false); + + if (statReport != null) { + LOG.info(AaiUiMsgs.INFO_GENERIC, statReport); + } + } + } + + if (synchronizer.getState() == SynchronizerState.IDLE + || synchronizer.getState() == SynchronizerState.ABORTED) { + totalFinished++; + } + } + + if ( System.currentTimeMillis() > nextReportTimeStampInMs) { + dumpPeriodicStatReport = true; + nextReportTimeStampInMs = System.currentTimeMillis() + 30000L; + } else { + dumpPeriodicStatReport = false; + } + + allDone = (totalFinished == registeredSynchronizers.size()); + + try { + Thread.sleep(250); + } catch (InterruptedException exc) { + LOG.error(AaiUiMsgs.ERROR_GENERIC, + "An error occurred while waiting for sync to complete. Error = " + exc.getMessage()); + } + + } + + changeInternalState(InternalState.POST_SYNC, SyncActions.SYNC_COMPLETE); + + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.SyncControllerInterface#getState() + */ + @Override + public SynchronizerState getState() { + + switch (currentInternalState) { + + case IDLE: { + return SynchronizerState.IDLE; + } + + default: { + return SynchronizerState.PERFORMING_SYNCHRONIZATION; + + } + } + + } + + @Override + public Calendar getCreationTime() { + return creationTime; + } + + @Override + public String getNextSyncTime() { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean isPeriodicSyncEnabled() { + return syncControllerConfig.isPeriodicSyncEnabled(); + } + + @Override + public boolean isRunOnceSyncEnabled() { + return syncControllerConfig.isRunOnceSyncEnabled(); + } + +}
\ No newline at end of file diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerRegistrar.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerRegistrar.java new file mode 100644 index 0000000..f20eceb --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerRegistrar.java @@ -0,0 +1,29 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +public interface SyncControllerRegistrar { + public void registerController(); +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerRegistry.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerRegistry.java new file mode 100644 index 0000000..d4013d3 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerRegistry.java @@ -0,0 +1,50 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.util.ArrayList; +import java.util.List; + +public class SyncControllerRegistry { + + private List<SyncController> controllers; + + public SyncControllerRegistry() { + controllers = new ArrayList<SyncController>(); + } + + public void registerSyncController(SyncController controller) { + controllers.add(controller); + } + + public List<SyncController> getControllers() { + return controllers; + } + + public void setControllers(List<SyncController> controllers) { + this.controllers = controllers; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerService.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerService.java new file mode 100644 index 0000000..07653b7 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SyncControllerService.java @@ -0,0 +1,222 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.lang.Thread.UncaughtExceptionHandler; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.sparky.logging.AaiUiMsgs; +import org.onap.aai.sparky.sync.SyncControllerImpl.SyncActions; +import org.onap.aai.sparky.sync.enumeration.OperationState; +import org.onap.aai.sparky.sync.enumeration.SynchronizerState; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ApplicationContextEvent; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +public class SyncControllerService implements ApplicationListener<ApplicationContextEvent> { + + private SyncControllerRegistry syncControllerRegistry; + private ExecutorService runonceSyncExecutor; + private ScheduledExecutorService periodicSyncExecutor; + private boolean syncStarted; + + private static final Logger LOG = + LoggerFactory.getInstance().getLogger(SyncControllerService.class); + + private class SyncControllerTask implements Runnable { + + private SyncController controller; + + public SyncControllerTask(SyncController controller) { + this.controller = controller; + } + + @Override + public void run() { + + try { + + if (controller.getState() == SynchronizerState.IDLE) { + + /* + * This is a blocking-call, but would be nicer if it was async internally within the + * controller but at the moment, that's not the way it works. + */ + + if (controller.performAction(SyncActions.SYNCHRONIZE) != OperationState.OK) { + + LOG.info(AaiUiMsgs.INFO_GENERIC, + controller.getControllerName() + " is not idle, sync attempt has been skipped."); + } + } else { + + LOG.info(AaiUiMsgs.INFO_GENERIC, + controller.getControllerName() + " is not idle, sync attempt has been skipped."); + } + + } catch (Exception exception) { + LOG.error(AaiUiMsgs.ERROR_GENERIC, + "Error while attempting synchronization. Error = " + exception.getMessage()); + } + + } + + } + + public SyncControllerService(SyncControllerRegistry syncControllerRegistry, int numRunOnceWorkers, + int numPeriodicWorkers) { + this.syncControllerRegistry = syncControllerRegistry; + this.syncStarted = false; + + UncaughtExceptionHandler uncaughtExceptionHandler = new Thread.UncaughtExceptionHandler() { + + @Override + public void uncaughtException(Thread thread, Throwable exc) { + LOG.error(AaiUiMsgs.ERROR_GENERIC, thread.getName() + ": " + exc); + } + }; + + runonceSyncExecutor = Executors.newFixedThreadPool(numRunOnceWorkers, + new ThreadFactoryBuilder().setNameFormat("RunonceSyncWorker-%d") + .setUncaughtExceptionHandler(uncaughtExceptionHandler).build()); + + + periodicSyncExecutor = Executors.newScheduledThreadPool(numPeriodicWorkers, + new ThreadFactoryBuilder().setNameFormat("PeriodicSyncWorker-%d") + .setUncaughtExceptionHandler(uncaughtExceptionHandler).build()); + + } + + public SyncControllerRegistry getSyncControllerRegistry() { + return syncControllerRegistry; + } + + public void startSync() { + + long syncInitialDelayInMs = 0; + + for (SyncController controller : syncControllerRegistry.getControllers()) { + + syncInitialDelayInMs = controller.getDelayInMs(); + + if (!controller.isPeriodicSyncEnabled()) { + + if (controller.isRunOnceSyncEnabled()) { + LOG.info(AaiUiMsgs.INFO_GENERIC, controller.getControllerName() + " is enabled."); + runonceSyncExecutor.submit(new SyncControllerTask(controller)); + } else { + LOG.info(AaiUiMsgs.INFO_GENERIC, controller.getControllerName() + " is disabled."); + } + + } else { + + /** + * Do both. We'll take one instance of the SyncController and wrap the object instance into + * two SyncControllerTasks. The responsibility for preventing a conflicting sync should live + * in the SyncController instance. If a sync is underway when the periodic sync kicks in, + * then it will be ignored by the SyncController which is already underway. + * + * The SyncController instance itself would then also be stateful such that it would know + * the last time it ran, and the next time it is supposed to run, the number times a sync + * has executed, etc. + */ + + if (controller.isRunOnceSyncEnabled()) { + LOG.info(AaiUiMsgs.INFO_GENERIC, + controller.getControllerName() + " run-once sync is enabled."); + runonceSyncExecutor.submit(new SyncControllerTask(controller)); + } else { + LOG.info(AaiUiMsgs.INFO_GENERIC, + controller.getControllerName() + " run-once sync is disabled."); + } + + /* + * The controller knows it's configuredfrequency and we can just ask it to tell us what the + * delay and frequency needs to be, rather than trying to calculate the configured frequency + * per controller which "could" be different for each controller. + */ + + if (controller.isPeriodicSyncEnabled()) { + + LOG.info(AaiUiMsgs.INFO_GENERIC, + controller.getControllerName() + " periodic sync is enabled and scheduled to start @ " + + controller.getNextSyncTime()); + + periodicSyncExecutor.scheduleAtFixedRate(new SyncControllerTask(controller), + controller.getDelayInMs(), controller.getSyncFrequencyInMs(), TimeUnit.MILLISECONDS); + + } else { + + LOG.info(AaiUiMsgs.INFO_GENERIC, + controller.getControllerName() + " periodic sync is disabled."); + + } + + } + + } + + } + + public void shutdown() { + + if (runonceSyncExecutor != null) { + runonceSyncExecutor.shutdown(); + } + + if (periodicSyncExecutor != null) { + periodicSyncExecutor.shutdown(); + } + + if (syncControllerRegistry != null) { + for (SyncController controller : syncControllerRegistry.getControllers()) { + controller.shutdown(); + } + } + + } + + @Override + public synchronized void onApplicationEvent(ApplicationContextEvent arg0) { + + /* + * Start sync service processing when spring-context-initialization has finished + */ + + if (!syncStarted) { + syncStarted = true; + startSync(); + } + + } + + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SynchronizerConstants.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SynchronizerConstants.java new file mode 100644 index 0000000..9201f07 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/SynchronizerConstants.java @@ -0,0 +1,65 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * The Class SynchronizerConstants. + */ +public final class SynchronizerConstants { + // Error values for invalid user input + public static final int DEFAULT_CONFIG_ERROR_INT_VALUE = Integer.MAX_VALUE; + public static final Date DEFAULT_CONFIG_ERROR_DATE_VALUE = new Date(Long.MAX_VALUE); + public static final SimpleDateFormat SIMPLE_DATE_FORMAT = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + + public static final String DEPTH_MODIFIER = "?depth=0"; + public static final String DEPTH_ALL_MODIFIER = "?depth=all"; + public static final String DEPTH_AND_NODES_ONLY_MODIFIER = "?depth=0&nodes-only"; + public static final String NODES_ONLY_MODIFIER = "?nodes-only"; + + // constants for scheduling synchronizer + public static final int COMPONENTS_IN_TIMESTAMP = 2; + public static final String DEFAULT_INITIAL_DELAY_IN_MS = "0"; + public static final String DEFAULT_TASK_FREQUENCY_IN_DAY = "0"; + public static final String DEFAULT_START_TIMESTAMP = "05:00:00 UTC"; + public static final long DELAY_NO_STARTUP_SYNC_IN_MS = 0; + public static final long DELAY_NO_PERIODIC_SYNC_IN_MS = 0; + public static final int IDX_TIME_IN_TIMESTAMP = 0; + public static final int IDX_TIMEZONE_IN_TIMESTAMP = 1; + public static final long MILLISEC_IN_A_MIN = 60000; + public static final long MILLISEC_IN_A_DAY = 24 * 60 * 60 * 1000; + public static final String TIME_STD = "GMT"; + public static final String TIME_CONFIG_STD = "UTC"; + public static final String TIMESTAMP24HOURS_PATTERN = + "([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] UTC[+|-][0-5][0-9]:[0-5][0-9]"; + + /** + * Instantiates a new synchronizer constants. + */ + private SynchronizerConstants() {} +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/TaskProcessingStats.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/TaskProcessingStats.java new file mode 100644 index 0000000..48625c0 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/TaskProcessingStats.java @@ -0,0 +1,135 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import org.onap.aai.sparky.analytics.AbstractStatistics; +import org.onap.aai.sparky.sync.config.NetworkStatisticsConfig; + +/** + * The Class TaskProcessingStats. + */ +public class TaskProcessingStats extends AbstractStatistics { + + private static String TASK_AGE_STATS = "taskAgeStats"; + private static String TASK_RESPONSE_STATS = "taskResponseStats"; + private static String RESPONSE_SIZE_IN_BYTES = "taskResponseSizeInBytes"; + // private static String QUEUE_ITEM_LENGTH = "queueItemLength"; + private static String TPS = "transactionsPerSecond"; + + /** + * Instantiates a new task processing stats. + * + * @param config the config + */ + public TaskProcessingStats(NetworkStatisticsConfig config) { + + addHistogram(TASK_AGE_STATS, config.getTaskAgeHistogramLabel(), + config.getTaskAgeHistogramMaxYAxis(), config.getTaskAgeHistogramNumBins(), + config.getTaskAgeHistogramNumDecimalPoints()); + + addHistogram(TASK_RESPONSE_STATS, config.getResponseTimeHistogramLabel(), + config.getResponseTimeHistogramMaxYAxis(), config.getResponseTimeHistogramNumBins(), + config.getResponseTimeHistogramNumDecimalPoints()); + + addHistogram(RESPONSE_SIZE_IN_BYTES, config.getBytesHistogramLabel(), + config.getBytesHistogramMaxYAxis(), config.getBytesHistogramNumBins(), + config.getBytesHistogramNumDecimalPoints()); + + /* + * addHistogram(QUEUE_ITEM_LENGTH, config.getQueueLengthHistogramLabel(), + * config.getQueueLengthHistogramMaxYAxis(), config.getQueueLengthHistogramNumBins(), + * config.getQueueLengthHistogramNumDecimalPoints()); + */ + + addHistogram(TPS, config.getTpsHistogramLabel(), config.getTpsHistogramMaxYAxis(), + config.getTpsHistogramNumBins(), config.getTpsHistogramNumDecimalPoints()); + + } + + /* + * public void updateQueueItemLengthHistogram(long value) { updateHistogram(QUEUE_ITEM_LENGTH, + * value); } + */ + + /** + * Update task age stats histogram. + * + * @param value the value + */ + public void updateTaskAgeStatsHistogram(long value) { + updateHistogram(TASK_AGE_STATS, value); + } + + /** + * Update task response stats histogram. + * + * @param value the value + */ + public void updateTaskResponseStatsHistogram(long value) { + updateHistogram(TASK_RESPONSE_STATS, value); + } + + /** + * Update response size in bytes histogram. + * + * @param value the value + */ + public void updateResponseSizeInBytesHistogram(long value) { + updateHistogram(RESPONSE_SIZE_IN_BYTES, value); + } + + /** + * Update transactions per second histogram. + * + * @param value the value + */ + public void updateTransactionsPerSecondHistogram(long value) { + updateHistogram(TPS, value); + } + + /** + * Gets the statistics report. + * + * @param verboseEnabled the verbose enabled + * @param indentPadding the indent padding + * @return the statistics report + */ + public String getStatisticsReport(boolean verboseEnabled, String indentPadding) { + + StringBuilder sb = new StringBuilder(); + + sb.append("\n").append(getHistogramStats(TASK_AGE_STATS, verboseEnabled, indentPadding)); + // sb.append("\n").append(getHistogramStats(QUEUE_ITEM_LENGTH, verboseEnabled, indentPadding)); + sb.append("\n").append(getHistogramStats(TASK_RESPONSE_STATS, verboseEnabled, indentPadding)); + sb.append("\n") + .append(getHistogramStats(RESPONSE_SIZE_IN_BYTES, verboseEnabled, indentPadding)); + sb.append("\n").append(getHistogramStats(TPS, verboseEnabled, indentPadding)); + + return sb.toString(); + + } + + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/TransactionRateMonitor.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/TransactionRateMonitor.java new file mode 100644 index 0000000..e639eff --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/TransactionRateMonitor.java @@ -0,0 +1,75 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.onap.aai.sparky.analytics.AveragingRingBuffer; +import org.onap.aai.sparky.sync.config.NetworkStatisticsConfig; + +public class TransactionRateMonitor { + + private AtomicInteger numTransactions; + private AveragingRingBuffer responseTimeTracker; + private long startTimeInMs; + + /** + * Instantiates a new transaction rate controller. + * + * @param config the config + */ + public TransactionRateMonitor(int numWorkerThreads, NetworkStatisticsConfig config) { + + this.responseTimeTracker = new AveragingRingBuffer( + config.getNumSamplesPerThreadForRunningAverage() * numWorkerThreads); + this.startTimeInMs = System.currentTimeMillis(); + this.numTransactions = new AtomicInteger(0); + } + + /** + * Track response time. + * + * @param responseTimeInMs the response time in ms + */ + public void trackResponseTime(long responseTimeInMs) { + this.numTransactions.incrementAndGet(); + responseTimeTracker.addSample(responseTimeInMs); + } + + public long getAvg() { + return responseTimeTracker.getAvg(); + } + + public double getCurrentTps() { + if (numTransactions.get() > 0) { + double timeDelta = System.currentTimeMillis() - startTimeInMs; + double numTxns = numTransactions.get(); + return (numTxns / timeDelta) * 1000.0; + } + + return 0.0; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/ElasticSearchEndpointConfig.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/ElasticSearchEndpointConfig.java new file mode 100644 index 0000000..0197b9b --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/ElasticSearchEndpointConfig.java @@ -0,0 +1,72 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.config; + +public class ElasticSearchEndpointConfig { + + private String esIpAddress; + private String esServerPort; + private int scrollContextTimeToLiveInMinutes; + private int scrollContextBatchRequestSize; + + public ElasticSearchEndpointConfig() { + + } + + public String getEsIpAddress() { + return esIpAddress; + } + + public void setEsIpAddress(String esIpAddress) { + this.esIpAddress = esIpAddress; + } + + public String getEsServerPort() { + return esServerPort; + } + + public void setEsServerPort(String esServerPort) { + this.esServerPort = esServerPort; + } + + public int getScrollContextTimeToLiveInMinutes() { + return scrollContextTimeToLiveInMinutes; + } + + public void setScrollContextTimeToLiveInMinutes(int scrollContextTimeToLiveInMinutes) { + this.scrollContextTimeToLiveInMinutes = scrollContextTimeToLiveInMinutes; + } + + public int getScrollContextBatchRequestSize() { + return scrollContextBatchRequestSize; + } + + public void setScrollContextBatchRequestSize(int scrollContextBatchRequestSize) { + this.scrollContextBatchRequestSize = scrollContextBatchRequestSize; + } + + + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/ElasticSearchSchemaConfig.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/ElasticSearchSchemaConfig.java new file mode 100644 index 0000000..4f134c3 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/ElasticSearchSchemaConfig.java @@ -0,0 +1,77 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.config; + +public class ElasticSearchSchemaConfig { + + private String indexName; + private String indexDocType; + private String indexSettingsFileName; + private String indexMappingsFileName; + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getIndexDocType() { + return indexDocType; + } + + public void setIndexDocType(String indexDocType) { + this.indexDocType = indexDocType; + } + + public String getIndexSettingsFileName() { + return indexSettingsFileName; + } + + public void setIndexSettingsFileName(String indexSettingsFileName) { + this.indexSettingsFileName = indexSettingsFileName; + } + + public String getIndexMappingsFileName() { + return indexMappingsFileName; + } + + public void setIndexMappingsFileName(String indexMappingsFileName) { + this.indexMappingsFileName = indexMappingsFileName; + } + + @Override + public String toString() { + return "ElasticSearchSchemaConfig [" + + (indexName != null ? "indexName=" + indexName + ", " : "") + + (indexDocType != null ? "indexDocType=" + indexDocType + ", " : "") + + (indexSettingsFileName != null ? "indexSettingsFileName=" + indexSettingsFileName + ", " + : "") + + (indexMappingsFileName != null ? "indexMappingsFileName=" + indexMappingsFileName : "") + + "]"; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/NetworkStatisticsConfig.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/NetworkStatisticsConfig.java new file mode 100644 index 0000000..0d089a4 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/NetworkStatisticsConfig.java @@ -0,0 +1,239 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.config; + +public class NetworkStatisticsConfig { + + private int numSamplesPerThreadForRunningAverage; + + private String bytesHistogramLabel; + + private long bytesHistogramMaxYAxis; + + private int bytesHistogramNumBins; + + private int bytesHistogramNumDecimalPoints; + + private String queueLengthHistogramLabel; + + private long queueLengthHistogramMaxYAxis; + + private int queueLengthHistogramNumBins; + + private int queueLengthHistogramNumDecimalPoints; + + private String taskAgeHistogramLabel; + + private long taskAgeHistogramMaxYAxis; + + private int taskAgeHistogramNumBins; + + private int taskAgeHistogramNumDecimalPoints; + + private String responseTimeHistogramLabel; + + private long responseTimeHistogramMaxYAxis; + + private int responseTimeHistogramNumBins; + + private int responseTimeHistogramNumDecimalPoints; + + private String tpsHistogramLabel; + + private long tpsHistogramMaxYAxis; + + private int tpsHistogramNumBins; + + private int tpsHistogramNumDecimalPoints; + + public int getNumSamplesPerThreadForRunningAverage() { + return numSamplesPerThreadForRunningAverage; + } + + public void setNumSamplesPerThreadForRunningAverage(int numSamplesPerThreadForRunningAverage) { + this.numSamplesPerThreadForRunningAverage = numSamplesPerThreadForRunningAverage; + } + + public String getBytesHistogramLabel() { + return bytesHistogramLabel; + } + + public void setBytesHistogramLabel(String bytesHistogramLabel) { + this.bytesHistogramLabel = bytesHistogramLabel; + } + + public long getBytesHistogramMaxYAxis() { + return bytesHistogramMaxYAxis; + } + + public void setBytesHistogramMaxYAxis(long bytesHistogramMaxYAxis) { + this.bytesHistogramMaxYAxis = bytesHistogramMaxYAxis; + } + + public int getBytesHistogramNumBins() { + return bytesHistogramNumBins; + } + + public void setBytesHistogramNumBins(int bytesHistogramNumBins) { + this.bytesHistogramNumBins = bytesHistogramNumBins; + } + + public int getBytesHistogramNumDecimalPoints() { + return bytesHistogramNumDecimalPoints; + } + + public void setBytesHistogramNumDecimalPoints(int bytesHistogramNumDecimalPoints) { + this.bytesHistogramNumDecimalPoints = bytesHistogramNumDecimalPoints; + } + + public String getQueueLengthHistogramLabel() { + return queueLengthHistogramLabel; + } + + public void setQueueLengthHistogramLabel(String queueLengthHistogramLabel) { + this.queueLengthHistogramLabel = queueLengthHistogramLabel; + } + + public long getQueueLengthHistogramMaxYAxis() { + return queueLengthHistogramMaxYAxis; + } + + public void setQueueLengthHistogramMaxYAxis(long queueLengthHistogramMaxYAxis) { + this.queueLengthHistogramMaxYAxis = queueLengthHistogramMaxYAxis; + } + + public int getQueueLengthHistogramNumBins() { + return queueLengthHistogramNumBins; + } + + public void setQueueLengthHistogramNumBins(int queueLengthHistogramNumBins) { + this.queueLengthHistogramNumBins = queueLengthHistogramNumBins; + } + + public int getQueueLengthHistogramNumDecimalPoints() { + return queueLengthHistogramNumDecimalPoints; + } + + public void setQueueLengthHistogramNumDecimalPoints(int queueLengthHistogramNumDecimalPoints) { + this.queueLengthHistogramNumDecimalPoints = queueLengthHistogramNumDecimalPoints; + } + + public String getTaskAgeHistogramLabel() { + return taskAgeHistogramLabel; + } + + public void setTaskAgeHistogramLabel(String taskAgeHistogramLabel) { + this.taskAgeHistogramLabel = taskAgeHistogramLabel; + } + + public long getTaskAgeHistogramMaxYAxis() { + return taskAgeHistogramMaxYAxis; + } + + public void setTaskAgeHistogramMaxYAxis(long taskAgeHistogramMaxYAxis) { + this.taskAgeHistogramMaxYAxis = taskAgeHistogramMaxYAxis; + } + + public int getTaskAgeHistogramNumBins() { + return taskAgeHistogramNumBins; + } + + public void setTaskAgeHistogramNumBins(int taskAgeHistogramNumBins) { + this.taskAgeHistogramNumBins = taskAgeHistogramNumBins; + } + + public int getTaskAgeHistogramNumDecimalPoints() { + return taskAgeHistogramNumDecimalPoints; + } + + public void setTaskAgeHistogramNumDecimalPoints(int taskAgeHistogramNumDecimalPoints) { + this.taskAgeHistogramNumDecimalPoints = taskAgeHistogramNumDecimalPoints; + } + + public String getResponseTimeHistogramLabel() { + return responseTimeHistogramLabel; + } + + public void setResponseTimeHistogramLabel(String responseTimeHistogramLabel) { + this.responseTimeHistogramLabel = responseTimeHistogramLabel; + } + + public long getResponseTimeHistogramMaxYAxis() { + return responseTimeHistogramMaxYAxis; + } + + public void setResponseTimeHistogramMaxYAxis(long responseTimeHistogramMaxYAxis) { + this.responseTimeHistogramMaxYAxis = responseTimeHistogramMaxYAxis; + } + + public int getResponseTimeHistogramNumBins() { + return responseTimeHistogramNumBins; + } + + public void setResponseTimeHistogramNumBins(int responseTimeHistogramNumBins) { + this.responseTimeHistogramNumBins = responseTimeHistogramNumBins; + } + + public int getResponseTimeHistogramNumDecimalPoints() { + return responseTimeHistogramNumDecimalPoints; + } + + public void setResponseTimeHistogramNumDecimalPoints(int responseTimeHistogramNumDecimalPoints) { + this.responseTimeHistogramNumDecimalPoints = responseTimeHistogramNumDecimalPoints; + } + + public String getTpsHistogramLabel() { + return tpsHistogramLabel; + } + + public void setTpsHistogramLabel(String tpsHistogramLabel) { + this.tpsHistogramLabel = tpsHistogramLabel; + } + + public long getTpsHistogramMaxYAxis() { + return tpsHistogramMaxYAxis; + } + + public void setTpsHistogramMaxYAxis(long tpsHistogramMaxYAxis) { + this.tpsHistogramMaxYAxis = tpsHistogramMaxYAxis; + } + + public int getTpsHistogramNumBins() { + return tpsHistogramNumBins; + } + + public void setTpsHistogramNumBins(int tpsHistogramNumBins) { + this.tpsHistogramNumBins = tpsHistogramNumBins; + } + + public int getTpsHistogramNumDecimalPoints() { + return tpsHistogramNumDecimalPoints; + } + + public void setTpsHistogramNumDecimalPoints(int tpsHistogramNumDecimalPoints) { + this.tpsHistogramNumDecimalPoints = tpsHistogramNumDecimalPoints; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/SyncControllerConfig.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/SyncControllerConfig.java new file mode 100644 index 0000000..566c249 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/config/SyncControllerConfig.java @@ -0,0 +1,305 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.config; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.onap.aai.sparky.sync.SynchronizerConstants; + +public class SyncControllerConfig { + + private static final String UNKNOWN_CONTROLLER_NAME = "UnknownControllerName"; + + private String controllerName; + private boolean enabled; + private int syncTaskDelayInMs; + private int syncTaskFrequencyInDays; + + private int numSyncControllerWorkers; + private boolean runOnceSyncEnabled; + private boolean periodicSyncEnabled; + + private String targetSyncStartTimeStamp; + + private int numInternalSyncWorkers; + private int numSyncElasticWorkers; + private int numSyncActiveInventoryWorkers; + + /* + * calculated variables based on incoming config + */ + private String timeZoneOfSyncStartTimeStamp; + private int syncTaskStartTimeHr; + private int syncTaskStartTimeMin; + private int syncTaskStartTimeSec; + + + + public SyncControllerConfig() { + controllerName = UNKNOWN_CONTROLLER_NAME; + enabled = false; + syncTaskDelayInMs = 0; + syncTaskFrequencyInDays = 365; + numSyncControllerWorkers = 1; + runOnceSyncEnabled = false; + periodicSyncEnabled = false; + targetSyncStartTimeStamp = SynchronizerConstants.DEFAULT_START_TIMESTAMP; + numInternalSyncWorkers = 2; + numSyncElasticWorkers = 5; + numSyncActiveInventoryWorkers = 5; + } + + protected void initializeSyncTimeParameters() { + + if (syncTaskDelayInMs < 0) { + throw new IllegalArgumentException("syncTaskDelayInMs must >= 0"); + } + + Pattern pattern = Pattern.compile(SynchronizerConstants.TIMESTAMP24HOURS_PATTERN); + Matcher matcher = pattern.matcher(targetSyncStartTimeStamp); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid time format for targetSyncStartTimeStamp"); + } + + List<String> timestampVal = Arrays.asList(targetSyncStartTimeStamp.split(" ")); + + if (timestampVal.size() == SynchronizerConstants.COMPONENTS_IN_TIMESTAMP) { + + // Need both time and timezone offset + timeZoneOfSyncStartTimeStamp = timestampVal + .get(SynchronizerConstants.IDX_TIMEZONE_IN_TIMESTAMP).replaceAll("UTC", "GMT"); + + String time = timestampVal.get(SynchronizerConstants.IDX_TIME_IN_TIMESTAMP); + DateFormat format = new SimpleDateFormat("HH:mm:ss"); + + Date date = null; + + try { + date = format.parse(time); + } catch (ParseException parseException) { + throw new IllegalArgumentException(parseException); + } + + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + + syncTaskStartTimeHr = calendar.get(Calendar.HOUR_OF_DAY); + syncTaskStartTimeMin = calendar.get(Calendar.MINUTE); + syncTaskStartTimeSec = calendar.get(Calendar.SECOND); + } else { + throw new IllegalArgumentException("Invalid timestamp format from targetSyncStartTimeStamp"); + } + + } + + + public int getNumInternalSyncWorkers() { + return numInternalSyncWorkers; + } + + public void setNumInternalSyncWorkers(int numInternalSyncWorkers) { + this.numInternalSyncWorkers = numInternalSyncWorkers; + } + + public int getNumSyncElasticWorkers() { + return numSyncElasticWorkers; + } + + public void setNumSyncElasticWorkers(int numSyncElasticWorkers) { + this.numSyncElasticWorkers = numSyncElasticWorkers; + } + + public int getNumSyncActiveInventoryWorkers() { + return numSyncActiveInventoryWorkers; + } + + public void setNumSyncActiveInventoryWorkers(int numSyncActiveInventoryWorkers) { + this.numSyncActiveInventoryWorkers = numSyncActiveInventoryWorkers; + } + + public String getTargetSyncStartTimeStamp() { + return targetSyncStartTimeStamp; + } + + public void setTargetSyncStartTimeStamp(String targetSyncStartTimeStamp) { + this.targetSyncStartTimeStamp = targetSyncStartTimeStamp; + initializeSyncTimeParameters(); + } + + public String getControllerName() { + return controllerName; + } + + public void setControllerName(String controllerName) { + this.controllerName = controllerName; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public int getSyncTaskDelayInMs() { + return syncTaskDelayInMs; + } + + public void setSyncTaskDelayInMs(int syncTaskDelayInMs) { + this.syncTaskDelayInMs = syncTaskDelayInMs; + } + + public int getSyncTaskFrequencyInDays() { + return syncTaskFrequencyInDays; + } + + public void setSyncTaskFrequencyInDays(int syncTaskFrequencyInDays) { + this.syncTaskFrequencyInDays = syncTaskFrequencyInDays; + } + + public int getNumSyncControllerWorkers() { + return numSyncControllerWorkers; + } + + public void setNumSyncControllerWorkers(int numSyncControllerWorkers) { + this.numSyncControllerWorkers = numSyncControllerWorkers; + } + + public boolean isRunOnceSyncEnabled() { + return runOnceSyncEnabled; + } + + public void setRunOnceSyncEnabled(boolean runOnceSyncEnabled) { + this.runOnceSyncEnabled = runOnceSyncEnabled; + } + + public boolean isPeriodicSyncEnabled() { + return periodicSyncEnabled; + } + + public void setPeriodicSyncEnabled(boolean periodicSyncEnabled) { + this.periodicSyncEnabled = periodicSyncEnabled; + } + + public long getSyncFrequencyInMs() { + + return (syncTaskFrequencyInDays * SynchronizerConstants.MILLISEC_IN_A_DAY); + + } + + public Calendar getTargetSyncTime() { + + TimeZone tz = TimeZone.getTimeZone(timeZoneOfSyncStartTimeStamp); + Calendar targetSyncTime = Calendar.getInstance(tz); + + targetSyncTime.set(Calendar.HOUR_OF_DAY, syncTaskStartTimeHr); + targetSyncTime.set(Calendar.MINUTE, syncTaskStartTimeMin); + targetSyncTime.set(Calendar.SECOND, syncTaskStartTimeSec); + + return targetSyncTime; + + } + + + public String getNextSyncTime() { + + int taskFrequencyInSeconds = 0; + if (getSyncFrequencyInMs() > 0) { + taskFrequencyInSeconds = (int) (getSyncFrequencyInMs() / 1000); + } + + if (taskFrequencyInSeconds < 86400) { + + TimeZone tz = TimeZone.getTimeZone(timeZoneOfSyncStartTimeStamp); + Calendar targetSyncTime = Calendar.getInstance(tz); + targetSyncTime.add(Calendar.SECOND, taskFrequencyInSeconds); + + return SynchronizerConstants.SIMPLE_DATE_FORMAT.format(targetSyncTime.getTimeInMillis()) + .replaceAll(SynchronizerConstants.TIME_STD, SynchronizerConstants.TIME_CONFIG_STD); + + } else { + + return SynchronizerConstants.SIMPLE_DATE_FORMAT + .format(getNextSyncTime(getTargetSyncTime(), taskFrequencyInSeconds)) + .replaceAll(SynchronizerConstants.TIME_STD, SynchronizerConstants.TIME_CONFIG_STD); + + } + + } + + public long getNextSyncTime(Calendar syncTime, int taskFrequencyInSeconds) { + + TimeZone tz = TimeZone.getTimeZone(timeZoneOfSyncStartTimeStamp); + Calendar timeNow = Calendar.getInstance(tz); + + return getNextSyncTime(syncTime, timeNow.getTimeInMillis(), taskFrequencyInSeconds); + } + + /** + * Gets the first sync time. + * + * @param calendar the calendar + * @param timeNow the time now in ms + * @param taskFrequencyInMs task period in ms + * @return the first sync time + */ + + public long getNextSyncTime(Calendar syncTime, long timeNowInMs, int taskFrequencyInSeconds) { + if (taskFrequencyInSeconds == 0) { + return 0; + } else if (timeNowInMs > syncTime.getTimeInMillis()) { + + /* + * If current time is after the scheduled sync start time, then we'll skip ahead to the next + * sync time period + */ + + syncTime.add(Calendar.SECOND, taskFrequencyInSeconds); + } + + return syncTime.getTimeInMillis(); + } + + public String getTimeZoneOfSyncStartTimeStamp() { + return timeZoneOfSyncStartTimeStamp; + } + + public void setTimeZoneOfSyncStartTimeStamp(String timeZoneOfSyncStartTimeStamp) { + this.timeZoneOfSyncStartTimeStamp = timeZoneOfSyncStartTimeStamp; + } + + + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/AggregationEntity.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/AggregationEntity.java new file mode 100644 index 0000000..30d4e71 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/AggregationEntity.java @@ -0,0 +1,99 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.util.HashMap; +import java.util.Map; + +import org.onap.aai.sparky.util.NodeUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * The Class AggregationEntity. + */ +public class AggregationEntity extends IndexableEntity implements IndexDocument { + private Map<String, String> attributes = new HashMap<String, String>(); + protected ObjectMapper mapper = new ObjectMapper(); + + /** + * Instantiates a new aggregation entity. + */ + public AggregationEntity() { + super(); + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.entity.IndexDocument#deriveFields() + */ + @Override + public void deriveFields() { + + /* + * We'll try and create a unique identity key that we can use for differencing the previously + * imported record sets as we won't have granular control of what is created/removed and when. + * The best we can hope for is identification of resources by generated Id until the + * Identity-Service UUID is tagged against all resources, then we can use that instead. + */ + this.id = + NodeUtils.generateUniqueShaDigest(link); + } + + public void copyAttributeKeyValuePair(Map<String, Object> map){ + for(String key: map.keySet()){ + if (!key.equalsIgnoreCase("relationship-list")){ // ignore relationship data which is not required in aggregation + this.attributes.put(key, map.get(key).toString()); // not sure if entity attribute can contain an object as value + } + } + } + + public void addAttributeKeyValuePair(String key, String value){ + this.attributes.put(key, value); + } + + @Override + public String getAsJson() { + ObjectNode rootNode = mapper.createObjectNode(); + rootNode.put("link", this.getLink()); + rootNode.put("lastmodTimestamp", this.getEntityTimeStamp()); + for (String key: this.attributes.keySet()){ + rootNode.put(key, this.attributes.get(key)); + } + return rootNode.toString(); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "IndexDocument [" + (entityType != null ? "entityType=" + entityType + ", " : "") + + (entityPrimaryKeyValue != null ? "entityPrimaryKeyValue=" + entityPrimaryKeyValue + ", " + : "") + + (mapper != null ? "mapper=" + mapper + ", " : "") + (id != null ? "id=" + id + ", " : "") + + (lastmodTimestamp != null ? "lastmodTimestamp=" + lastmodTimestamp + ", " : "") + "]"; + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/AggregationSuggestionEntity.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/AggregationSuggestionEntity.java new file mode 100644 index 0000000..06f60b3 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/AggregationSuggestionEntity.java @@ -0,0 +1,111 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.util.ArrayList; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.onap.aai.sparky.search.filters.config.FiltersConfig; +import org.onap.aai.sparky.search.filters.config.UiFilterListItemConfig; +import org.onap.aai.sparky.search.filters.config.UiViewListItemConfig; +import org.onap.aai.sparky.util.NodeUtils; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class AggregationSuggestionEntity extends IndexableEntity implements IndexDocument { + + private static final String FILTER_ID = "filterId"; + private static final String FILTER_LIST = "filterList"; + + private List<String> inputs = new ArrayList<>(); + private final String outputString = "VNFs"; + protected ObjectMapper mapper = new ObjectMapper(); + List<String> filterIds = new ArrayList<>(); + + @JsonIgnore + private FiltersConfig filtersConfig; + + public AggregationSuggestionEntity(FiltersConfig filtersConfig) { + super(); + this.filtersConfig = filtersConfig; + inputs.add("VNFs"); + inputs.add("generic-vnfs"); + } + + @Override + public void deriveFields() { + this.id = NodeUtils.generateUniqueShaDigest(this.outputString); + } + + @Override + public String getAsJson() { + JSONArray inputArray = new JSONArray(); + for (String input: inputs) { + input = input.replace(",","" ); + input = input.replace("[","" ); + input = input.replace("]","" ); + inputArray.put(input); + } + + JSONObject entitySuggest = new JSONObject(); + entitySuggest.put("input", inputArray); + entitySuggest.put("output", this.outputString); + entitySuggest.put("weight", 100); + + JSONArray payloadFilters = new JSONArray(); + + for (String filterId : filterIds) { + JSONObject filterPayload = new JSONObject(); + filterPayload.put(FILTER_ID, filterId); + payloadFilters.put(filterPayload); + } + + JSONObject payloadNode = new JSONObject(); + payloadNode.put(FILTER_LIST, payloadFilters); + entitySuggest.put("payload", payloadNode); + + JSONObject rootNode = new JSONObject(); + rootNode.put("entity_suggest", entitySuggest); + + return rootNode.toString(); + } + + public void initializeFilters() { + for (UiViewListItemConfig view : filtersConfig.getViewsConfig().getViews()) { + if (view.getViewName().equals("vnfSearch")) { + for (UiFilterListItemConfig currentViewFilter : view.getFilters()) { + filterIds.add(currentViewFilter.getFilterId()); + } + } + } + } + + public void setFilterIds(List<String> filterIds) { + this.filterIds = filterIds; + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexDocument.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexDocument.java new file mode 100644 index 0000000..6ee3351 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexDocument.java @@ -0,0 +1,41 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +/** + * The Interface IndexDocument. + */ +public interface IndexDocument { + + /** + * Derive fields. + */ + public void deriveFields(); + + public String getId(); + + public String getAsJson() throws Exception; + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java new file mode 100644 index 0000000..0c94227 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java @@ -0,0 +1,97 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.util.ArrayList; + +import org.onap.aai.sparky.util.NodeUtils; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; + + +/** + * The Class IndexableCrossEntityReference. + */ + +public class IndexableCrossEntityReference extends IndexableEntity implements IndexDocument { + + protected String crossReferenceEntityValues; + protected ArrayList<String> crossEntityReferenceCollection = new ArrayList<String>(); + + /** + * Instantiates a new indexable cross entity reference. + */ + public IndexableCrossEntityReference() { + super(); + } + + /** + * Adds the cross entity reference value. + * + * @param crossEntityReferenceValue the cross entity reference value + */ + public void addCrossEntityReferenceValue(String crossEntityReferenceValue) { + if (!crossEntityReferenceCollection.contains(crossEntityReferenceValue)) { + crossEntityReferenceCollection.add(crossEntityReferenceValue); + } + } + + public String getCrossReferenceEntityValues() { + return crossReferenceEntityValues; + } + + public void setCrossReferenceEntityValues(String crossReferenceEntityValues) { + this.crossReferenceEntityValues = crossReferenceEntityValues; + } + + /* + * (non-Javadoc) + * + * @see org.openecomp.sparky.synchronizer.entity.IndexDocument#deriveFields() + */ + @Override + public void deriveFields() { + this.id = NodeUtils.generateUniqueShaDigest(link); + this.crossReferenceEntityValues = NodeUtils.concatArray(crossEntityReferenceCollection, ";"); + } + + @JsonIgnore // Since this is a "get" we need to JSON ignore otherwise it will be called when converting to JSON + @Override + public String getAsJson() throws JsonProcessingException { + + return NodeUtils.convertObjectToJson(this, false); + + } + + @Override + public String toString() { + return "IndexableCrossEntityReference [" + + (crossReferenceEntityValues != null + ? "crossReferenceEntityValues=" + crossReferenceEntityValues + ", " : "") + + (crossEntityReferenceCollection != null + ? "crossEntityReferenceCollection=" + crossEntityReferenceCollection + ", " : ""); + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableEntity.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableEntity.java new file mode 100644 index 0000000..9a7f865 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableEntity.java @@ -0,0 +1,100 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * The Class IndexableEntity. + */ +public abstract class IndexableEntity { + protected String id; // generated, SHA-256 digest + protected String entityType; + protected String entityPrimaryKeyValue; + protected String lastmodTimestamp; + protected String link; + + private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; + + /** + * Instantiates a new indexable entity. + */ + public IndexableEntity() { + SimpleDateFormat dateFormat = new SimpleDateFormat(TIMESTAMP_FORMAT); + Timestamp timestamp = new Timestamp(System.currentTimeMillis()); + String currentFormattedTimeStamp = dateFormat.format(timestamp); + this.setEntityTimeStamp(currentFormattedTimeStamp); + } + + @JsonIgnore + public String getId() { + return id; + } + + @JsonProperty("entityType") + public String getEntityType() { + return entityType; + } + + @JsonProperty("entityPrimaryKeyValue") + public String getEntityPrimaryKeyValue() { + return entityPrimaryKeyValue; + } + + @JsonProperty("lastmodTimestamp") + public String getEntityTimeStamp() { + return lastmodTimestamp; + } + + public void setId(String id) { + this.id = id; + } + + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + public void setEntityPrimaryKeyValue(String fieldValue) { + this.entityPrimaryKeyValue = fieldValue; + } + + public void setEntityTimeStamp(String lastmodTimestamp) { + this.lastmodTimestamp = lastmodTimestamp; + } + + @JsonProperty("link") + public String getLink() { + return link; + } + + public void setLink(String link) { + this.link = link; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/MergableEntity.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/MergableEntity.java new file mode 100644 index 0000000..82c874a --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/MergableEntity.java @@ -0,0 +1,59 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; + +/** + * The Class MergableEntity. + */ +public class MergableEntity { + private Map<String, String> other = new HashMap<String, String>(); + + /** + * Any. + * + * @return the map + */ + @JsonAnyGetter + public Map<String, String> any() { + return other; + } + + /** + * Sets the. + * + * @param name the name + * @param value the value + */ + @JsonAnySetter + public void set(String name, String value) { + other.put(name, value); + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/ObjectIdCollection.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/ObjectIdCollection.java new file mode 100644 index 0000000..8462bc9 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/ObjectIdCollection.java @@ -0,0 +1,78 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +/** + * The Class ObjectIdCollection. + */ +public class ObjectIdCollection { + + protected ConcurrentHashMap<String, String> importedObjectIds = + new ConcurrentHashMap<String, String>(); + + public Collection<String> getImportedObjectIds() { + return importedObjectIds.values(); + } + + /** + * Adds the object id. + * + * @param id the id + */ + public void addObjectId(String id) { + importedObjectIds.putIfAbsent(id, id); + } + + public int getSize() { + return importedObjectIds.values().size(); + } + + /** + * Adds the all. + * + * @param items the items + */ + public void addAll(List<String> items) { + if (items == null) { + return; + } + + items.stream().forEach((item) -> { + importedObjectIds.putIfAbsent(item, item); + }); + + } + + /** + * Clear. + */ + public void clear() { + importedObjectIds.clear(); + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SearchableEntity.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SearchableEntity.java new file mode 100644 index 0000000..84fc093 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SearchableEntity.java @@ -0,0 +1,142 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.util.ArrayList; +import java.util.List; + +import org.onap.aai.sparky.util.NodeUtils; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * The Class SearchableEntity. + */ +public class SearchableEntity extends IndexableEntity implements IndexDocument { + + @JsonIgnore + protected List<String> searchTagCollection = new ArrayList<String>(); + + @JsonIgnore + protected List<String> searchTagIdCollection = new ArrayList<String>(); + + @JsonIgnore + protected ObjectMapper mapper = new ObjectMapper(); + + /** + * Instantiates a new searchable entity. + */ + public SearchableEntity() { + super(); + } + + /* + * Generated fields, leave the settings for junit overrides + */ + + protected String searchTags; // generated based on searchTagCollection values + + protected String searchTagIDs; + + /** + * Generates the sha based id. + */ + public void generateId() { + this.id = NodeUtils.generateUniqueShaDigest(link); + } + + /* (non-Javadoc) + * @see org.openecomp.sparky.synchronizer.entity.IndexDocument#deriveFields() + */ + @Override + public void deriveFields() { + + /* + * We'll try and create a unique identity key that we can use for differencing the previously + * imported record sets as we won't have granular control of what is created/removed and when. + * The best we can hope for is identification of resources by generated Id until the + * Identity-Service UUID is tagged against all resources, then we can use that instead. + */ + generateId(); + this.searchTags = NodeUtils.concatArray(searchTagCollection, ";"); + this.searchTagIDs = NodeUtils.concatArray(this.searchTagIdCollection, ";"); + } + + /** + * Adds the search tag with key. + * + * @param searchTag the search tag + * @param searchTagKey the key associated with the search tag (key:value) + */ + public void addSearchTagWithKey(String searchTag, String searchTagKey) { + searchTagIdCollection.add(searchTagKey); + searchTagCollection.add(searchTag); + } + + public List<String> getSearchTagCollection() { + return searchTagCollection; + } + + @JsonProperty("searchTags") + public String getSearchTags() { + return searchTags; + } + + @JsonProperty("searchTagIDs") + public String getSearchTagIDs() { + return searchTagIDs; + } + + @JsonIgnore + public List<String> getSearchTagIdCollection() { + return searchTagIdCollection; + } + + @Override + @JsonIgnore + public String getAsJson() throws JsonProcessingException { + return NodeUtils.convertObjectToJson(this, false); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "IndexDocument [" + (entityType != null ? "entityType=" + entityType + ", " : "") + + (entityPrimaryKeyValue != null ? "entityPrimaryKeyValue=" + entityPrimaryKeyValue + ", " + : "") + + (searchTagCollection != null ? "searchTagCollection=" + searchTagCollection + ", " : "") + + (searchTagIdCollection != null ? "searchTagIDCollection=" + searchTagIdCollection + ", " + : "") + + (mapper != null ? "mapper=" + mapper + ", " : "") + (id != null ? "id=" + id + ", " : "") + + (lastmodTimestamp != null ? "lastmodTimestamp=" + lastmodTimestamp + ", " : "") + + (searchTags != null ? "searchTags=" + searchTags + ", " : "") + + (searchTagIDs != null ? "searchTagIDs=" + searchTagIDs : "") + "]"; + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SelfLinkDescriptor.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SelfLinkDescriptor.java new file mode 100644 index 0000000..2b3f858 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SelfLinkDescriptor.java @@ -0,0 +1,90 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +/** + * The Class SelfLinkDescriptor. + */ +public class SelfLinkDescriptor { + private String selfLink; + private String entityType; + private String depthModifier; + + public String getDepthModifier() { + return depthModifier; + } + + public void setDepthModifier(String depthModifier) { + this.depthModifier = depthModifier; + } + + public String getSelfLink() { + return selfLink; + } + + public void setSelfLink(String selfLink) { + this.selfLink = selfLink; + } + + public String getEntityType() { + return entityType; + } + + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + public SelfLinkDescriptor(String selfLink) { + this(selfLink, null, null); + } + + /** + * Instantiates a new self link descriptor. + * + * @param selfLink the self link + * @param entityType the entity type + */ + public SelfLinkDescriptor(String selfLink, String entityType) { + this(selfLink, null, entityType); + } + + public SelfLinkDescriptor(String selfLink, String depthModifier, String entityType) { + this.selfLink = selfLink; + this.entityType = entityType; + this.depthModifier = depthModifier; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "SelfLinkDescriptor [" + (selfLink != null ? "selfLink=" + selfLink + ", " : "") + + (entityType != null ? "entityType=" + entityType + ", " : "") + + (depthModifier != null ? "depthModifier=" + depthModifier : "") + "]"; + } + +} + diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SuggestionSearchEntity.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SuggestionSearchEntity.java new file mode 100644 index 0000000..3648b53 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/SuggestionSearchEntity.java @@ -0,0 +1,327 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.onap.aai.sparky.config.oxm.SuggestionEntityLookup; +import org.onap.aai.sparky.search.filters.config.FiltersConfig; +import org.onap.aai.sparky.search.filters.config.FiltersDetailsConfig; +import org.onap.aai.sparky.search.filters.config.UiFilterConfig; +import org.onap.aai.sparky.util.NodeUtils; +import org.onap.aai.sparky.util.SuggestionsPermutation; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class SuggestionSearchEntity extends IndexableEntity implements IndexDocument { + private static final String FILTER_ID = "filterId"; + private static final String FILTER_VALUE = "filterValue"; + private static final String FILTER_LIST = "filterList"; + + private String entityType; + private List<String> suggestionConnectorWords = new ArrayList<String>(); + private List<String> suggestionAttributeTypes = new ArrayList<String>(); + private List<String> suggestionAttributeValues = new ArrayList<String>(); + private List<String> suggestionTypeAliases = new ArrayList<String>(); + private List<String> suggestionInputPermutations = new ArrayList<String>(); + private List<String> suggestableAttr = new ArrayList<String>(); + + private Map<String, String> inputOutputData = new HashMap<String, String>(); + Map<String, UiFilterConfig> filters = new HashMap<String, UiFilterConfig>(); + private JSONObject payload = new JSONObject(); + private JSONArray payloadFilters = new JSONArray(); + private StringBuffer outputString = new StringBuffer(); + private String aliasToUse; + + private SuggestionEntityLookup entityLookup; + + public JSONObject getPayload() { + return payload; + } + + public void setPayload(JSONObject payload) { + this.payload = payload; + } + + protected ObjectMapper mapper = new ObjectMapper(); + + public SuggestionSearchEntity(FiltersConfig filtersConfig) { + super(); + + FiltersDetailsConfig filterConfigList = filtersConfig.getFiltersConfig(); + // Populate the map with keys that will match the suggestableAttr values + for(UiFilterConfig filter : filterConfigList.getFilters()) { + if(filter.getDataSource() != null) { + filters.put(filter.getDataSource().getFieldName(), filter); + } + } + } + + public SuggestionSearchEntity(FiltersConfig filtersConfig, SuggestionEntityLookup entityLookup) { + + this.entityLookup = entityLookup; + + FiltersDetailsConfig filterConfigList = filtersConfig.getFiltersConfig(); + // Populate the map with keys that will match the suggestableAttr values + for(UiFilterConfig filter : filterConfigList.getFilters()) { + if(filter.getDataSource() != null) { + filters.put(filter.getDataSource().getFieldName(), filter); + } + } + } + + public SuggestionSearchEntity(SuggestionEntityLookup entityLookup, FiltersConfig config) { + + FiltersDetailsConfig filterConfigList = config.getFiltersConfig(); + // Populate the map with keys that will match the suggestableAttr values + for(UiFilterConfig filter : filterConfigList.getFilters()) { + if(filter.getDataSource() != null) { + filters.put(filter.getDataSource().getFieldName(), filter); + } + } + } + + public void setSuggestableAttr(ArrayList<String> attributes) { + for (String attribute : attributes) { + this.suggestableAttr.add(attribute); + } + } + + public void setPayloadFromResponse(JsonNode node) { + if (suggestableAttr != null) { + JSONObject nodePayload = new JSONObject(); + for (String attribute : suggestableAttr) { + if (node.get(attribute) != null) { + inputOutputData.put(attribute, node.get(attribute).asText()); + this.payload.put(attribute, node.get(attribute).asText()); + } + } + } + } + + public void setFilterBasedPayloadFromResponse(JsonNode node, String entityName, ArrayList<String> uniqueList) { + + HashMap<String, String> desc = entityLookup.getSuggestionSearchEntityOxmModel().get(entityName); + + if ( desc == null ) { + return; + } + + String attr = desc.get("suggestibleAttributes"); + + if ( attr == null ) { + return; + } + + List<String> suggestableAttrOxm = Arrays.asList(attr.split(",")); + + /* + * Note: + * (1) 'uniqueList' is one item within the power set of the suggestable attributes. + * (2) 'inputeOutputData' is used to generate permutations of strings + */ + for (String selectiveAttr: uniqueList) { + if (node.get(selectiveAttr) != null) { + inputOutputData.put(selectiveAttr, node.get(selectiveAttr).asText()); + } + } + + if (suggestableAttrOxm != null) { + for (String attribute : suggestableAttrOxm) { + if (node.get(attribute) != null && uniqueList.contains(attribute)) { + UiFilterConfig filterConfig = filters.get(attribute); + if(filterConfig != null) { + JSONObject filterPayload = new JSONObject(); + filterPayload.put(FILTER_ID, filterConfig.getFilterId()); + filterPayload.put(FILTER_VALUE, node.get(attribute).asText()); + this.payloadFilters.put(filterPayload); + } else { + this.payload.put(attribute, node.get(attribute).asText()); + } + } else { + UiFilterConfig emptyValueFilterConfig = filters.get(attribute); + if(emptyValueFilterConfig != null) { + JSONObject emptyValueFilterPayload = new JSONObject(); + emptyValueFilterPayload.put(FILTER_ID, emptyValueFilterConfig.getFilterId()); + this.payloadFilters.put(emptyValueFilterPayload); + } + } + } + this.payload.put(FILTER_LIST, this.payloadFilters); + } + } + + @Override + public String getEntityType() { + return entityType; + } + + @Override + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + public List<String> getSuggestionConnectorWords() { + return suggestionConnectorWords; + } + + public void setSuggestionConnectorWords(List<String> suggestionConnectorWords) { + this.suggestionConnectorWords = suggestionConnectorWords; + } + + public List<String> getSuggestionPropertyTypes() { + return this.suggestionAttributeTypes; + } + + public void setSuggestionPropertyTypes(List<String> suggestionPropertyTypes) { + this.suggestionAttributeTypes = suggestionPropertyTypes; + } + + public List<String> getSuggestionAttributeValues() { + return this.suggestionAttributeValues; + } + + public void setSuggestionAttributeValues(List<String> suggestionAttributeValues) { + this.suggestionAttributeValues = suggestionAttributeValues; + } + + public List<String> getSuggestionAliases() { + return this.suggestionTypeAliases; + } + + public void setSuggestionAliases(List<String> suggestionAliases) { + this.suggestionTypeAliases = suggestionAliases; + } + + public List<String> getSuggestionInputPermutations() { + return this.suggestionInputPermutations; + } + + public void setSuggestionInputPermutations(List<String> permutations) { + this.suggestionInputPermutations = permutations; + } + + public void generateSuggestionInputPermutations() { + + List<String> entityNames = new ArrayList<>(); + entityNames.add(entityType); + HashMap<String, String> desc = entityLookup.getSuggestionSearchEntityOxmModel().get(this.entityType); + String attr = desc.get("suggestionAliases"); + String[] suggestionAliasesArray = attr.split(","); + suggestionTypeAliases = Arrays.asList(suggestionAliasesArray); + this.setAliasToUse(suggestionAliasesArray[suggestionAliasesArray.length - 1]); + for (String alias : suggestionTypeAliases) { + entityNames.add(alias); + } + + ArrayList<String> listToPermutate = new ArrayList<>(inputOutputData.values()); + + for (String entity : entityNames){ + listToPermutate.add(entity); // add entity-name or alias in list to permutate + List<List<String>> lists = SuggestionsPermutation.getListPermutations(listToPermutate); + for (List<String> li : lists){ + suggestionInputPermutations.add(String.join(" ", li)); + } + // prepare for the next pass: remove the entity-name or alias from the list + listToPermutate.remove(entity); + } + } + + public boolean isSuggestableDoc() { + return this.getPayload().length() != 0; + } + + + @Override + public void deriveFields() { + + int entryCounter = 1; + for (Map.Entry<String, String> outputValue : inputOutputData.entrySet()) { + if (outputValue.getValue() != null && outputValue.getValue().length() > 0) { + this.outputString.append(outputValue.getValue()); + if (entryCounter < inputOutputData.entrySet().size()) { + this.outputString.append(" and "); + } else{ + this.outputString.append(" "); + } + } + entryCounter++; + } + + this.outputString.append(this.getAliasToUse()); + this.id = NodeUtils.generateUniqueShaDigest(outputString.toString()); + } + + @Override + public String getAsJson() { + // TODO Auto-generated method stub + JSONObject rootNode = new JSONObject(); + + JSONArray suggestionsArray = new JSONArray(); + for (String suggestion : suggestionInputPermutations) { + suggestionsArray.put(suggestion); + } + + JSONObject entitySuggest = new JSONObject(); + + entitySuggest.put("input", suggestionsArray); + entitySuggest.put("output", this.outputString); + entitySuggest.put("payload", this.payload); + rootNode.put("entity_suggest", entitySuggest); + + return rootNode.toString(); + } + + public String getAliasToUse() { + return aliasToUse; + } + + public void setAliasToUse(String aliasToUse) { + this.aliasToUse = aliasToUse; + } + + public Map<String, String> getInputOutputData() { + return inputOutputData; + } + + public void setInputOutputData(Map<String, String> inputOutputData) { + this.inputOutputData = inputOutputData; + } + + @Override + public String toString() { + return "SuggestionSearchEntity [entityType=" + entityType + ", suggestionConnectorWords=" + + suggestionConnectorWords + ", suggestionAttributeTypes=" + suggestionAttributeTypes + + ", suggestionAttributeValues=" + suggestionAttributeValues + ", suggestionTypeAliases=" + + suggestionTypeAliases + ", mapper=" + mapper + "]"; + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/TransactionStorageType.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/TransactionStorageType.java new file mode 100644 index 0000000..52ea891 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/TransactionStorageType.java @@ -0,0 +1,56 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.entity; + +/** + * The Enum TransactionStorageType. + */ +public enum TransactionStorageType { + EDGE_TAG_QUERY(0, "aaiOffline/edge-tag-query"), ACTIVE_INVENTORY_QUERY(1, + "aaiOffline/active-inventory-query"); + + private Integer index; + private String outputFolder; + + /** + * Instantiates a new transaction storage type. + * + * @param index the index + * @param outputFolder the output folder + */ + TransactionStorageType(Integer index, String outputFolder) { + this.index = index; + this.outputFolder = outputFolder; + } + + public Integer getIndex() { + return index; + } + + public String getOutputFolder() { + return outputFolder; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/enumeration/OperationState.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/enumeration/OperationState.java new file mode 100644 index 0000000..b8c12c9 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/enumeration/OperationState.java @@ -0,0 +1,32 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.enumeration; + +/** + * The Enum OperationState. + */ +public enum OperationState { + INIT, OK, ERROR, ABORT, PENDING, IGNORED_SYNC_NOT_IDLE +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/enumeration/SynchronizerState.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/enumeration/SynchronizerState.java new file mode 100644 index 0000000..01b28e0 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/enumeration/SynchronizerState.java @@ -0,0 +1,32 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.enumeration; + +/** + * The Enum SynchronizerState. + */ +public enum SynchronizerState { + IDLE, PERFORMING_SYNCHRONIZATION, ABORTED +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformActiveInventoryRetrieval.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformActiveInventoryRetrieval.java new file mode 100644 index 0000000..a986dfc --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformActiveInventoryRetrieval.java @@ -0,0 +1,97 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.task; + +import java.util.Map; +import java.util.function.Supplier; + +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ActiveInventoryAdapter; +import org.onap.aai.sparky.dal.NetworkTransaction; +import org.onap.aai.sparky.logging.AaiUiMsgs; +import org.slf4j.MDC; + +/* + * Consider abstraction the tasks into common elemnts, because most of them repeat a generic call + * flow pattern + */ + +/** + * The Class PerformActiveInventoryRetrieval. + */ +public class PerformActiveInventoryRetrieval implements Supplier<NetworkTransaction> { + + private static Logger logger = LoggerFactory.getInstance().getLogger(PerformActiveInventoryRetrieval.class); + + private NetworkTransaction txn; + private ActiveInventoryAdapter aaiAdapter; + private Map<String, String> contextMap; + + /** + * Instantiates a new perform active inventory retrieval. + * + * @param txn the txn + * @param aaiProvider the aai provider + */ + public PerformActiveInventoryRetrieval(NetworkTransaction txn, + ActiveInventoryAdapter aaiAdapter) { + this.txn = txn; + this.aaiAdapter = aaiAdapter; + this.contextMap = MDC.getCopyOfContextMap(); + } + + /* (non-Javadoc) + * @see java.util.function.Supplier#get() + */ + @Override + public NetworkTransaction get() { + + txn.setTaskAgeInMs(); + + long startTimeInMs = System.currentTimeMillis(); + MDC.setContextMap(contextMap); + OperationResult result = null; + try { + + final String absoluteSelfLink = aaiAdapter.repairSelfLink(txn.getLink(), txn.getQueryParameters()); + result = aaiAdapter.queryActiveInventoryWithRetries(absoluteSelfLink, "application/json", 5); + } catch (Exception exc) { + logger.error(AaiUiMsgs.ERROR_GENERIC,"Failure to resolve self link from AAI. Error = " + exc.getMessage()); + result = new OperationResult(500, + "Caught an exception while trying to resolve link = " + exc.getMessage()); + } finally { + txn.setOperationResult(result); + txn.setOpTimeInMs(System.currentTimeMillis() - startTimeInMs); + } + + return txn; + } + + protected void setContextMap(Map<String, String> contextMap) { + this.contextMap = contextMap; + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchPut.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchPut.java new file mode 100644 index 0000000..cbc9ccb --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchPut.java @@ -0,0 +1,90 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.task; + +import java.util.Map; +import java.util.function.Supplier; + +import javax.ws.rs.core.MediaType; + +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ElasticSearchAdapter; +import org.onap.aai.sparky.dal.NetworkTransaction; +import org.slf4j.MDC; + +/** + * The Class PerformElasticSearchPut. + */ +public class PerformElasticSearchPut implements Supplier<NetworkTransaction> { + + private ElasticSearchAdapter esAdapter; + private String jsonPayload; + private NetworkTransaction txn; + private Map<String, String> contextMap; + + /** + * Instantiates a new perform elastic search put. + * + * @param jsonPayload the json payload + * @param txn the txn + * @param restDataProvider the rest data provider + */ + public PerformElasticSearchPut(String jsonPayload, NetworkTransaction txn, + ElasticSearchAdapter esAdapter) { + this.jsonPayload = jsonPayload; + this.txn = txn; + this.esAdapter = esAdapter; + this.contextMap = MDC.getCopyOfContextMap(); + } + + public PerformElasticSearchPut(String jsonPayload, NetworkTransaction txn, + ElasticSearchAdapter esAdapter, Map<String, String> contextMap) { + this.jsonPayload = jsonPayload; + this.txn = txn; + this.esAdapter = esAdapter; + this.contextMap = contextMap; + } + + /* + * (non-Javadoc) + * + * @see java.util.function.Supplier#get() + */ + @Override + public NetworkTransaction get() { + txn.setTaskAgeInMs(); + MDC.setContextMap(contextMap); + + long startTimeInMs = System.currentTimeMillis(); + + OperationResult or = + esAdapter.doPut(txn.getLink(), jsonPayload, MediaType.APPLICATION_JSON_TYPE); + + txn.setOperationResult(or); + txn.setOpTimeInMs(System.currentTimeMillis() - startTimeInMs); + + return txn; + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchRetrieval.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchRetrieval.java new file mode 100644 index 0000000..048d19e --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchRetrieval.java @@ -0,0 +1,72 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.task; + +import java.util.Map; +import java.util.function.Supplier; + +import javax.ws.rs.core.MediaType; + +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ElasticSearchAdapter; +import org.onap.aai.sparky.dal.NetworkTransaction; +import org.slf4j.MDC; + +/** + * The Class PerformElasticSearchRetrieval. + */ +public class PerformElasticSearchRetrieval implements Supplier<NetworkTransaction> { + + private NetworkTransaction txn; + private ElasticSearchAdapter esAdapter; + private Map<String, String> contextMap; + + /** + * Instantiates a new perform elastic search retrieval. + * + * @param elasticSearchTxn the elastic search txn + * @param restDataProvider the rest data provider + */ + public PerformElasticSearchRetrieval(NetworkTransaction elasticSearchTxn, + ElasticSearchAdapter esAdapter) { + this.txn = elasticSearchTxn; + this.esAdapter = esAdapter; + this.contextMap = MDC.getCopyOfContextMap(); + } + + /* (non-Javadoc) + * @see java.util.function.Supplier#get() + */ + @Override + public NetworkTransaction get() { + MDC.setContextMap(contextMap); + long startTimeInMs = System.currentTimeMillis(); + OperationResult or = esAdapter.doGet(txn.getLink(), MediaType.APPLICATION_JSON_TYPE); + txn.setOperationResult(or); + txn.setOpTimeInMs(System.currentTimeMillis() - startTimeInMs); + return txn; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchUpdate.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchUpdate.java new file mode 100644 index 0000000..2e329e5 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformElasticSearchUpdate.java @@ -0,0 +1,82 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.task; + +import java.util.Map; +import java.util.function.Supplier; + +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ElasticSearchAdapter; +import org.onap.aai.sparky.dal.NetworkTransaction; +import org.slf4j.MDC; + +/** + * The Class PerformElasticSearchUpdate. + */ +public class PerformElasticSearchUpdate implements Supplier<NetworkTransaction> { + + private ElasticSearchAdapter esAdapter; + private NetworkTransaction operationTracker; + private String updatePayload; + private String updateUrl; + private Map<String, String> contextMap; + + /** + * Instantiates a new perform elastic search update. + * + * @param updateUrl the update url + * @param updatePayload the update payload + * @param esDataProvider the es data provider + * @param transactionTracker the transaction tracker + */ + public PerformElasticSearchUpdate(String updateUrl, String updatePayload, + ElasticSearchAdapter esAdapter, NetworkTransaction transactionTracker) { + this.updateUrl = updateUrl; + this.updatePayload = updatePayload; + this.esAdapter = esAdapter; + this.contextMap = MDC.getCopyOfContextMap(); + this.operationTracker = new NetworkTransaction(); + operationTracker.setEntityType(transactionTracker.getEntityType()); + operationTracker.setDescriptor(transactionTracker.getDescriptor()); + operationTracker.setOperationType(transactionTracker.getOperationType()); + } + + /* + * (non-Javadoc) + * + * @see java.util.function.Supplier#get() + */ + @Override + public NetworkTransaction get() { + operationTracker.setTaskAgeInMs(); + MDC.setContextMap(contextMap); + long startTimeInMs = System.currentTimeMillis(); + OperationResult or = esAdapter.doBulkOperation(updateUrl, updatePayload); + operationTracker.setOperationResult(or); + operationTracker.setOpTimeInMs(System.currentTimeMillis() - startTimeInMs); + return operationTracker; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformGizmoRetrieval.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformGizmoRetrieval.java new file mode 100644 index 0000000..972b049 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/PerformGizmoRetrieval.java @@ -0,0 +1,95 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.task; + +import java.util.Map; +import java.util.function.Supplier; + +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.GizmoAdapter; +import org.onap.aai.sparky.dal.NetworkTransaction; +import org.onap.aai.sparky.logging.AaiUiMsgs; +import org.slf4j.MDC; + +/* + * Consider abstraction the tasks into common elemnts, because most of them repeat a generic call + * flow pattern + */ + +/** + * The Class PerformActiveInventoryRetrieval. + */ +public class PerformGizmoRetrieval implements Supplier<NetworkTransaction> { + + private static Logger logger = LoggerFactory.getInstance().getLogger(PerformGizmoRetrieval.class); + + private NetworkTransaction txn; + private GizmoAdapter gizmoAdapter; + private Map<String, String> contextMap; + + /** + * Instantiates a new perform active inventory retrieval. + * + * @param txn the txn + * @param aaiProvider the aai provider + */ + public PerformGizmoRetrieval(NetworkTransaction txn, + GizmoAdapter gizmoAdapter) { + this.txn = txn; + this.gizmoAdapter = gizmoAdapter; + this.contextMap = MDC.getCopyOfContextMap(); + } + + /* (non-Javadoc) + * @see java.util.function.Supplier#get() + */ + @Override + public NetworkTransaction get() { + + txn.setTaskAgeInMs(); + + long startTimeInMs = System.currentTimeMillis(); + MDC.setContextMap(contextMap); + OperationResult result = null; + try { + result = gizmoAdapter.queryGizmoWithRetries(txn.getLink(), "application/json", 5); + } catch (Exception exc) { + logger.error(AaiUiMsgs.ERROR_GENERIC,"Failure to resolve self link from AAI. Error = " + exc.getMessage()); + result = new OperationResult(500, + "Caught an exception while trying to resolve link = " + exc.getMessage()); + } finally { + txn.setOperationResult(result); + txn.setOpTimeInMs(System.currentTimeMillis() - startTimeInMs); + } + + return txn; + } + + protected void setContextMap(Map<String, String> contextMap) { + this.contextMap = contextMap; + } +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/StoreDocumentTask.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/StoreDocumentTask.java new file mode 100644 index 0000000..f92ccd3 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/StoreDocumentTask.java @@ -0,0 +1,90 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.task; + +import java.util.Map; +import java.util.function.Supplier; + +import javax.ws.rs.core.MediaType; + +import org.onap.aai.restclient.client.OperationResult; +import org.onap.aai.sparky.dal.ElasticSearchAdapter; +import org.onap.aai.sparky.dal.NetworkTransaction; +import org.onap.aai.sparky.sync.entity.IndexDocument; +import org.slf4j.MDC; + +/** + * The Class StoreDocumentTask. + */ +public class StoreDocumentTask implements Supplier<NetworkTransaction> { + + private IndexDocument doc; + + private NetworkTransaction txn; + + private ElasticSearchAdapter esAdapter; + private Map<String, String> contextMap; + + /** + * Instantiates a new store document task. + * + * @param doc the doc + * @param txn the txn + * @param esDataProvider the es data provider + */ + public StoreDocumentTask(IndexDocument doc, NetworkTransaction txn, + ElasticSearchAdapter esAdapter) { + this.doc = doc; + this.txn = txn; + this.esAdapter = esAdapter; + this.contextMap = MDC.getCopyOfContextMap(); + } + + /* (non-Javadoc) + * @see java.util.function.Supplier#get() + */ + @Override + public NetworkTransaction get() { + txn.setTaskAgeInMs(); + + long startTimeInMs = System.currentTimeMillis(); + MDC.setContextMap(contextMap); + OperationResult operationResult = null; + + try { + + operationResult = + esAdapter.doPut(txn.getLink(), doc.getAsJson(), MediaType.APPLICATION_JSON_TYPE); + txn.setOpTimeInMs(System.currentTimeMillis() - startTimeInMs); + } catch (Exception exception) { + operationResult.setResult(500, exception.getMessage()); + } + + txn.setOperationResult(operationResult); + + return txn; + } + +} diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/SyncControllerTask.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/SyncControllerTask.java new file mode 100644 index 0000000..92f3683 --- /dev/null +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/task/SyncControllerTask.java @@ -0,0 +1,55 @@ +/** + * ============LICENSE_START=================================================== + * SPARKY (AAI UI service) + * ============================================================================ + * Copyright © 2017 AT&T Intellectual Property. + * Copyright © 2017 Amdocs + * All rights reserved. + * ============================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END===================================================== + * + * ECOMP and OpenECOMP are trademarks + * and service marks of AT&T Intellectual Property. + */ +package org.onap.aai.sparky.sync.task; + +import org.onap.aai.sparky.sync.SyncController; +import org.onap.aai.sparky.sync.SyncControllerImpl.SyncActions; +import org.onap.aai.sparky.sync.enumeration.SynchronizerState; + +public class SyncControllerTask implements Runnable { + + private SyncController controller; + + public SyncControllerTask(SyncController controller) { + this.controller = controller; + } + + @Override + public void run() { + + controller.performAction(SyncActions.SYNCHRONIZE); + + while (controller.getState() == SynchronizerState.PERFORMING_SYNCHRONIZATION) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // exit out of the sync-wait-loop + break; + } + } + + } + +} |