diff options
author | LaMont, William (wl2432) <wl2432@att.com> | 2019-01-22 15:02:47 -0500 |
---|---|---|
committer | LaMont, William (wl2432) <wl2432@att.com> | 2019-01-22 15:03:05 -0500 |
commit | 34ca2877182d1a505015578b0728f32452ac5cbb (patch) | |
tree | 324e52ca003d2acac7d2a40b8786d89b89886cb5 /src/main | |
parent | 4e579fc1bca9fdc42d294b21541cfe80cb0c60a2 (diff) |
update cacher repo with v15
Issue-ID: AAI-2112
Change-Id: I8aab3d9c498d217638d50b8fde5f500b556d7fd0
Signed-off-by: LaMont, William (wl2432) <wl2432@att.com>
Diffstat (limited to 'src/main')
32 files changed, 1304 insertions, 693 deletions
diff --git a/src/main/docker/docker-entrypoint.sh b/src/main/docker/docker-entrypoint.sh index 06fd7f4..7d53478 100644 --- a/src/main/docker/docker-entrypoint.sh +++ b/src/main/docker/docker-entrypoint.sh @@ -3,11 +3,6 @@ APP_HOME=$(pwd); RESOURCES_HOME=${APP_HOME}/resources/; -export CHEF_CONFIG_REPO=${CHEF_CONFIG_REPO:-aai-config}; -export CHEF_GIT_URL=${CHEF_GIT_URL:-http://gerrit.onap.org/r/aai}; -export CHEF_CONFIG_GIT_URL=${CHEF_CONFIG_GIT_URL:-$CHEF_GIT_URL}; -export CHEF_DATA_GIT_URL=${CHEF_DATA_GIT_URL:-$CHEF_GIT_URL}; - export SERVER_PORT=${SERVER_PORT:-8444}; USER_ID=${LOCAL_USER_ID:-9001} @@ -29,7 +24,7 @@ if [ $(cat /etc/passwd | grep aaiadmin | wc -l) -eq 0 ]; then } fi; -chown -R aaiadmin:aaiadmin /opt/app /opt/aai/logroot /var/chef +chown -R aaiadmin:aaiadmin /opt/app /opt/aai/logroot find /opt/app/ -name "*.sh" -exec chmod +x {} + gosu aaiadmin ln -s bin scripts @@ -66,8 +61,7 @@ fi; MIN_HEAP_SIZE=${MIN_HEAP_SIZE:-512m}; MAX_HEAP_SIZE=${MAX_HEAP_SIZE:-1024m}; -MAX_PERM_SIZE=${MAX_PERM_SIZE:-512m}; -PERM_SIZE=${PERM_SIZE:-512m}; +MAX_METASPACE_SIZE=${MAX_METASPACE_SIZE:-512m}; JAVA_CMD="exec gosu aaiadmin java"; @@ -78,8 +72,7 @@ JVM_OPTS="${JVM_OPTS} -Xmx${MAX_HEAP_SIZE}"; JVM_OPTS="${JVM_OPTS} -XX:+PrintGCDetails"; JVM_OPTS="${JVM_OPTS} -XX:+PrintGCTimeStamps"; -JVM_OPTS="${JVM_OPTS} -XX:MaxPermSize=${MAX_PERM_SIZE}"; -JVM_OPTS="${JVM_OPTS} -XX:PermSize=${PERM_SIZE}"; +JVM_OPTS="${JVM_OPTS} -XX:MaxMetaspaceSize=${MAX_METASPACE_SIZE}"; JVM_OPTS="${JVM_OPTS} -server"; JVM_OPTS="${JVM_OPTS} -XX:NewSize=512m"; @@ -114,6 +107,7 @@ JAVA_OPTS="${JAVA_OPTS} -DAFT_ENVIRONMENT=${AFT_ENVIRONMENT}"; JAVA_OPTS="${JAVA_OPTS} -DAAI_BUILD_VERSION=${AAI_BUILD_VERSION}"; JAVA_OPTS="${JAVA_OPTS} -Djava.security.egd=file:/dev/./urandom"; JAVA_OPTS="${JAVA_OPTS} -Dlogback.configurationFile=./resources/logback.xml"; +JAVA_OPTS="${JAVA_OPTS} -Dgroovy.use.classvalue=true"; JAVA_OPTS="${JAVA_OPTS} -Dloader.path=$APP_HOME/resources"; if [ ! -z ${PROXY_HOST} ]; then JAVA_OPTS="${JAVA_OPTS} -DproxySet=true -DproxyHost=${PROXY_HOST} -DproxyPort=8080"; diff --git a/src/main/java/org/onap/aai/cacher/Application.java b/src/main/java/org/onap/aai/cacher/Application.java index 9382f4b..6d40b68 100644 --- a/src/main/java/org/onap/aai/cacher/Application.java +++ b/src/main/java/org/onap/aai/cacher/Application.java @@ -19,6 +19,8 @@ */ package org.onap.aai.cacher; +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; import org.onap.aai.cacher.config.PropertyPasswordConfiguration; import org.onap.aai.logging.LoggingContext; import org.onap.aai.logging.LoggingContext.StatusCode; @@ -30,18 +32,20 @@ import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfigurat import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.web.support.SpringBootServletInitializer; import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.EnableScheduling; +import javax.annotation.PreDestroy; import java.util.UUID; @SpringBootApplication @EnableScheduling @EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class, HibernateJpaAutoConfiguration.class }) @ComponentScan(basePackages = { "org.onap.aai.cacher", "com" }) -@PropertySource("classpath:application.properties") public class Application extends SpringBootServletInitializer { + private static final String APP_NAME = "cacher"; + private static final EELFLogger logger = EELFManager.getInstance().getLogger(Application.class.getName()); + @Override protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { @@ -66,6 +70,21 @@ public class Application extends SpringBootServletInitializer { app.addInitializers(new PropertyPasswordConfiguration()); app.run(args); + logger.info("Cacher MicroService Started"); + logger.error("Cacher MicroService Started"); + logger.debug("Cacher MicroService Started"); + + System.out.println("Cacher MicroService Started"); + + } + + @PreDestroy + public void cleanup(){ + logger.info("Cacher MicroService shutting down."); + logger.error("Cacher MicroService shutting down."); + logger.debug("Cacher MicroService shutting down."); + + System.out.println("Cacher MicroService shutting down."); } public static void setDefaultProps() { diff --git a/src/main/java/org/onap/aai/cacher/common/MongoConfig.java b/src/main/java/org/onap/aai/cacher/common/MongoConfig.java index d08930b..5c1a541 100644 --- a/src/main/java/org/onap/aai/cacher/common/MongoConfig.java +++ b/src/main/java/org/onap/aai/cacher/common/MongoConfig.java @@ -24,23 +24,27 @@ import com.att.eelf.configuration.EELFManager; import com.mongodb.DB; import com.mongodb.MongoClient; import com.mongodb.client.MongoDatabase; +import de.flapdoodle.embed.mongo.Command; import de.flapdoodle.embed.mongo.MongodExecutable; import de.flapdoodle.embed.mongo.MongodProcess; import de.flapdoodle.embed.mongo.MongodStarter; -import de.flapdoodle.embed.mongo.config.IMongodConfig; -import de.flapdoodle.embed.mongo.config.MongodConfigBuilder; -import de.flapdoodle.embed.mongo.config.Net; +import de.flapdoodle.embed.mongo.config.*; import de.flapdoodle.embed.mongo.distribution.Version; +import de.flapdoodle.embed.process.config.io.ProcessOutput; +import de.flapdoodle.embed.process.io.Processors; +import de.flapdoodle.embed.process.io.Slf4jLevel; import de.flapdoodle.embed.process.runtime.Network; import org.onap.aai.exceptions.AAIException; import org.onap.aai.logging.ErrorLogHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import javax.annotation.PostConstruct; import java.io.IOException; + @Configuration public class MongoConfig { @@ -53,8 +57,10 @@ public class MongoConfig { @Value("${mongodb.port}") private int MONGO_DB_PORT; + private MongodProcess mongod; + @Bean - public MongoClient mongoClient() { + public MongoClient mongoClient(MongodProcess mongodProcess) { try { // To connect to mongodb server MongoClient mongoC = new MongoClient(MONGO_DB_HOST, MONGO_DB_PORT); @@ -83,21 +89,33 @@ public class MongoConfig { } @Bean - @PostConstruct - public MongodProcess mongoEmbedded() throws IOException, InterruptedException { + public MongodProcess mongoEmbedded() throws IOException { - MongodStarter starter = MongodStarter.getDefaultInstance(); - - String bindIp = MONGO_DB_HOST; + Logger logger = LoggerFactory.getLogger("mongo"); int port = MONGO_DB_PORT; - IMongodConfig mongodConfig = new MongodConfigBuilder().version(Version.Main.PRODUCTION) - .net(new Net(port, Network.localhostIsIPv6())).configServer(false).build(); - MongodExecutable mongodExecutable = starter.prepare(mongodConfig); - // Thread.sleep(20000L); - MongodProcess mongod = mongodExecutable.start(); + IMongodConfig mongoConfigConfig = new MongodConfigBuilder() + .version(Version.Main.PRODUCTION) + .net(new Net(port, Network.localhostIsIPv6())) + .cmdOptions(new MongoCmdOptionsBuilder().enableTextSearch(true).useNoPrealloc(false).build()) + .configServer(false) + .build(); + + ProcessOutput processOutput = new ProcessOutput(Processors.logTo(logger, Slf4jLevel.WARN), Processors.logTo(logger, + Slf4jLevel.WARN), Processors.logTo(logger, Slf4jLevel.WARN)); + + MongodExecutable mongodExecutable = MongodStarter + .getInstance((new RuntimeConfigBuilder()) + .defaults(Command.MongoD) + .processOutput(processOutput) + .build()) + .prepare(mongoConfigConfig); + + mongod = mongodExecutable.start(); + if (mongod.isProcessRunning()) { - System.out.println("RUNNING"); + System.out.println("Embedded mongo RUNNING"); + logger.info("Embedded mongo RUNNING"); } return mongod; } diff --git a/src/main/java/org/onap/aai/cacher/common/MongoHelperSingleton.java b/src/main/java/org/onap/aai/cacher/common/MongoHelperSingleton.java index 63638f1..a94f36a 100644 --- a/src/main/java/org/onap/aai/cacher/common/MongoHelperSingleton.java +++ b/src/main/java/org/onap/aai/cacher/common/MongoHelperSingleton.java @@ -23,6 +23,7 @@ import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.google.common.collect.Lists; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.mongodb.*; import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoCollection; @@ -62,6 +63,8 @@ public class MongoHelperSingleton { private MongoDatabase mongoDatabase; + private JsonParser jsonParser = new JsonParser(); + @Autowired public MongoHelperSingleton(DB db, MongoDatabase mongoDatabase) { this.mongoDatabase = mongoDatabase; @@ -167,11 +170,10 @@ public class MongoHelperSingleton { MongoCollection<Document> collection = mongoDatabase.getCollection(cacheEntry.getCollection()); Document findQuery = Document.parse(cacheEntry.getFindQuery().toString()); - Document payload = Document.parse(cacheEntry.getPayload().toString()); if (!cacheEntry.isNested()) { + Document payload = Document.parse(cacheEntry.getPayload().toString()); UpdateResult updateResult = collection.replaceOne(findQuery, payload, new UpdateOptions().upsert(true)); - return updateResult.wasAcknowledged(); } else { CacheEntry localCacheEntry = CacheEntry.CacheEntryBuilder.createCacheEntry().deepCopy(cacheEntry).build(); @@ -184,7 +186,12 @@ public class MongoHelperSingleton { ArrayList<Document> filters = this.getFiltersAndUpdateNestedField(localCacheEntry); Document doc = new Document(); - doc.put(localCacheEntry.getNestedField(), payload); + if (localCacheEntry.isNestedPayloadString()) { + doc.put(localCacheEntry.getNestedField(), localCacheEntry.getNestedString()); + } else { + Document payload = Document.parse(cacheEntry.getPayload().toString()); + doc.put(localCacheEntry.getNestedField(), payload); + } Document push = new Document(); push.put("$push", doc); @@ -266,7 +273,7 @@ public class MongoHelperSingleton { Document.parse(localCacheEntry.getNestedFieldIdentifierObj().toString())); Document pull = new Document(); pull.put("$pull", pullObj); - collection.findOneAndUpdate(nestedFind, pull, new FindOneAndUpdateOptions().arrayFilters(filters).upsert(true)); + collection.findOneAndUpdate(nestedFind, pull, new FindOneAndUpdateOptions().upsert(true)); // TODO remove wrapping if there are no entries in array. } @@ -309,4 +316,36 @@ public class MongoHelperSingleton { return filters; } + public List<JsonObject> findByIds(String collection, List<String> ids) { + final List<JsonObject> results = new ArrayList<>(ids.size()); + Document findDoc = Document.parse("{}"); + Document findIn = Document.parse("{}"); + findIn.put("$in", ids); + findDoc.put("_id", findIn); + this.mongoDatabase.getCollection(collection) + .find(findDoc) + .iterator() + .forEachRemaining(doc -> results.add(jsonParser.parse(doc.toJson()).getAsJsonObject())); + return results; + } + + /** + * Find in the specified collection all of the documents who's _id starts with (inclusive) prefix. + * @param collection collection to be searched + * @param prefix id prefix to be used in the search + * @return Collection of JsonObjects satisfying the search + */ + public List<JsonObject> findAllWithIdsStartingWith(String collection, String prefix) { + final List<JsonObject> results = new ArrayList<>(); + Document findDoc = Document.parse("{}"); + Document findIn = Document.parse("{}"); + findIn.put("$regex", "^"+prefix); + findDoc.put("_id", findIn); + this.mongoDatabase.getCollection(collection) + .find(findDoc) + .iterator() + .forEachRemaining(doc -> results.add(jsonParser.parse(doc.toJson()).getAsJsonObject())); + return results; + } + } diff --git a/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIDmaapEventProcessor.java b/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIDmaapEventProcessor.java index 5c684a2..3f28a26 100644 --- a/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIDmaapEventProcessor.java +++ b/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIDmaapEventProcessor.java @@ -21,13 +21,14 @@ package org.onap.aai.cacher.dmaap.consumer; import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; +import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; import com.mongodb.MongoCommandException; -import org.bson.Document; -import org.json.JSONException; -import org.json.JSONObject; import org.onap.aai.cacher.common.MongoHelperSingleton; +import org.onap.aai.cacher.egestion.printer.PayloadPrinterService; +import org.onap.aai.cacher.egestion.printer.strategy.PayloadPrinterType; import org.onap.aai.cacher.injestion.parser.PayloadParserService; import org.onap.aai.cacher.injestion.parser.strategy.PayloadParserType; import org.onap.aai.cacher.model.CacheEntry; @@ -37,7 +38,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.List; -import java.util.Optional; @Service public class AAIDmaapEventProcessor implements DmaapProcessor { @@ -46,21 +46,18 @@ public class AAIDmaapEventProcessor implements DmaapProcessor { private final JsonParser parser = new JsonParser(); - private JSONObject event; - private JSONObject eventHeader; - private JSONObject eventBody; - private MongoHelperSingleton mongoHelper; private PayloadParserService payloadParserService; + private PayloadPrinterService payloadPrinterService; @Autowired - public AAIDmaapEventProcessor(MongoHelperSingleton mongoHelper, PayloadParserService payloadParserService) { + public AAIDmaapEventProcessor(MongoHelperSingleton mongoHelper, PayloadParserService payloadParserService, PayloadPrinterService payloadPrinterService) { this.mongoHelper = mongoHelper; this.payloadParserService = payloadParserService; + this.payloadPrinterService = payloadPrinterService; } - public AAIDmaapEventProcessor() { - } + public AAIDmaapEventProcessor(){} /** * @@ -68,14 +65,13 @@ public class AAIDmaapEventProcessor implements DmaapProcessor { * @return */ public void process(String eventMessage) throws Exception { - this.event = null; - this.eventHeader = null; - this.eventBody = null; + JsonObject event; + JsonObject eventHeader; try { LOGGER.debug("Processing event: " + eventMessage); - this.event = new JSONObject(eventMessage); - } catch (JSONException je) { + event = parser.parse(eventMessage).getAsJsonObject(); + } catch (JsonSyntaxException | IllegalStateException je) { LOGGER.error("ERROR: Event is not valid JSON [" + eventMessage + "]."); ErrorLogHelper.logException(new AAIException("AAI_4000", je)); throw je; @@ -83,8 +79,8 @@ public class AAIDmaapEventProcessor implements DmaapProcessor { try { LOGGER.debug("Validating event header."); - this.validateEventHeader(this.event); - } catch (JSONException je) { + eventHeader = this.getEventHeader(event); + } catch (JsonSyntaxException | IllegalStateException je) { LOGGER.error("ERROR: Event header is not valid [" + eventMessage + "]."); ErrorLogHelper.logException(new AAIException("AAI_4000", je)); throw je; @@ -92,30 +88,32 @@ public class AAIDmaapEventProcessor implements DmaapProcessor { try { LOGGER.debug("Processing entity."); - eventBody = this.event.getJSONObject("entity"); - } catch (JSONException je) { + validateEventBody(event); + } catch (JsonSyntaxException | IllegalStateException je) { LOGGER.error("ERROR: Event body is not valid JSON [" + eventMessage + "]."); ErrorLogHelper.logException(new AAIException("AAI_4000", je)); throw je; } - List<CacheEntry> dmaapCacheEntries = payloadParserService.doParse("aai-dmaap", - parser.parse(eventMessage).getAsJsonObject(), PayloadParserType.AAI_RESOURCE_DMAAP); + JsonObject dmaapJson = parser.parse(eventMessage).getAsJsonObject(); // Get existing object if is update - Optional<Document> existingObj = Optional.empty(); - if (this.eventHeader != null && "UPDATE".equals(eventHeader.getString("action"))) { - existingObj = mongoHelper.getObject(dmaapCacheEntries.get(0)); + if (eventHeader != null && "UPDATE".equals(eventHeader.get("action").getAsString())) { + String uri = eventHeader.get("entity-link").getAsString() + .replaceAll("/aai/v\\d+", ""); + String type = eventHeader.getAsJsonObject().get("entity-type").getAsString(); + + List<JsonObject> found = mongoHelper.findAllWithIdsStartingWith(type, uri); + if (!found.isEmpty()) { + JsonArray ja = new JsonArray(); + found.forEach(ja::add); + JsonObject existing = payloadPrinterService.createJson(type, ja, PayloadPrinterType.AAI_RESOURCE_GET_ALL_PRINTER); + dmaapJson.add("existing-obj", existing); + } } - // Add existing object to payload to be parsed by AAI_RESOURCE_DMAAP parser - if (existingObj.isPresent()) { - JsonObject eventMessageObj = parser.parse(eventMessage).getAsJsonObject(); - eventMessageObj.add("existing-obj", parser.parse(existingObj.get().toJson()).getAsJsonObject()); - eventMessage = eventMessageObj.toString(); - dmaapCacheEntries = payloadParserService.doParse("aai-dmaap", parser.parse(eventMessage).getAsJsonObject(), - PayloadParserType.AAI_RESOURCE_DMAAP); - } + List<CacheEntry> dmaapCacheEntries = payloadParserService.doParse("aai-dmaap", + dmaapJson, PayloadParserType.AAI_RESOURCE_DMAAP); for (CacheEntry cacheEntry : dmaapCacheEntries) { try { @@ -139,37 +137,34 @@ public class AAIDmaapEventProcessor implements DmaapProcessor { } + private void validateEventBody(JsonObject event) { + if (!event.has("entity")) { + throw new JsonSyntaxException("Event header missing."); + } + + event.get("entity").getAsJsonObject(); + } + /** * Validates that the event header has the id and source name for processing. * (needed for status response msg) * * @param event - * @throws JSONException + * @throws JsonSyntaxException */ - private void validateEventHeader(JSONObject event) throws JSONException { - eventHeader = event.getJSONObject("event-header"); - if (this.eventHeader.getString("id") == null || this.eventHeader.getString("id").isEmpty()) { - throw new JSONException("Event header id missing."); - } else if (this.eventHeader.getString("source-name") == null - || this.eventHeader.getString("source-name").isEmpty()) { - throw new JSONException("Event header source-name missing."); - } - } + private JsonObject getEventHeader(JsonObject event) throws JsonSyntaxException { - /** - * - * @return - */ - public JSONObject getEventHeader() { + if (!event.has("event-header")) { + throw new JsonSyntaxException("Event header missing."); + } + JsonObject eventHeader = event.get("event-header").getAsJsonObject(); + if (eventHeader.get("id") == null || eventHeader.get("id").getAsString().isEmpty()) { + throw new JsonSyntaxException("Event header id missing."); + } else if (eventHeader.get("source-name") == null || eventHeader.get("source-name").getAsString().isEmpty()) { + throw new JsonSyntaxException("Event header source-name missing."); + } return eventHeader; - } - /** - * - * @return - */ - public JSONObject getEventBody() { - return eventBody; } } diff --git a/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIParentEventConsumer.java b/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIParentEventConsumer.java index 8d64116..2530828 100644 --- a/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIParentEventConsumer.java +++ b/src/main/java/org/onap/aai/cacher/dmaap/consumer/AAIParentEventConsumer.java @@ -46,6 +46,7 @@ public class AAIParentEventConsumer { protected String dmaapPropertyHome = ""; protected String dmaapConusmerId = ""; + private String dmaapGroup; protected String transId = ""; protected Properties aaiDmaapEventConsumerProperties = new Properties(); @@ -74,6 +75,11 @@ public class AAIParentEventConsumer { } this.dmaapConusmerId = dmaapConsumerSingleton.getDmaapConsumerId(); + if (dmaapConsumerSingleton.getDmaapGroup() == null) { + dmaapConsumerSingleton.setDmaapGroup("cacher-" + UUID.randomUUID().toString()); + } + this.dmaapGroup = dmaapConsumerSingleton.getDmaapGroup(); + processPropertyFiles(consumerPropFile); if (!injestConsumer) { this.aaiDmaapEventConsumer = MRClientFactory.createConsumer(this.aaiDmaapEventConsumerProperties.toString()); @@ -127,8 +133,10 @@ public class AAIParentEventConsumer { this.aaiDmaapEventConsumerProperties.load(reader); } + aaiDmaapEventConsumerProperties.setProperty("id", this.dmaapConusmerId); - LOGGER.debug("Updated " + this.aaiDmaapEventConsumerPropertiesFile + " id " + this.dmaapConusmerId); + aaiDmaapEventConsumerProperties.setProperty("group", this.dmaapGroup); + LOGGER.debug("Updated " + this.aaiDmaapEventConsumerPropertiesFile + " group" + this.dmaapGroup + " id " + this.dmaapConusmerId); aaiDmaapEventConsumerProperties.setProperty("DME2preferredRouterFilePath", this.preferredRouterFilePath); if (aaiDmaapEventConsumerProperties.getProperty("password") != null @@ -144,11 +152,12 @@ public class AAIParentEventConsumer { } LOGGER.debug("Using limit " + aaiDmaapEventConsumerProperties.getProperty("limit")); LOGGER.debug("Using filter " + aaiDmaapEventConsumerProperties.getProperty("filter")); + + LOGGER.debug("Dmaap Properties = " + aaiDmaapEventConsumerProperties); } public void startProcessing(DmaapProcessor dmaapProcessor) throws Exception { int fetchFailCounter = 0; - while (AAIConfig.get("aai.cacher.dmaap.consumer.enableEventProcessing").equals("true")) { try { LOGGER.debug("processEvents=" + dmaapConsumerSingleton.getProcessEvents() + " isInitialized=" diff --git a/src/main/java/org/onap/aai/cacher/dmaap/consumer/DmaapConsumerSingleton.java b/src/main/java/org/onap/aai/cacher/dmaap/consumer/DmaapConsumerSingleton.java index 5c2faa3..a88e48d 100644 --- a/src/main/java/org/onap/aai/cacher/dmaap/consumer/DmaapConsumerSingleton.java +++ b/src/main/java/org/onap/aai/cacher/dmaap/consumer/DmaapConsumerSingleton.java @@ -25,6 +25,8 @@ public class DmaapConsumerSingleton { private boolean isInitialized; private String dmaapConsumerId; + private String dmaapGroup; + private String firstEventMessage; @@ -70,6 +72,13 @@ public class DmaapConsumerSingleton { public void setDmaapConsumerId(String dmaapConsumerId) { this.dmaapConsumerId = dmaapConsumerId; } + public String getDmaapGroup() { + return dmaapGroup; + } + + public void setDmaapGroup(String dmaapGroup) { + this.dmaapGroup = dmaapGroup; + } public String getDmaapConsumerId() { return dmaapConsumerId; diff --git a/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/AAIResourceGetAllPayloadPrinterStrategy.java b/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/AAIResourceGetAllPayloadPrinterStrategy.java index 8dfeb0b..4b483c3 100644 --- a/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/AAIResourceGetAllPayloadPrinterStrategy.java +++ b/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/AAIResourceGetAllPayloadPrinterStrategy.java @@ -21,29 +21,216 @@ package org.onap.aai.cacher.egestion.printer.strategy; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import org.onap.aai.cacher.injestion.parser.strategy.aai.AAIResourcesUriTemplates; +import org.onap.aai.cacher.injestion.parser.strategy.aai.AAIUriSegment; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import java.util.*; +import java.util.stream.Collectors; + +/** + * This strategy re constructs the aai objects as the parser for this strategy + * would have taken all nested objects and stored them separately + */ @Component(value = "aai-resource-get-all-printer") @Scope(scopeName = ConfigurableBeanFactory.SCOPE_SINGLETON) public class AAIResourceGetAllPayloadPrinterStrategy implements PayloadPrinterStrategy { + private static final String RELATIONSHIP = "relationship"; + + private AAIResourcesUriTemplates aaiResourcesUriTemplates; + + @Autowired + public AAIResourceGetAllPayloadPrinterStrategy(AAIResourcesUriTemplates aaiResourcesUriTemplates) { + this.aaiResourcesUriTemplates = aaiResourcesUriTemplates; + } + + /** * Create a jsonObject from the jsonArray for a specific collection - * - * @param collectionName - * @param jsonArray - * @return + * + * @param collectionName name of the collection the json array is from. + * @param jsonArray json objects from the collection to be aggregated + * @return the cache payload reconstructed */ @Override public JsonObject createJson(String collectionName, JsonArray jsonArray) { - if (jsonArray != null && jsonArray.size() > 0) { - JsonObject jsonObj = new JsonObject(); - jsonObj.add(collectionName, jsonArray); - return jsonObj; - } else { + + List<JsonObject> topLevels = new ArrayList<>(); + Map<String, JsonObject> nested = new HashMap<>(); + + if (jsonArray == null || jsonArray.size() == 0) { return null; } + + for (int i = 0; i < jsonArray.size(); i++) { + final JsonObject jsonObject = jsonArray.get(i).getAsJsonObject(); + String id = jsonObject.get("_id").getAsString(); + List<AAIUriSegment> segments = aaiResourcesUriTemplates.getAaiUriSegments(id); + if (segments.size() == 1) { + topLevels.add(jsonObject); + } else { + nested.put(id, jsonObject); + } + } + + jsonArray = new JsonArray(); + topLevels.forEach(top -> populateNested(top, nested)); + topLevels.stream().filter(jsonObject -> jsonObject.size() > 0).forEach(jsonArray::add); + JsonObject jsonObj = new JsonObject(); + jsonObj.add(collectionName, jsonArray); + return jsonObj; + + } + + /** + * For the json object collect and populate all of the nested objects + * @param jo JsonObject to be populated + */ + private void populateNested(JsonObject jo, Map<String, JsonObject> nested) { + List<String> uris = new ArrayList<>(); + MultiValueMap<Integer, String> lvlToUri = new LinkedMultiValueMap<>(); + Map<String, JsonObject> uriToJson = new HashMap<>(); + int lvl = 0; + String uri = jo.getAsJsonPrimitive("_id").getAsString(); + lvlToUri.add(lvl, uri); + uriToJson.put(uri, jo); + uris.add(uri); + + boolean foundNested = true; + int start = 0; + while (foundNested) { + List<JsonObject> nestedObjs = new ArrayList<>(); + for (int i = start; i < uris.size(); i++) { + Set<String> nestedProps = nestedPropsToBePopulated(uriToJson.get(uris.get(i))); + nestedProps = cleanEmptyProps(uriToJson.get(uris.get(i)), nestedProps, nested); + nestedObjs.addAll(getNestedObjs(uriToJson.get(uris.get(i)), nestedProps, nested)); + } + if (nestedObjs.isEmpty()) { + foundNested = false; + } else { + lvl++; + start = uris.size(); + for (JsonObject nestedObj : nestedObjs) { + String u = nestedObj.get("_id").getAsString(); + uris.add(u); + uriToJson.put(u, nestedObj); + lvlToUri.add(lvl, u); + } + } + } + + uriToJson.values().forEach(jsonObject -> jsonObject.remove("_id")); + + // skips the last lvl as it will not have any nested + for (int i = lvl-1; i >= 0; i--) { + lvlToUri.get(i).forEach(u -> { + Set<String> nestedProps = nestedPropsToBePopulated(uriToJson.get(u)); + nestedProps.forEach(prop -> { + if (uriToJson.get(u).get(prop).isJsonArray()) { + JsonArray objs = new JsonArray(); + uriToJson.get(u).get(prop).getAsJsonArray().forEach(jsonElement -> { + if (jsonElement != null + && jsonElement.isJsonPrimitive() + && jsonElement.getAsJsonPrimitive().isString() + && uriToJson.containsKey(jsonElement.getAsString())) { + objs.add(uriToJson.get(jsonElement.getAsString())); + uriToJson.remove(jsonElement.getAsString()); + } + }); + uriToJson.get(u).add(prop, objs); + } else if (uriToJson.get(u).get(prop).isJsonObject()) { + JsonArray objs = new JsonArray(); + JsonObject jsonObject = uriToJson.get(u).get(prop).getAsJsonObject(); + String key = jsonObject.entrySet().iterator().next().getKey(); + jsonObject.get(key).getAsJsonArray().forEach(jsonElement -> { + if (jsonElement != null + && jsonElement.isJsonPrimitive() + && jsonElement.getAsJsonPrimitive().isString() + && uriToJson.containsKey(jsonElement.getAsString())) { + objs.add(uriToJson.get(jsonElement.getAsString())); + uriToJson.remove(jsonElement.getAsString()); + } + }); + uriToJson.get(u).get(prop).getAsJsonObject().add(key, objs); + } + }); + }); + } + //return uriToJson.get(uri); + } + + private Set<String> cleanEmptyProps(JsonObject jsonObject, Set<String> nestedProps, Map<String, JsonObject> nested) { + Set<String> updatedNested = new HashSet<>(nestedProps.size()); + for (String nestedProp : nestedProps) { + if (jsonObject.get(nestedProp).isJsonObject()) { + String key = jsonObject.get(nestedProp).getAsJsonObject().entrySet().iterator().next().getKey(); + if (jsonObject.get(nestedProp).getAsJsonObject().get(key).getAsJsonArray().size() == 0) { + jsonObject.remove(nestedProp); + } else { + updatedNested.add(nestedProp); + } + } else if (jsonObject.get(nestedProp).isJsonArray()) { + if (jsonObject.get(nestedProp).getAsJsonArray().size() == 0) { + jsonObject.remove(nestedProp); + } else { + updatedNested.add(nestedProp); + } + } + } + + return updatedNested; + } + + /** + * Collects the nested json objects from storage + * @param jsonObject the object with nested to be collected + * @param nestedProps the properties with nested objs to be collected + * @return Collected json object + */ + private List<JsonObject> getNestedObjs(JsonObject jsonObject, Set<String> nestedProps, Map<String, JsonObject> nested) { + final List<JsonObject> objs = new ArrayList<>(); + + for (String nestedProp : nestedProps) { + List<String> uris = new ArrayList<>(); + if (jsonObject.get(nestedProp).isJsonObject()) { + String key = jsonObject.get(nestedProp).getAsJsonObject().entrySet().iterator().next().getKey(); + jsonObject.get(nestedProp).getAsJsonObject().get(key).getAsJsonArray() + .iterator().forEachRemaining(jsonElement -> uris.add(jsonElement.getAsString())); + } else if (jsonObject.get(nestedProp).isJsonArray()) { + jsonObject.get(nestedProp).getAsJsonArray() + .iterator().forEachRemaining(jsonElement -> uris.add(jsonElement.getAsString())); + } + + uris.stream().filter(nested::containsKey).map(nested::get).forEach(objs::add); + } + return objs; + } + + /** + * For the given JsonObject determine which properties have nested to be populated + * @param jsonObject to be scanned + * @return Set of property keys that have nested to be populated + */ + private Set<String> nestedPropsToBePopulated(JsonObject jsonObject) { + Set<String> props = jsonObject.entrySet() + .stream() + .filter(nestedEntry -> nestedEntry.getValue().isJsonObject() && + aaiResourcesUriTemplates.hasType(nestedEntry.getValue().getAsJsonObject().entrySet().iterator().next().getKey())) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + props.addAll(jsonObject.entrySet() + .stream() + .filter(nestedEntry -> nestedEntry.getValue().isJsonArray() && + aaiResourcesUriTemplates.hasType(nestedEntry.getKey())) + .map(Map.Entry::getKey) + .collect(Collectors.toSet())); + props.remove("relationship-list"); + return props; } } diff --git a/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/NonePayloadPrinterStrategy.java b/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/NonePayloadPrinterStrategy.java index 3aecfe8..52001c9 100644 --- a/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/NonePayloadPrinterStrategy.java +++ b/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/NonePayloadPrinterStrategy.java @@ -34,6 +34,7 @@ public class NonePayloadPrinterStrategy implements PayloadPrinterStrategy { @Override public JsonObject createJson(String collectionName, JsonArray jsonArray) { if (jsonArray != null && jsonArray.size() > 0) { + jsonArray.get(0).getAsJsonObject().remove("_id"); return jsonArray.get(0).getAsJsonObject(); } else { return null; diff --git a/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/PayloadPrinterType.java b/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/PayloadPrinterType.java index d83d622..b05bcab 100644 --- a/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/PayloadPrinterType.java +++ b/src/main/java/org/onap/aai/cacher/egestion/printer/strategy/PayloadPrinterType.java @@ -20,10 +20,19 @@ package org.onap.aai.cacher.egestion.printer.strategy; import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; public enum PayloadPrinterType { - NONE_PRINTER("none-printer"), AAI_RESOURCE_GET_ALL_PRINTER("aai-resource-get-all-printer"); + NONE_PRINTER("none-printer"), + AAI_RESOURCE_GET_ALL_PRINTER("aai-resource-get-all-printer"); + + private static final Map<String, PayloadPrinterType> MAP; + static { + MAP = Arrays.stream(values()).collect(Collectors.toMap(PayloadPrinterType::getValue, Function.identity())); + } private final String value; @@ -41,6 +50,6 @@ public enum PayloadPrinterType { } public static PayloadPrinterType fromString(String text) { - return Arrays.stream(values()).filter(bl -> bl.getValue().equalsIgnoreCase(text)).findFirst().orElse(null); + return MAP.get(text); } } diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/AAIResourceDmaapParserStrategy.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/AAIResourceDmaapParserStrategy.java deleted file mode 100644 index 6de0585..0000000 --- a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/AAIResourceDmaapParserStrategy.java +++ /dev/null @@ -1,470 +0,0 @@ -/** - * ============LICENSE_START======================================================= - * org.onap.aai - * ================================================================================ - * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END========================================================= - */ -package org.onap.aai.cacher.injestion.parser.strategy; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import org.apache.commons.lang3.StringUtils; -import org.onap.aai.cacher.injestion.parser.AAIResourcesUriTemplates; -import org.onap.aai.cacher.injestion.parser.PayloadParserService; -import org.onap.aai.cacher.model.CacheEntry; -import org.onap.aai.cacher.model.DBAction; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * AAI resource get all parser strategy - */ -@Component(value = "aai-resource-dmaap") -@Scope(scopeName = ConfigurableBeanFactory.SCOPE_SINGLETON) -public class AAIResourceDmaapParserStrategy implements PayloadParserStrategy { - - protected AAIResourcesUriTemplates aaiResourcesUriTemplates; - - private PayloadParserService payloadParserService; - - private DmaapAction actionType; - - @Autowired - public AAIResourceDmaapParserStrategy(AAIResourcesUriTemplates aaiResourcesUriTemplates, - PayloadParserService payloadParserService) { - this.aaiResourcesUriTemplates = aaiResourcesUriTemplates; - this.payloadParserService = payloadParserService; - } - - /** - * Parses aai resources specific payloads generating the details for caching. - * - * @param originalKey - * @param jsonObject - * @return - */ - @Override - public List<CacheEntry> process(String originalKey, JsonObject jsonObject) { - final List<CacheEntry> cacheEntries = new ArrayList<>(); - - JsonObject header = jsonObject.getAsJsonObject("event-header"); - JsonObject entity = jsonObject.getAsJsonObject("entity"); - String topEntity = header.get("top-entity-type").getAsString(); - - actionType = DmaapAction.valueOf(header.get("action").getAsString()); - boolean isTopLevel = topEntity.equals(header.get("entity-type").getAsString()); - String fullUri = getFullUri(header); - - CacheEntry cacheEntry = generateCacheEntry(entity, actionType, isTopLevel, fullUri); - - cacheEntries.add(cacheEntry); - - // determine relationships on the other end that need to be modified. - MultiValueMap<String, AAIResourceDmaapParserStrategy.AAIRelatedToDetails> relationships; - if (isTopLevel) { - relationships = getFromRelationshipFullUriToRelationshipObj(entity, fullUri); - } else { - relationships = getFromRelationshipFullUriToRelationshipObj(entity, getBaseUri(fullUri)); - } - if (jsonObject.has("existing-obj")) { - adjustRelationshipsBasedOnExisting(jsonObject, fullUri, relationships); - } - - JsonObject relatedToObj; - for (Map.Entry<String, List<AAIResourceDmaapParserStrategy.AAIRelatedToDetails>> relationship : relationships - .entrySet()) { - for (AAIResourceDmaapParserStrategy.AAIRelatedToDetails aaiRelatedToDetails : relationship.getValue()) { - relatedToObj = fullUriToRelationshipObj(relationship.getKey(), aaiRelatedToDetails.getLabel()); - cacheEntries.add(generateCacheEntry(relatedToObj, aaiRelatedToDetails.getActionType(), false, - aaiRelatedToDetails.getFullUri() + "/relationship-list/relationship/" - + aaiResourcesUriTemplates.encodeProp(relationship.getKey()))); - } - } - - return cacheEntries; - } - - private String getBaseUri(String fullUri) { - String uri = getUri(fullUri); - List<AAIUriSegment> uriSegmentList = getAaiUriSegments(uri); - return uriSegmentList.get(0).getSegment(); - } - - protected String getFullUriPrefix(String fullUri) { - return StringUtils.substring(fullUri, 0, StringUtils.ordinalIndexOf(fullUri, "/", 3)); - } - - protected CacheEntry generateCacheEntry(JsonObject entity, DmaapAction actionType, boolean isTopLevel, - String fullUri) { - String uri = getUri(fullUri); - List<AAIUriSegment> uriSegmentList = getAaiUriSegments(uri); - String id = uriSegmentList.get(0).getSegment(); - String collection = uriSegmentList.get(0).getSegmentSingular(); - JsonObject entityBody = getEntityBody(entity, uriSegmentList); - JsonObject findQuery = getFindQuery(uriSegmentList); - JsonObject nestedFindQuery = getNestedFindQuery(uriSegmentList); - String nestedField = getNestedField(uriSegmentList); - JsonObject nestedIdentifier = getNestedIdentifier(uriSegmentList); - DBAction dbAction = getDBAction(actionType); - - return CacheEntry.CacheEntryBuilder.createCacheEntry().inCollection(collection).withDbAction(dbAction) - .withId(id).isNested(!isTopLevel).withPayload(entityBody).withFindQuery(findQuery) - .withNestedFind(nestedFindQuery).withNestedField(nestedField) - .withNestedFieldIdentifierObj(nestedIdentifier).build(); - } - - protected DBAction getDBAction(DmaapAction actionType) { - DBAction dbAction = DBAction.INSERT_REPLACE; - switch (actionType) { - case CREATE: - dbAction = DBAction.INSERT_REPLACE; - break; - case DELETE: - dbAction = DBAction.DELETE; - break; - case UPDATE: - dbAction = DBAction.UPDATE; - break; - } - return dbAction; - } - - protected JsonObject getNestedIdentifier(List<AAIUriSegment> uriSegmentList) { - final JsonObject nestedIdentifier = new JsonObject(); - if (uriSegmentList.size() > 1) { - AAIUriSegment lastSegment = uriSegmentList.get(uriSegmentList.size() - 1); - lastSegment.getSegmentKeyValues().forEach(nestedIdentifier::addProperty); - } - return nestedIdentifier; - } - - protected String getNestedField(List<AAIUriSegment> uriSegmentList) { - StringBuilder nestedField = new StringBuilder(); - - if (uriSegmentList.size() > 1) { - if (uriSegmentList.get(1).getSegmentPlural().isPresent()) { - nestedField.append(uriSegmentList.get(1).getSegmentPlural().get()).append(".") - .append(uriSegmentList.get(1).getSegmentSingular()); - } else { - nestedField.append(uriSegmentList.get(1).getSegmentSingular()); - } - - for (int i = 2; i < uriSegmentList.size(); i++) { - if (uriSegmentList.get(i).getSegmentPlural().isPresent()) { - nestedField.append(".$.").append(uriSegmentList.get(i).getSegmentPlural().get()).append(".") - .append(uriSegmentList.get(i).getSegmentSingular()); - } else { - nestedField.append(".$.").append(uriSegmentList.get(i).getSegmentSingular()); - } - } - } - return nestedField.toString(); - } - - protected JsonObject getNestedFindQuery(List<AAIUriSegment> uriSegmentList) { - return getFindQuery(uriSegmentList, true); - } - - protected JsonObject getFindQuery(List<AAIUriSegment> uriSegmentList) { - return getFindQuery(uriSegmentList, false); - } - - protected JsonObject getFindQuery(List<AAIUriSegment> uriSegmentList, boolean isNested) { - final JsonObject findQuery = new JsonObject(); - if (uriSegmentList.isEmpty()) { - return findQuery; - } - - AAIUriSegment aaiUriSegment = uriSegmentList.get(0); - findQuery.addProperty("_id", aaiUriSegment.getSegment()); - aaiUriSegment.getSegmentKeyValues().forEach(findQuery::addProperty); - - StringBuilder nestedField = new StringBuilder(); - int segmentToProcess = uriSegmentList.size(); - if (!isNested) { - segmentToProcess--; - } - for (int i = 1; i < segmentToProcess; i++) { - aaiUriSegment = uriSegmentList.get(i); - if (nestedField.length() != 0) { - nestedField.append("."); - } - if (aaiUriSegment.getSegmentPlural().isPresent()) { - nestedField.append(aaiUriSegment.getSegmentPlural().get()).append("."); - } - nestedField.append(aaiUriSegment.getSegmentSingular()); - aaiUriSegment.getSegmentKeyValues() - .forEach((k, v) -> findQuery.addProperty(nestedField.toString() + "." + k, v)); - } - return findQuery; - } - - /** - * strips away the parent wrapping from the dmaap events entity payload - * - * @param entity - * @param uriSegmentList - * @return - */ - protected JsonObject getEntityBody(JsonObject entity, List<AAIUriSegment> uriSegmentList) { - - if (uriSegmentList.size() == 1) { - return entity; - } - - JsonObject entityBody = entity.getAsJsonObject(); - - // if processing relationship no need to look for nested obj, entity is the obj - if (!"relationship".equals(uriSegmentList.get(uriSegmentList.size() - 1).getSegmentSingular())) { - for (int i = 1; i < uriSegmentList.size(); i++) { - if (uriSegmentList.get(i).getSegmentPlural().isPresent()) { - entityBody = entityBody.getAsJsonObject(uriSegmentList.get(i).getSegmentPlural().get()) - .getAsJsonArray(uriSegmentList.get(i).getSegmentSingular()).get(0).getAsJsonObject(); - } else { - entityBody = entityBody.getAsJsonArray(uriSegmentList.get(i).getSegmentSingular()).get(0) - .getAsJsonObject(); - } - - } - } - - return entityBody; - - } - - protected List<AAIUriSegment> getAaiUriSegments(String uri) { - List<String> uriSegmentTemplates = aaiResourcesUriTemplates.uriToTemplates(uri); - List<String> uriSegments = aaiResourcesUriTemplates.uriToSegments(uri); - - List<AAIUriSegment> uriSegmentList = new ArrayList<>(uriSegments.size()); - - AAIUriSegment aus; - for (int i = 0; i < uriSegments.size(); i++) { - aus = new AAIUriSegment(uriSegments.get(i), uriSegmentTemplates.get(i)); - aus.setSegmentKeyValues( - aaiResourcesUriTemplates.getUriTemplateMappings(aus.getSegment(), aus.getSegmentTemplate())); - uriSegmentList.add(aus); - } - return uriSegmentList; - } - - /** - * For update events with an existing obj available adjust the cache actions to - * be taken on relationship objects. - * - * @param jsonObject - * @param fullUri - * @param newObjectRelationships - */ - private void adjustRelationshipsBasedOnExisting(JsonObject jsonObject, String fullUri, - MultiValueMap<String, AAIResourceDmaapParserStrategy.AAIRelatedToDetails> newObjectRelationships) { - JsonObject existingObj = jsonObject.getAsJsonObject("existing-obj"); - MultiValueMap<String, AAIResourceDmaapParserStrategy.AAIRelatedToDetails> oldRelationships = getFromRelationshipFullUriToRelationshipObj( - existingObj, fullUri); - oldRelationships.forEach((k, v) -> { - if (newObjectRelationships.containsKey(k)) { - v.forEach(oldA -> { - int found = -1; - for (int i = 0; i < newObjectRelationships.get(k).size(); i++) { - if (newObjectRelationships.get(k).get(i).getFullUri().equals(oldA.getFullUri())) { - found = i; - break; - } - } - if (found != -1) { - newObjectRelationships.get(k).remove(newObjectRelationships.get(k).get(found)); - } else { - oldA.setActionType(DmaapAction.DELETE); - newObjectRelationships.get(k).add(oldA); - } - }); - } else { - v.forEach(aaiRelatedToDetails -> { - aaiRelatedToDetails.setActionType(DmaapAction.DELETE); - newObjectRelationships.add(k, aaiRelatedToDetails); - }); - } - }); - } - - /** - * Given fullUri uri generate an aai relationship obj - * - * @param fullUri - * @return - */ - protected JsonObject fullUriToRelationshipObj(String fullUri, String label) { - final JsonObject relObj = new JsonObject(); - final JsonArray relData = new JsonArray(); - String uri = getUri(fullUri); - List<AAIUriSegment> uriSegmentList = getAaiUriSegments(uri); - - relObj.addProperty("related-to", uriSegmentList.get(uriSegmentList.size() - 1).getSegmentSingular()); - if (label != null) { - relObj.addProperty("relationship-label", label); - } - relObj.addProperty("related-link", fullUri); - - for (AAIUriSegment aaiUriSegment : uriSegmentList) { - aaiUriSegment.getSegmentKeyValues().forEach((k, v) -> { - JsonObject relDataEntry; - relDataEntry = new JsonObject(); - relDataEntry.addProperty("relationship-key", aaiUriSegment.getSegmentSingular() + "." + k); - relDataEntry.addProperty("relationship-value", v); - relData.add(relDataEntry); - }); - } - relObj.add("relationship-data", relData); - - return relObj; - } - - /** - * - * @param entity - * @param fullUri - * @return - */ - protected MultiValueMap<String, AAIResourceDmaapParserStrategy.AAIRelatedToDetails> getFromRelationshipFullUriToRelationshipObj( - JsonObject entity, String fullUri) { - final MultiValueMap<String, AAIResourceDmaapParserStrategy.AAIRelatedToDetails> relationshipMapping = new LinkedMultiValueMap<>(); - for (Map.Entry<String, JsonElement> e : entity.entrySet()) { - if (e.getKey().equals("relationship-list") && e.getValue().isJsonObject()) { - JsonArray relationships = e.getValue().getAsJsonObject().getAsJsonArray("relationship"); - for (JsonElement relationship : relationships) { - relationshipMapping.add(fullUri, new AAIResourceDmaapParserStrategy.AAIRelatedToDetails( - relationship.getAsJsonObject().get("related-link").getAsString(), - relationship.getAsJsonObject().get("relationship-label").getAsString(), actionType)); - } - } else if (e.getValue().isJsonObject() && e.getValue().getAsJsonObject().entrySet().size() == 1) { - Map.Entry<String, JsonElement> entry = e.getValue().getAsJsonObject().entrySet().iterator().next(); - if (entry.getValue().isJsonArray()) { - String type = entry.getKey(); - JsonArray children = entry.getValue().getAsJsonArray(); - for (JsonElement child : children) { - relationshipMapping.putAll(getFromRelationshipFullUriToRelationshipObj(child.getAsJsonObject(), - fullUri + aaiResourcesUriTemplates.getUri(type, child.getAsJsonObject()))); - } - } - } - } - return relationshipMapping; - } - - protected String getUri(String fullUri) { - return fullUri.replaceAll("/aai/v\\d+", ""); - } - - protected String getFullUri(JsonObject header) { - return header.get("entity-link").getAsString(); - } - - protected enum DmaapAction { - DELETE, UPDATE, CREATE - } - - class AAIUriSegment { - - private String segment; - private String segmentTemplate; - private Optional<String> segmentPlural = Optional.empty(); - private String segmentSingular; - private Map<String, String> segmentKeyValues; - - AAIUriSegment(String segment, String template) { - this.segment = segment; - this.segmentTemplate = template; - String[] segmentSplit = segment.split("/"); - String[] templateSplit = template.split("/"); - for (int i = 0; i < templateSplit.length; i++) { - if (templateSplit[i].contains("{")) { - segmentSingular = segmentSplit[i - 1]; - if (!"".equals(segmentSplit[i - 2])) { - segmentPlural = Optional.of(segmentSplit[i - 2]); - } - break; - } - } - } - - String getSegment() { - return segment; - } - - String getSegmentTemplate() { - return segmentTemplate; - } - - Map<String, String> getSegmentKeyValues() { - return segmentKeyValues; - } - - void setSegmentKeyValues(Map<String, String> segmentKeyValues) { - this.segmentKeyValues = segmentKeyValues; - } - - Optional<String> getSegmentPlural() { - return segmentPlural; - } - - String getSegmentSingular() { - return segmentSingular; - } - } - - class AAIRelatedToDetails { - private String fullUri; - private String label; - private DmaapAction actionType; - - public AAIRelatedToDetails(String fullUri, String label, DmaapAction actionType) { - this.fullUri = fullUri; - this.label = label; - this.actionType = actionType; - } - - public String getFullUri() { - return fullUri; - } - - public String getLabel() { - return label; - } - - public DmaapAction getActionType() { - return actionType; - } - - public void setActionType(DmaapAction actionType) { - this.actionType = actionType; - } - - @Override - public String toString() { - return fullUri + " : " + label; - } - } -} diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/AAIResourceGetAllPayloadParserStrategy.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/AAIResourceGetAllPayloadParserStrategy.java index 1c2f6cf..c03af62 100644 --- a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/AAIResourceGetAllPayloadParserStrategy.java +++ b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/AAIResourceGetAllPayloadParserStrategy.java @@ -17,12 +17,13 @@ * limitations under the License. * ============LICENSE_END========================================================= */ + package org.onap.aai.cacher.injestion.parser.strategy; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import org.onap.aai.cacher.injestion.parser.AAIResourcesUriTemplates; +import org.onap.aai.cacher.injestion.parser.strategy.aai.AAIResourcesUriTemplates; import org.onap.aai.cacher.model.CacheEntry; import org.onap.aai.cacher.model.DBAction; import org.springframework.beans.factory.annotation.Autowired; @@ -30,56 +31,177 @@ import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; /** - * AAI resource get all parser strategy + * This parser strategy breaks down the all nested + * aai objects to be stored separately */ -@Component(value = "aai-resource-get-all") +@Component(value="aai-resource-get-all") @Scope(scopeName = ConfigurableBeanFactory.SCOPE_SINGLETON) public class AAIResourceGetAllPayloadParserStrategy implements PayloadParserStrategy { - AAIResourcesUriTemplates aaiResourcesUriTemplates; - - @Autowired - public AAIResourceGetAllPayloadParserStrategy(AAIResourcesUriTemplates aaiResourcesUriTemplates) { - this.aaiResourcesUriTemplates = aaiResourcesUriTemplates; - } - - /** - * Parses aai resources specific payloads generating the details for . - * - * @param originalKey - * @param jsonObject - * @return - */ - @Override - public List<CacheEntry> process(String originalKey, JsonObject jsonObject) { - final List<CacheEntry> cacheEntries = new ArrayList<>(); - - String type = jsonObject.entrySet().iterator().next().getKey(); - - JsonArray ja = jsonObject.getAsJsonArray(type); - CacheEntry cacheEntry; - String uri; - JsonObject jo; - for (JsonElement jsonElement : ja) { - jo = jsonElement.getAsJsonObject(); - uri = aaiResourcesUriTemplates.getUri(type, jo); - jsonObject.addProperty("_id", uri); - cacheEntry = CacheEntry.CacheEntryBuilder.createCacheEntry().withId(uri).inCollection(originalKey) - .withFindQuery(getFindQuery(uri)).withPayload(jo).withDbAction(DBAction.INSERT_REPLACE).build(); - cacheEntries.add(cacheEntry); - } - - return cacheEntries; - } - - protected JsonObject getFindQuery(String uri) { - JsonObject jo = new JsonObject(); - jo.addProperty("_id", uri); - return jo; - } + private static final String RELATIONSHIP = "relationship"; + + protected AAIResourcesUriTemplates aaiResourcesUriTemplates; + + @Autowired + public AAIResourceGetAllPayloadParserStrategy(AAIResourcesUriTemplates aaiResourcesUriTemplates) { + this.aaiResourcesUriTemplates = aaiResourcesUriTemplates; + } + + + /** + * Parses aai resources specific payloads generating the details for its cache entries. + * @param originalKey cache key for this entry + * @param jsonObject object to be broken down via this strategy + * @return List of cache entries to be stored + */ + @Override + public List<CacheEntry> process(String originalKey, JsonObject jsonObject) { + return internalProcess(originalKey, jsonObject, ""); + } + + protected List<CacheEntry> internalProcess(String originalKey, JsonObject jsonObject, String baseUri) { + final List<CacheEntry> cacheEntries = getCacheEntries(originalKey, baseUri, jsonObject); + List<CacheEntry> nestedCacheEntries; + Set<String> nestedProperties; + + boolean foundNested = true; + int start = 0; + while (foundNested) { + nestedCacheEntries = new ArrayList<>(); + for (int i = start; i < cacheEntries.size(); i++) { + nestedProperties = nestedPropsToBePopulated(cacheEntries.get(i).getPayload()); + if (!nestedProperties.isEmpty()) { + nestedCacheEntries.addAll(processEntriesWithNested(originalKey, cacheEntries.get(i), nestedProperties)); + } + } + if (nestedCacheEntries.isEmpty()) { + foundNested = false; + } else { + start = cacheEntries.size(); + cacheEntries.addAll(nestedCacheEntries); + foundNested = true; + } + } + + cacheEntries.forEach(cacheEntry -> + cacheEntry.getPayload().addProperty("_id", cacheEntry.getId())); + return cacheEntries; + } + + private Set<String> nestedPropsToBePopulated(JsonObject jsonObject) { + Set<String> props = jsonObject.entrySet() + .stream() + .filter(nestedEntry -> nestedEntry.getValue().isJsonObject() && + aaiResourcesUriTemplates.hasType(nestedEntry.getValue().getAsJsonObject().entrySet().iterator().next().getKey())) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + props.addAll(jsonObject.entrySet() + .stream() + .filter(nestedEntry -> nestedEntry.getValue().isJsonArray() && + aaiResourcesUriTemplates.hasType(nestedEntry.getKey())) + .map(Map.Entry::getKey) + .collect(Collectors.toSet())); + props.remove("relationship-list"); + return props; + } + + + /** + * Replaces the nested object with the id to the object and generates a cache entry per object + * @param originalKey top node type + * @param cacheEntry with nested aai object + * @param nestedProperties the keys that contain nested properties + * @return List of CacheEntry for each nested json object + */ + private List<CacheEntry> processEntriesWithNested(String originalKey, CacheEntry cacheEntry, Set<String> nestedProperties) { + final List<CacheEntry> cacheEntries = new ArrayList<>(); + + for (String nestedProperty : nestedProperties) { + List<CacheEntry> nestedCacheEntries = new ArrayList<>(); + if (cacheEntry.getPayload().get(nestedProperty).isJsonObject()) { + String type = cacheEntry.getPayload().get(nestedProperty).getAsJsonObject().entrySet().iterator().next().getKey(); + nestedCacheEntries = + getCacheEntries(originalKey, cacheEntry.getId(), cacheEntry.getPayload().get(nestedProperty).getAsJsonObject()); + JsonArray nestedElementsIds = new JsonArray(); + nestedCacheEntries.forEach(nestedCacheEntry -> nestedElementsIds.add(nestedCacheEntry.getId())); + cacheEntry.getPayload().get(nestedProperty).getAsJsonObject().add(type, nestedElementsIds); + } else if (cacheEntry.getPayload().get(nestedProperty).isJsonArray()) { + nestedCacheEntries = + getCacheEntries(originalKey, cacheEntry.getId(), nestedProperty, cacheEntry.getPayload().get(nestedProperty).getAsJsonArray()); + JsonArray nestedElementsIds = new JsonArray(); + nestedCacheEntries.forEach(nestedCacheEntry -> nestedElementsIds.add(nestedCacheEntry.getId())); + cacheEntry.getPayload().add(nestedProperty, nestedElementsIds); + } + cacheEntries.addAll(nestedCacheEntries); + } + + return cacheEntries; + } + + + /** + * Generates the CacheEntries for all of the nested objects in the JsonObject + * @param originalKey the aai-node-type + * @param baseUri the base "parent" url used to populate full uri + * @param jsonObject the object to be scanned + * @return List of CacheEntries for all of the nested objects in the payload + */ + protected List<CacheEntry> getCacheEntries(String originalKey, String baseUri, JsonObject jsonObject) { + String type = jsonObject.entrySet().iterator().next().getKey(); + JsonArray ja = jsonObject.getAsJsonArray(type); + return getCacheEntries(originalKey, baseUri, type, ja); + } + + /** + * Generates the CacheEntries for all of the nested objects in the JsonArray + * @param originalKey the top aai-node-type + * @param baseUri the base "parent" url used to populate full uri + * @param ja the JsonArray to be scanned + * @return List of CacheEntries for all of the nested objects in the payload + */ + private List<CacheEntry> getCacheEntries(String originalKey, String baseUri, String type, JsonArray ja) { + final List<CacheEntry> cacheEntries = new ArrayList<>(ja.size()); + final Set<String> uris = new HashSet<>(ja.size()); + for (JsonElement jsonElement : ja) { + String uri; + JsonObject jo = jsonElement.getAsJsonObject(); + if (RELATIONSHIP.equals(type)) { + uri = jo.get("related-link").getAsString().replaceAll("/aai/v\\d+", ""); + } else { + uri = baseUri + aaiResourcesUriTemplates.getUri(type, jo); + } + // checks for nested dupes and does not create a cache entry for them + if (uris.contains(uri)) { + continue; + } else { + uris.add(uri); + } + CacheEntry cacheEntry = CacheEntry.CacheEntryBuilder.createCacheEntry() + .withId(uri) + .inCollection(originalKey) + .withFindQuery(getFindQuery(uri)) + .withPayload(jo) + .withDbAction(DBAction.INSERT_REPLACE) + .build(); + cacheEntries.add(cacheEntry); + } + + return cacheEntries; + } + + /** + * Creates a find query payload + * @param uri used as the _id field for lookup + * @return the find query JsonObject + */ + private JsonObject getFindQuery(String uri) { + JsonObject jo = new JsonObject(); + jo.addProperty("_id", uri); + return jo; + } } diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/PayloadParserType.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/PayloadParserType.java index b497a8c..0e46b2f 100644 --- a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/PayloadParserType.java +++ b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/PayloadParserType.java @@ -20,26 +20,38 @@ package org.onap.aai.cacher.injestion.parser.strategy; import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; public enum PayloadParserType { - NONE("none"), AAI_RESOURCE_GET_ALL("aai-resource-get-all"), AAI_RESOURCE_DMAAP("aai-resource-dmaap"); - private final String value; + NONE("none"), + AAI_RESOURCE_GET_ALL("aai-resource-get-all"), + AAI_RESOURCE_DMAAP("aai-resource-dmaap"); - PayloadParserType(String input) { - this.value = input; - } + private static final Map<String, PayloadParserType> MAP; + static { + MAP = Arrays.stream(values()).collect(Collectors.toMap(PayloadParserType::getValue, Function.identity())); + } - public String getValue() { - return this.value; - } + private final String value; - @Override - public String toString() { - return this.value; - } + PayloadParserType(String input) { + this.value = input; + } - public static PayloadParserType fromString(String text) { - return Arrays.stream(values()).filter(bl -> bl.getValue().equalsIgnoreCase(text)).findFirst().orElse(null); - } + public String getValue() { + return this.value; + } + + @Override + public String toString() { + return this.value; + } + + public static PayloadParserType fromString(String text) { + return MAP.get(text); + } } + diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/AAIResourcesUriTemplates.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/AAIResourcesUriTemplates.java index 0885851..887e4d3 100644 --- a/src/main/java/org/onap/aai/cacher/injestion/parser/AAIResourcesUriTemplates.java +++ b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/AAIResourcesUriTemplates.java @@ -17,11 +17,15 @@ * limitations under the License. * ============LICENSE_END========================================================= */ -package org.onap.aai.cacher.injestion.parser; +package org.onap.aai.cacher.injestion.parser.strategy.aai; +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; import com.google.gson.JsonObject; import org.apache.commons.lang3.StringUtils; +import org.onap.aai.annotations.Metadata; import org.onap.aai.cacher.util.AAIConstants; +import org.reflections.Reflections; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -29,6 +33,7 @@ import org.springframework.web.util.UriTemplate; import org.springframework.web.util.UriUtils; import javax.ws.rs.core.UriBuilder; +import javax.xml.bind.annotation.XmlRootElement; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -39,6 +44,8 @@ import java.util.*; @Scope(scopeName = ConfigurableBeanFactory.SCOPE_SINGLETON) public class AAIResourcesUriTemplates { + private final static EELFLogger LOGGER = EELFManager.getInstance().getLogger(AAIResourcesUriTemplates.class); + private final Map<String, String> typeToUriTemplate; public AAIResourcesUriTemplates() throws IOException { @@ -53,6 +60,22 @@ public class AAIResourcesUriTemplates { typeToUriTemplate.put("relationship", "/relationship-list/relationship/{related-link}"); } } + + Reflections reflections = new Reflections("org.onap.aai.domain.yang"); + reflections.getTypesAnnotatedWith(Metadata.class) + .stream() + .filter(aClass -> "org.onap.aai.domain.yang".equals(aClass.getPackage().getName())) + .filter(aClass -> !aClass.getAnnotation(Metadata.class).uriTemplate().isEmpty()) + .forEach(aClass -> typeToUriTemplate.put( + aClass.getAnnotation(XmlRootElement.class).name(), + aClass.getAnnotation(Metadata.class).uriTemplate()) + ); + + LOGGER.info("AAI uri templates: " + typeToUriTemplate); + } + + public boolean hasType(String type) { + return this.typeToUriTemplate.containsKey(type); } /** @@ -65,6 +88,12 @@ public class AAIResourcesUriTemplates { return typeToUriTemplate.get(type); } + /** + * For the given template and uri get the variable key value pairs + * @param uri + * @param template + * @return + */ public Map<String, String> getUriTemplateMappings(String uri, String template) { UriTemplate uriTemplate = new UriTemplate(template); @@ -85,13 +114,20 @@ public class AAIResourcesUriTemplates { List<String> uriTemplateList = new ArrayList<>(); String template = ""; String truncatedUri = uri; + Optional<String> matchingStartingTemplate; while (truncatedUri.contains("/")) { - template = this.getMatchingStartingTemplate(truncatedUri).get(); + matchingStartingTemplate = this.getMatchingStartingTemplate(truncatedUri); + if ( !matchingStartingTemplate.isPresent()) { + LOGGER.error("failed in uriToTemplates for truncatedUri " + truncatedUri); + // exception expected for missing template + } + template = matchingStartingTemplate.get(); uriTemplateList.add(template); int count = StringUtils.countMatches(template, "/"); if (count < StringUtils.countMatches(truncatedUri, "/")) { - truncatedUri = StringUtils.substring(truncatedUri, + truncatedUri = StringUtils.substring( + truncatedUri, StringUtils.ordinalIndexOf(truncatedUri, "/", count + 1)); } else { truncatedUri = ""; @@ -205,4 +241,28 @@ public class AAIResourcesUriTemplates { return ""; } } + + public String getAAIUriFromEntityUri(String fullUri) { + return fullUri.replaceAll("/aai/v\\d+", ""); + } + + public String getAAIUriFromEntityUriPrefix(String fullUri) { + return StringUtils.substring(fullUri, 0, StringUtils.ordinalIndexOf(fullUri, "/", 3)); + } + + public List<AAIUriSegment> getAaiUriSegments(String uri) { + List<String> uriSegmentTemplates = uriToTemplates(uri); + List<String> uriSegments = uriToSegments(uri); + + List<AAIUriSegment> uriSegmentList = new ArrayList<>(uriSegments.size()); + + AAIUriSegment aus; + for (int i = 0; i < uriSegments.size(); i++) { + aus = new AAIUriSegment(uriSegments.get(i), uriSegmentTemplates.get(i)); + aus.setSegmentKeyValues( + getUriTemplateMappings(aus.getSegment(), aus.getSegmentTemplate())); + uriSegmentList.add(aus); + } + return uriSegmentList; + } }
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/AAIUriSegment.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/AAIUriSegment.java new file mode 100644 index 0000000..7624a91 --- /dev/null +++ b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/AAIUriSegment.java @@ -0,0 +1,73 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.cacher.injestion.parser.strategy.aai; + +import java.util.Map; +import java.util.Optional; + +public class AAIUriSegment { + + private String segment; + private String segmentTemplate; + private Optional<String> segmentPlural = Optional.empty(); + private String segmentSingular; + private Map<String, String> segmentKeyValues; + + public AAIUriSegment(String segment, String template) { + this.segment = segment; + this.segmentTemplate = template; + String[] segmentSplit = segment.split("/"); + String[] templateSplit = template.split("/"); + for (int i = 0; i < templateSplit.length; i++) { + if (templateSplit[i].contains("{")) { + segmentSingular = segmentSplit[i - 1]; + if (!"".equals(segmentSplit[i - 2])) { + segmentPlural = Optional.of(segmentSplit[i - 2]); + } + break; + } + } + } + + public String getSegment() { + return segment; + } + + public String getSegmentTemplate() { + return segmentTemplate; + } + + public Map<String, String> getSegmentKeyValues() { + return segmentKeyValues; + } + + public void setSegmentKeyValues(Map<String, String> segmentKeyValues) { + this.segmentKeyValues = segmentKeyValues; + } + + public Optional<String> getSegmentPlural() { + return segmentPlural; + } + + public String getSegmentSingular() { + return segmentSingular; + } +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/AAIRelatedToDetails.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/AAIRelatedToDetails.java new file mode 100644 index 0000000..cd18ee8 --- /dev/null +++ b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/AAIRelatedToDetails.java @@ -0,0 +1,54 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.cacher.injestion.parser.strategy.aai.dmaap; + +public class AAIRelatedToDetails { + private String fullUri; + private String label; + private DmaapAction actionType; + + public AAIRelatedToDetails(String fullUri, String label, DmaapAction actionType) { + this.fullUri = fullUri; + this.label = label; + this.actionType = actionType; + } + + public String getFullUri() { + return fullUri; + } + + public String getLabel() { + return label; + } + + public DmaapAction getActionType() { + return actionType; + } + + public void setActionType(DmaapAction actionType) { + this.actionType = actionType; + } + + @Override + public String toString() { + return fullUri + " : " + label; + } +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/AAIResourceDmaapParserStrategy.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/AAIResourceDmaapParserStrategy.java new file mode 100644 index 0000000..8d88a4a --- /dev/null +++ b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/AAIResourceDmaapParserStrategy.java @@ -0,0 +1,327 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.cacher.injestion.parser.strategy.aai.dmaap; + +import com.att.eelf.configuration.EELFLogger; +import com.att.eelf.configuration.EELFManager; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.onap.aai.cacher.injestion.parser.strategy.AAIResourceGetAllPayloadParserStrategy; +import org.onap.aai.cacher.injestion.parser.strategy.aai.AAIResourcesUriTemplates; +import org.onap.aai.cacher.injestion.parser.strategy.aai.AAIUriSegment; +import org.onap.aai.cacher.model.CacheEntry; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * AAI resource get all parser strategy + */ +@Component(value = "aai-resource-dmaap") +@Scope(scopeName = ConfigurableBeanFactory.SCOPE_SINGLETON) +public class AAIResourceDmaapParserStrategy extends AAIResourceGetAllPayloadParserStrategy { + + private final static EELFLogger LOGGER = EELFManager.getInstance().getLogger(AAIResourceDmaapParserStrategy.class); + + @Autowired + public AAIResourceDmaapParserStrategy(AAIResourcesUriTemplates aaiResourcesUriTemplates) { + super(aaiResourcesUriTemplates); + } + + /** + * Parses aai resources specific payloads generating the details for caching. + * + * @param originalKey + * @param jsonObject + * @return + */ + @Override + public List<CacheEntry> process(String originalKey, JsonObject jsonObject) { + final List<CacheEntry> cacheEntries; + + JsonObject header = jsonObject.getAsJsonObject("event-header"); + String topEntityType = header.get("top-entity-type").getAsString(); + String fullUri = header.get("entity-link").getAsString(); + if ( fullUri.endsWith("/")) { + fullUri = fullUri.substring(0, fullUri.length() - 1); + } + String entityUri = aaiResourcesUriTemplates.getAAIUriFromEntityUri(fullUri); + String fullUriPrefix = aaiResourcesUriTemplates.getAAIUriFromEntityUriPrefix(fullUri); + + DmaapAction actionType = DmaapAction.valueOf(header.get("action").getAsString()); + + List<AAIUriSegment> uriSegments = aaiResourcesUriTemplates.getAaiUriSegments(entityUri); + + // get base uri so if is top lvl use "" else go 1 back from current node type. to reuse exist functions. + String baseUri = getBaseUri(uriSegments); + + // get wrapped wrap obj it looks like a get-all result with 1 + // will wrap address lists even though they do not normally have a wrapper + JsonObject wrappedEntityObject = getWrappedEntityObject(jsonObject.getAsJsonObject("entity"), uriSegments); + + // get cache entries + cacheEntries = internalProcess(topEntityType, wrappedEntityObject, baseUri); + + // modify action to map to dmaap event type + cacheEntries.forEach(cacheEntry -> cacheEntry.setDbAction(actionType.getDbAction())); + + // cache entries for relationships + MultiValueMap<String, AAIRelatedToDetails> cacheEntriesRelationships = + getFromRelationshipFullUriToRelationshipObj(cacheEntries, fullUriPrefix, actionType); + + if (jsonObject.has("existing-obj")) { + MultiValueMap<String, AAIRelatedToDetails> existingCacheEntriesRelationships = + getFromRelationshipFullUriToRelationshipObj( + internalProcess(topEntityType, jsonObject.getAsJsonObject("existing-obj"), baseUri), + fullUriPrefix, + actionType + ); + adjustRelationshipsBasedOnExisting(existingCacheEntriesRelationships, cacheEntriesRelationships); + + } + + cacheEntries.addAll(getRelationshipCacheEntries(cacheEntriesRelationships)); + + getParentUpdateCacheEntryIfNeeded(topEntityType, entityUri, actionType, uriSegments).ifPresent(cacheEntries::add); + + return cacheEntries; + } + + private Optional<CacheEntry> getParentUpdateCacheEntryIfNeeded(String topEntityType, String entityUri, DmaapAction actionType, List<AAIUriSegment> uriSegments) { + + if (uriSegments.size() <= 1) { + return Optional.empty(); + } + + switch (actionType) { + case DELETE: + return Optional.of(getParentUpdateCacheEntry(topEntityType, entityUri, actionType, uriSegments)); + case CREATE: + return Optional.of(getParentUpdateCacheEntry(topEntityType, entityUri, DmaapAction.UPDATE, uriSegments)); + default: + return Optional.empty(); + } + } + + private CacheEntry getParentUpdateCacheEntry(String topEntityType, String entityUri, DmaapAction actionType, List<AAIUriSegment> uriSegments) { + String parentUri = String.join( + "", + uriSegments.stream().limit(uriSegments.size()-1).map(AAIUriSegment::getSegment).collect(Collectors.toList())); + JsonObject findQuery = new JsonObject(); + findQuery.addProperty("_id", parentUri); + JsonObject nestedFindQuery = new JsonObject(); + nestedFindQuery.addProperty("_id", parentUri); + StringBuilder nestedField = new StringBuilder(); + uriSegments.get(uriSegments.size()-1).getSegmentPlural().ifPresent(plural -> nestedField.append(plural).append(".")); + nestedField.append(uriSegments.get(uriSegments.size()-1).getSegmentSingular()); + JsonObject nestedIdentifier = new JsonObject(); + JsonArray ja = new JsonArray(); + ja.add(entityUri); + nestedIdentifier.add("$in", ja); + return CacheEntry.CacheEntryBuilder.createCacheEntry() + .inCollection(topEntityType) + .withDbAction(actionType.getDbAction()) + .withId(parentUri) + .isNested(true) + .isNestedPayloadString(true) + .withNestedString(entityUri) + .withFindQuery(findQuery) + .withNestedFind(nestedFindQuery) + .withNestedField(nestedField.toString()) + .withNestedFieldIdentifierObj(nestedIdentifier) + .build(); + } + + private List<CacheEntry> getRelationshipCacheEntries(MultiValueMap<String, AAIRelatedToDetails> cacheEntriesRelationships) { + final List<CacheEntry> cacheEntries = new ArrayList<>(); + JsonObject relatedToObj; + for (Map.Entry<String, List<AAIRelatedToDetails>> relationship : cacheEntriesRelationships.entrySet()) { + for (AAIRelatedToDetails aaiRelatedToDetails : relationship.getValue()) { + relatedToObj = fullUriToRelationshipObj(relationship.getKey(), aaiRelatedToDetails.getLabel()); + cacheEntries.add(generateRelationshipCacheEntry(relatedToObj, aaiRelatedToDetails.getActionType(), + aaiRelatedToDetails.getFullUri())); + } + } + return cacheEntries; + } + + private CacheEntry generateRelationshipCacheEntry(JsonObject entity, DmaapAction actionType, String fullUri) { + + String uri = aaiResourcesUriTemplates.getAAIUriFromEntityUri(fullUri); + List<AAIUriSegment> uriSegmentList = aaiResourcesUriTemplates.getAaiUriSegments(uri); + String collection = uriSegmentList.get(0).getSegmentSingular(); + JsonObject findQuery = new JsonObject(); + findQuery.addProperty("_id", uri); + JsonObject nestedFindQuery = new JsonObject(); + nestedFindQuery.addProperty("_id", uri); + nestedFindQuery.addProperty("relationship-list.relationship.related-link", entity.get("related-link").getAsString()); + String nestedField = "relationship-list.relationship"; + JsonObject nestedIdentifier = new JsonObject(); + nestedIdentifier.addProperty("related-link", entity.get("related-link").getAsString()); + + return CacheEntry.CacheEntryBuilder.createCacheEntry().inCollection(collection).withDbAction(actionType.getDbAction()) + .withId(uri).isNested(true).withPayload(entity).withFindQuery(findQuery) + .withNestedFind(nestedFindQuery).withNestedField(nestedField) + .withNestedFieldIdentifierObj(nestedIdentifier).build(); + } + + private void adjustRelationshipsBasedOnExisting(MultiValueMap<String, AAIRelatedToDetails> existingCacheEntriesRelationships, + MultiValueMap<String, AAIRelatedToDetails> cacheEntriesRelationships) { + existingCacheEntriesRelationships.forEach((k, v) -> { + if (cacheEntriesRelationships.containsKey(k)) { + v.forEach(oldA -> { + int found = -1; + for (int i = 0; i < cacheEntriesRelationships.get(k).size(); i++) { + if (cacheEntriesRelationships.get(k).get(i).getFullUri().equals(oldA.getFullUri())) { + found = i; + break; + } + } + if (found != -1) { + cacheEntriesRelationships.get(k).remove(cacheEntriesRelationships.get(k).get(found)); + } else { + oldA.setActionType(DmaapAction.DELETE); + cacheEntriesRelationships.get(k).add(oldA); + } + }); + } else { + v.forEach(aaiRelatedToDetails -> { + aaiRelatedToDetails.setActionType(DmaapAction.DELETE); + cacheEntriesRelationships.add(k, aaiRelatedToDetails); + }); + } + }); + } + + private MultiValueMap<String, AAIRelatedToDetails> getFromRelationshipFullUriToRelationshipObj( + List<CacheEntry> cacheEntries, + String fullUriPrefix, + DmaapAction actionType) { + final MultiValueMap<String, AAIRelatedToDetails> relationshipMapping = new LinkedMultiValueMap<>(); + for (CacheEntry cacheEntry : cacheEntries) { + for (Map.Entry<String, JsonElement> e : cacheEntry.getPayload().entrySet()) { + if (e.getKey().equals("relationship-list") && e.getValue().isJsonObject()) { + JsonArray relationships = e.getValue().getAsJsonObject().getAsJsonArray("relationship"); + for (JsonElement relationship : relationships) { + relationshipMapping.add(fullUriPrefix + cacheEntry.getId(), new AAIRelatedToDetails( + relationship.getAsJsonObject().get("related-link").getAsString(), + relationship.getAsJsonObject().get("relationship-label").getAsString(), actionType)); + } + } + } + } + return relationshipMapping; + } + + /** + * Given fullUri uri generate an aai relationship obj + * + * @param fullUri + * @return + */ + protected JsonObject fullUriToRelationshipObj(String fullUri, String label) { + final JsonObject relObj = new JsonObject(); + final JsonArray relData = new JsonArray(); + String uri = aaiResourcesUriTemplates.getAAIUriFromEntityUri(fullUri); + List<AAIUriSegment> uriSegmentList = aaiResourcesUriTemplates.getAaiUriSegments(uri); + + relObj.addProperty("related-to", uriSegmentList.get(uriSegmentList.size() - 1).getSegmentSingular()); + if (label != null) { + relObj.addProperty("relationship-label", label); + } + relObj.addProperty("related-link", fullUri); + + for (AAIUriSegment aaiUriSegment : uriSegmentList) { + aaiUriSegment.getSegmentKeyValues().forEach((k, v) -> { + JsonObject relDataEntry; + relDataEntry = new JsonObject(); + relDataEntry.addProperty("relationship-key", aaiUriSegment.getSegmentSingular() + "." + k); + relDataEntry.addProperty("relationship-value", v); + relData.add(relDataEntry); + }); + } + relObj.add("relationship-data", relData); + + return relObj; + } + + private JsonObject getWrappedEntityObject(JsonObject dmaapEntity, List<AAIUriSegment> uriSegments) { + JsonObject objectWrapper = new JsonObject(); + JsonArray arrayWrapper = new JsonArray(); + String arrayKey = uriSegments.get(0).getSegmentSingular(); + JsonObject entityBody = dmaapEntity.getAsJsonObject(); + + if (uriSegments.size() > 1) { + Optional<String> segmentPlural; + String segmentSingular; + String jsonStr; + JsonObject jsonObj; + JsonArray jsonArray; + JsonElement jsonElement; + + for (int i = 1; i < uriSegments.size(); i++) { + + if (uriSegments.get(i).getSegmentPlural().isPresent()) { + segmentPlural = uriSegments.get(i).getSegmentPlural(); + segmentSingular = uriSegments.get(i).getSegmentSingular(); + if ( segmentSingular.equals("cvlan-tag")) { + // map to what is in the entity + segmentSingular = "cvlan-tag-entry"; + } + jsonObj = entityBody.getAsJsonObject(uriSegments.get(i).getSegmentPlural().get()); + jsonArray = jsonObj.getAsJsonArray(segmentSingular); + if ( jsonArray == null ) { + LOGGER.error("failed in getWrappedEntityObject " + segmentSingular + " not found in " + jsonObj ); + // exception expected for missing template + } + entityBody = jsonArray.get(0).getAsJsonObject(); + arrayKey = uriSegments.get(i).getSegmentSingular(); + } else { + entityBody = entityBody.getAsJsonArray(uriSegments.get(i).getSegmentSingular()).get(0).getAsJsonObject(); + arrayKey = uriSegments.get(i).getSegmentSingular(); + } + + } + } + arrayWrapper.add(entityBody); + objectWrapper.add(arrayKey, arrayWrapper); + return objectWrapper; + } + + private String getBaseUri(List<AAIUriSegment> uriSegments) { + if (uriSegments.isEmpty() || uriSegments.size() == 1) { + return ""; + } + return String.join("", uriSegments.subList(0, uriSegments.size()-1).stream().map(AAIUriSegment::getSegment).collect(Collectors.toList())); + } + + + +}
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/DmaapAction.java b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/DmaapAction.java new file mode 100644 index 0000000..73aafd3 --- /dev/null +++ b/src/main/java/org/onap/aai/cacher/injestion/parser/strategy/aai/dmaap/DmaapAction.java @@ -0,0 +1,38 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.cacher.injestion.parser.strategy.aai.dmaap; + +import org.onap.aai.cacher.model.DBAction; + +public enum DmaapAction { + DELETE(DBAction.DELETE), + UPDATE(DBAction.UPDATE), + CREATE(DBAction.INSERT_REPLACE); + + private final DBAction dbAction; + DmaapAction(DBAction dbAction) { + this.dbAction = dbAction; + } + + public DBAction getDbAction() { + return dbAction; + } + }
\ No newline at end of file diff --git a/src/main/java/org/onap/aai/cacher/model/CacheEntry.java b/src/main/java/org/onap/aai/cacher/model/CacheEntry.java index ed6715e..0fd5c92 100644 --- a/src/main/java/org/onap/aai/cacher/model/CacheEntry.java +++ b/src/main/java/org/onap/aai/cacher/model/CacheEntry.java @@ -34,6 +34,8 @@ public class CacheEntry { protected JsonObject findQuery; protected boolean isNested = false; + protected boolean isNestedPayloadString = false; + protected String nestedString; protected String nestedField; protected JsonObject nestedFind; protected JsonObject nestedFieldIdentifierObj; @@ -89,6 +91,22 @@ public class CacheEntry { isNested = nested; } + public boolean isNestedPayloadString() { + return isNestedPayloadString; + } + + public void setNestedPayloadString(boolean nestedPayloadString) { + isNestedPayloadString = nestedPayloadString; + } + + public String getNestedString() { + return nestedString; + } + + public void setNestedString(String nestedString) { + this.nestedString = nestedString; + } + public String getNestedField() { return nestedField; } @@ -114,15 +132,17 @@ public class CacheEntry { } public static final class CacheEntryBuilder { - protected DBAction dbAction; - protected String id; - protected String collection; - protected JsonObject payload; - protected JsonObject findQuery; - protected boolean isNested; - protected String nestedField; - protected JsonObject nestedFind; - protected JsonObject nestedFieldIdentifierObj; + private DBAction dbAction; + private String id; + private String collection; + private JsonObject payload; + private JsonObject findQuery; + private boolean isNested; + private String nestedField; + private JsonObject nestedFind; + private JsonObject nestedFieldIdentifierObj; + private boolean isNestedPayloadString = false; + private String nestedString; private CacheEntryBuilder() { } @@ -141,6 +161,9 @@ public class CacheEntry { nestedField = cacheEntry.getNestedField(); nestedFind = cacheEntry.getNestedFind(); nestedFieldIdentifierObj = cacheEntry.getNestedFieldIdentifierObj(); + nestedString = cacheEntry.getNestedString(); + isNestedPayloadString = cacheEntry.isNestedPayloadString(); + return this; } @@ -174,6 +197,16 @@ public class CacheEntry { return this; } + public CacheEntryBuilder isNestedPayloadString(boolean isNestedPayloadString) { + this.isNestedPayloadString = isNestedPayloadString; + return this; + } + + public CacheEntryBuilder withNestedString(String nestedString) { + this.nestedString = nestedString; + return this; + } + public CacheEntryBuilder withNestedField(String nestedField) { this.nestedField = nestedField; return this; @@ -199,7 +232,10 @@ public class CacheEntry { cacheEntry.setNestedField(nestedField); cacheEntry.setNestedFind(nestedFind); cacheEntry.setNestedFieldIdentifierObj(nestedFieldIdentifierObj); - cacheEntry.isNested = this.isNested; + cacheEntry.setNested(this.isNested); + cacheEntry.setNestedPayloadString(this.isNestedPayloadString); + cacheEntry.setNestedString(this.nestedString); + return cacheEntry; } } diff --git a/src/main/java/org/onap/aai/cacher/service/helper/CacheHelperService.java b/src/main/java/org/onap/aai/cacher/service/helper/CacheHelperService.java index 8b85174..9997609 100644 --- a/src/main/java/org/onap/aai/cacher/service/helper/CacheHelperService.java +++ b/src/main/java/org/onap/aai/cacher/service/helper/CacheHelperService.java @@ -24,6 +24,8 @@ import com.att.eelf.configuration.EELFManager; import com.google.common.collect.Lists; import com.google.gson.*; import com.mongodb.*; + +import org.apache.commons.lang3.StringUtils; import org.onap.aai.cacher.common.MongoHelperSingleton; import org.onap.aai.cacher.egestion.printer.PayloadPrinterService; import org.onap.aai.cacher.injestion.parser.PayloadParserService; @@ -137,7 +139,6 @@ public class CacheHelperService { // remove "_id" property from cache response JsonParser parser = new JsonParser(); JsonObject jsonObj = (JsonObject) parser.parse(cursor.next().toString()); - jsonObj.remove("_id"); jsonArray.add(jsonObj); } } @@ -209,8 +210,11 @@ public class CacheHelperService { public Response forceSync(CacheKey ck) { if (isCurrentlyRunning(ck)) { - AAIException aaiException = new AAIException("AAI_4000", "Sync is currently running from another process."); - return buildExceptionResponse(aaiException); + AAIException aaiException = new AAIException("AAI_3000", "Sync is currently running from another process."); + ArrayList<String> templateVars = new ArrayList(); + templateVars.add("/sync"); + templateVars.add(ck.getCacheKey()); + return buildExceptionResponse(aaiException, templateVars); } else if (isKeyPresent(ck, AAIConstants.COLLECTION_CACHEKEY)) { // populate cache and return status on sync ResponseEntity resp = rchs.triggerRestCall(ck); @@ -232,8 +236,12 @@ public class CacheHelperService { result = this.retrieveCollectionString(ck, collection); if (result.equals("")) { - status = Status.NOT_FOUND; - EELF_LOGGER.error("Cannot not found the cache key from mongodb"); + EELF_LOGGER.error("Cannot not find the cache key from mongodb " + ck.getCacheKey()); + AAIException aaiException = new AAIException("AAI_3001", "cacheKey request"); + ArrayList<String> templateVars = new ArrayList(); + templateVars.add("/get"); + templateVars.add(ck.getCacheKey()); + return buildExceptionResponse(aaiException, templateVars); } return this.buildResponse(status, result); } catch (Exception e) { @@ -251,6 +259,8 @@ public class CacheHelperService { if (result.equals("")) { status = Status.NOT_FOUND; EELF_LOGGER.error("Cannot not found the cache key from mongodb"); + AAIException aaiException = new AAIException("AAI_3001", "cacheKey get for" + ck.getCacheKey() ); + return buildExceptionResponse(aaiException); } return this.buildResponse(status, result); } catch (Exception e) { @@ -285,7 +295,11 @@ public class CacheHelperService { result.append(cursor.next()); } } else { - status = Status.NOT_FOUND; + AAIException aaiException = new AAIException("AAI_3001", "cacheKey request"); + ArrayList<String> templateVars = new ArrayList(); + templateVars.add("/get"); + templateVars.add("ALL"); + return buildExceptionResponse(aaiException, templateVars); } return buildResponse(status, result.toString()); } catch (Exception e) { @@ -321,8 +335,11 @@ public class CacheHelperService { if (result.getN() > 0) { status = Status.OK; } else { - // TODO set proper status for no results updated meaning it didn't find the key - status = Status.NOT_FOUND; + AAIException aaiException = new AAIException("AAI_3001", "cacheKey request"); + ArrayList<String> templateVars = new ArrayList(); + templateVars.add("/update"); + templateVars.add(ck.getCacheKey()); + return buildExceptionResponse(aaiException, templateVars); } return buildResponse(status, "{}"); } catch (MongoException ex) { @@ -354,8 +371,11 @@ public class CacheHelperService { status = Status.NO_CONTENT; return buildResponse(status, "{}"); } else if (cacheKeyDelete.equals("NOT_FOUND")) { - status = Status.NOT_FOUND; - return buildResponse(status, "{}"); + AAIException aaiException = new AAIException("AAI_3001", "cacheKey request"); + ArrayList<String> templateVars = new ArrayList(); + templateVars.add("/delete"); + templateVars.add(id); + return buildExceptionResponse(aaiException, templateVars); } else { AAIException aaiException = new AAIException("AAI_5105"); return buildExceptionResponse(aaiException); @@ -369,8 +389,11 @@ public class CacheHelperService { status = Status.NO_CONTENT; return buildResponse(status, "{}"); } else if (cacheDelete.equals("NOT_FOUND")) { - status = Status.NOT_FOUND; - return buildResponse(status, "{}"); + AAIException aaiException = new AAIException("AAI_3001", "cacheKey request"); + ArrayList<String> templateVars = new ArrayList(); + templateVars.add("/delete"); + templateVars.add(id); + return buildExceptionResponse(aaiException, templateVars); } else { AAIException aaiException = new AAIException("AAI_5105"); return buildExceptionResponse(aaiException); @@ -425,17 +448,32 @@ public class CacheHelperService { ck.setLastSyncSuccessTime(formatter.format(System.currentTimeMillis())); ck.setLastSyncEndTime(formatter.format(System.currentTimeMillis())); updateCacheKey(ck); - return buildResponse(Status.CREATED, "{}"); + return buildResponse(Status.CREATED, null); } public Response buildResponse(Status status, String result) { + + if ( result == null ) { + return Response.status(status).type(MediaType.APPLICATION_JSON).build(); + } return Response.status(status).type(MediaType.APPLICATION_JSON).entity(result).build(); } + public Response buildMissingFieldResponse(List<String> fields) { + AAIException aaiException = new AAIException("AAI_6118"); + ArrayList<String> templateVars = new ArrayList<>(); + templateVars.add(StringUtils.join(fields, ", ")); + ErrorLogHelper.logException(aaiException); + return Response.status(aaiException.getErrorObject().getHTTPResponseCode()) + .entity(ErrorLogHelper.getRESTAPIErrorResponseWithLogging( + Lists.newArrayList(MediaType.APPLICATION_JSON_TYPE), aaiException, templateVars)) + .build(); + } + public Response buildValidationResponse(List<String> issues) { AAIException aaiException = new AAIException("AAI_3014"); ArrayList<String> templateVars = new ArrayList<>(); - + if (templateVars.isEmpty()) { templateVars.add(issues.toString()); } @@ -444,13 +482,21 @@ public class CacheHelperService { .entity(ErrorLogHelper.getRESTAPIErrorResponseWithLogging( Lists.newArrayList(MediaType.APPLICATION_JSON_TYPE), aaiException, templateVars)) .build(); - } + } + public Response buildExceptionResponse(AAIException aaiException, ArrayList templateVars) { + ErrorLogHelper.logException(aaiException); + return Response.status(aaiException.getErrorObject().getHTTPResponseCode()) + .entity(ErrorLogHelper.getRESTAPIErrorResponseWithLogging( + Lists.newArrayList(MediaType.APPLICATION_JSON_TYPE), aaiException, templateVars)) + .build(); + } + public Response buildExceptionResponse(AAIException aaiException) { ErrorLogHelper.logException(aaiException); return Response.status(aaiException.getErrorObject().getHTTPResponseCode()) .entity(ErrorLogHelper.getRESTAPIErrorResponseWithLogging( - Lists.newArrayList(MediaType.APPLICATION_JSON_TYPE), aaiException, new ArrayList<>())) + Lists.newArrayList(MediaType.APPLICATION_JSON_TYPE), aaiException, new ArrayList())) .build(); } diff --git a/src/main/java/org/onap/aai/cacher/service/rest/CacheKeyService.java b/src/main/java/org/onap/aai/cacher/service/rest/CacheKeyService.java index 962be77..432b23c 100644 --- a/src/main/java/org/onap/aai/cacher/service/rest/CacheKeyService.java +++ b/src/main/java/org/onap/aai/cacher/service/rest/CacheKeyService.java @@ -58,8 +58,13 @@ public class CacheKeyService { EELF_LOGGER.info("Got the request to add cache key to mongodb"); CacheKeyRequestValidation ckrv = new CacheKeyRequestValidation(CacheKeyRequestValidationType.ADD); JsonObject input = convertStringToJSON(payload); - List<String> issues = ckrv.validateCacheKeyRequest(input, chs); + List<String> missing = ckrv.checkMissingRequiredFields(input, chs); Response response; + if ( !missing.isEmpty()) { + response = chs.buildMissingFieldResponse(missing); + return response; + } + List<String> issues = ckrv.validateCacheKeyRequest(input, chs); if (!issues.isEmpty()) { response = chs.buildValidationResponse(issues); } else { diff --git a/src/main/java/org/onap/aai/cacher/service/rest/util/CacheKeyRequestValidation.java b/src/main/java/org/onap/aai/cacher/service/rest/util/CacheKeyRequestValidation.java index 4df1921..2f3ecde 100644 --- a/src/main/java/org/onap/aai/cacher/service/rest/util/CacheKeyRequestValidation.java +++ b/src/main/java/org/onap/aai/cacher/service/rest/util/CacheKeyRequestValidation.java @@ -33,6 +33,23 @@ public class CacheKeyRequestValidation { public CacheKeyRequestValidation(CacheKeyRequestValidationType type) { this.type = type; } + + public List<String> checkMissingRequiredFields(JsonObject input, CacheHelperService chs) { + ArrayList<String> results = new ArrayList<>(); + if (input == null) { + return results; + } + CacheKey cacheKey = CacheKey.fromJson(input); + String baseUrl = cacheKey.getBaseUrl(); + String uri = cacheKey.getURI(); + if ( CacheKey.DEFAULT_VALUE.equals(baseUrl)) { + results.add("baseUrl"); + } + if ( CacheKey.DEFAULT_VALUE.equals(uri)) { + results.add("URI"); + } + return results; + } public List<String> validateCacheKeyRequest(JsonObject input, CacheHelperService chs) { ArrayList<String> results = new ArrayList<>(); diff --git a/src/main/java/org/onap/aai/cacher/service/tasks/ScheduledTasks.java b/src/main/java/org/onap/aai/cacher/service/tasks/ScheduledTasks.java index 78c88c7..7497f10 100644 --- a/src/main/java/org/onap/aai/cacher/service/tasks/ScheduledTasks.java +++ b/src/main/java/org/onap/aai/cacher/service/tasks/ScheduledTasks.java @@ -98,7 +98,7 @@ public class ScheduledTasks { } // initialize the cache if (!cacheLoaded) { - LOGGER.info("Start loading cache @ " + dateFormat.format(new Date())); + LOGGER.info("initializing: Start loading cache @ " + dateFormat.format(new Date())); init(); cacheLoaded = true; } @@ -120,12 +120,13 @@ public class ScheduledTasks { for (CacheKey cacheKey : cacheKeys) { if ("onInit".equalsIgnoreCase(cacheKey.getTimingIndicator())) { try { + LOGGER.info("initializing: cacheKey " + cacheKey.getCacheKey() + " loading"); ResponseEntity respEntity = rchs.triggerRestCall(cacheKey); if (respEntity.getStatusCode().is2xxSuccessful()) { Response resp = chs.populateCache(cacheKey, (String) respEntity.getBody()); if (resp != null) { if (resp.getStatus() == Response.Status.CREATED.getStatusCode()) { - LOGGER.debug("cacheKey " + cacheKey.getCacheKey() + " loaded"); + LOGGER.info("initializing: cacheKey " + cacheKey.getCacheKey() + " loaded"); } else { LOGGER.error("unexpected 2xx response status for cacheKey " + cacheKey.getCacheKey() + " " + resp.getStatusInfo()); @@ -142,7 +143,7 @@ public class ScheduledTasks { } } } - + LOGGER.info("initializing: cache completed @ " + dateFormat.format(new Date())); DmaapConsumerSingleton.getInstance().setProcessEvents(true); } } diff --git a/src/main/java/org/onap/aai/cacher/util/RestClient.java b/src/main/java/org/onap/aai/cacher/util/RestClient.java index 0caffd3..9613d81 100644 --- a/src/main/java/org/onap/aai/cacher/util/RestClient.java +++ b/src/main/java/org/onap/aai/cacher/util/RestClient.java @@ -26,7 +26,6 @@ import org.apache.http.ssl.SSLContextBuilder; import org.onap.aai.util.AAIConfig; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.context.annotation.Bean; -import org.springframework.core.env.Environment; import org.springframework.http.*; import org.springframework.http.client.ClientHttpResponse; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; @@ -43,12 +42,9 @@ import java.nio.charset.Charset; import java.security.KeyStore; import java.util.Collections; -//@Component public class RestClient { private HttpClient restClient = null; - - private Environment environment; public RestClient() { this.restClient = initClient(); @@ -64,14 +60,15 @@ public class RestClient { */ private HttpClient initClient() { HttpClient rc; - try { String truststore_path = AAIConstants.AAI_HOME_ETC_AUTH + AAIConfig.get(AAIConstants.AAI_TRUSTSTORE_FILENAME); String truststore_password = AAIConfig.get(AAIConstants.AAI_TRUSTSTORE_PASSWD); - SSLContextBuilder sslContextBuilder = SSLContextBuilder.create(); - - SSLContext sslContext = sslContextBuilder + String keystore_path = AAIConstants.AAI_HOME_ETC_AUTH + AAIConfig.get(AAIConstants.AAI_KEYSTORE_FILENAME); + String keystore_password = AAIConfig.get(AAIConstants.AAI_KEYSTORE_PASSWD); + SSLContext sslContext = SSLContextBuilder.create() + .loadKeyMaterial(loadPfx(keystore_path, keystore_password.toCharArray()), + keystore_password.toCharArray()) .loadTrustMaterial(ResourceUtils.getFile(truststore_path), truststore_password.toCharArray()) .build(); @@ -118,7 +115,7 @@ public class RestClient { urlUpdate = baseUrl; } HttpEntity httpEntity = new HttpEntity(headers); - responseEntity = restTemplate.exchange(urlUpdate + endpoint, HttpMethod.GET, httpEntity, String.class); + responseEntity = restTemplate.exchange(baseUrl + endpoint, HttpMethod.GET, httpEntity, String.class); } catch (Exception e) { e.printStackTrace(); // TODO handle exceptions diff --git a/src/main/java/org/onap/aai/cacher/web/JerseyConfiguration.java b/src/main/java/org/onap/aai/cacher/web/JerseyConfiguration.java index 6498f2f..b18e840 100644 --- a/src/main/java/org/onap/aai/cacher/web/JerseyConfiguration.java +++ b/src/main/java/org/onap/aai/cacher/web/JerseyConfiguration.java @@ -33,7 +33,6 @@ import javax.annotation.Priority; import javax.ws.rs.ApplicationPath; import javax.ws.rs.container.ContainerRequestFilter; import javax.ws.rs.container.ContainerResponseFilter; - import java.util.List; import java.util.Set; import java.util.logging.Logger; diff --git a/src/main/resources/etc/appprops/aai-resources-uri-templates.properties b/src/main/resources/etc/appprops/aai-resources-uri-templates.properties index 44066e1..2e3de7a 100644 --- a/src/main/resources/etc/appprops/aai-resources-uri-templates.properties +++ b/src/main/resources/etc/appprops/aai-resources-uri-templates.properties @@ -10,7 +10,7 @@ constrained-element-set=/constrained-element-sets/constrained-element-set/{const ctag-assignment=/ctag-assignments/ctag-assignment/{vlan-id-inner} ctag-pool=/ctag-pools/ctag-pool/{target-pe}/{availability-zone-name} customer=/business/customers/customer/{global-customer-id} -cvlan-tag-entry=/cvlan-tags/cvlan-tag-entry/{cvlan-tag} +cvlan-tag=/cvlan-tags/cvlan-tag/{cvlan-tag} dvs-switch=/dvs-switches/dvs-switch/{switch-name} element-choice-set=/element-choice-sets/element-choice-set/{element-choice-set-uuid} entitlement=/entitlements/entitlement/{group-uuid}/{resource-uuid} diff --git a/src/main/resources/etc/appprops/aaiDmaaPEventConsumer.properties b/src/main/resources/etc/appprops/aaiDmaaPEventConsumer.properties index 398e175..378db93 100644 --- a/src/main/resources/etc/appprops/aaiDmaaPEventConsumer.properties +++ b/src/main/resources/etc/appprops/aaiDmaaPEventConsumer.properties @@ -1,23 +1,31 @@ -TransportType=TBD -Latitude=TBD -Longitude=TBD +TransportType=HTTPAAF +Latitude=39.099727 +Longitude=-94.578567 Version=1.0 -ServiceName=TBD +ServiceName=dmaap-v1.dev.dmaap.dt.saat.acsi.att.com/events Environment=TEST Partner=BOT_R routeOffer=MR1SBKCD SubContextPath=/ Protocol=http MethodType=GET -username=TBD -password=TBD +username=m08479@aai.ecomp.att.com +password=OBF:1qab1lbw1kfr1ffg1hzn1i231fhe1kcn1lfm1qcr contenttype=application/json -host=TBD +host=klsd048.ipcoe.att.com:3904,klsd055.ipcoe.att.com:3904,klsd056.ipcoe.att.com:3904 topic=AAI-EVENT group=aaiEventConsumer-dev id=NA timeout=15000 limit=1000 -filter={"event-header.domain":"devINT1"} -sessionstickinessrequired=no
\ No newline at end of file +filter={"event-header.domain":"uDev03"} +AFT_DME2_EXCHANGE_REQUEST_HANDLERS=com.att.nsa.test.PreferredRouteRequestHandler +AFT_DME2_EXCHANGE_REPLY_HANDLERS=com.att.nsa.test.PreferredRouteReplyHandler +AFT_DME2_REQ_TRACE_ON=true +AFT_ENVIRONMENT=AFTUAT +AFT_DME2_EP_CONN_TIMEOUT=15000 +AFT_DME2_ROUNDTRIP_TIMEOUT_MS=240000 +AFT_DME2_EP_READ_TIMEOUT_MS=50000 +sessionstickinessrequired=no +DME2preferredRouterFilePath=preferredRoute.txt
\ No newline at end of file diff --git a/src/main/resources/etc/appprops/aaiconfig.properties b/src/main/resources/etc/appprops/aaiconfig.properties index 0524c13..36d616c 100644 --- a/src/main/resources/etc/appprops/aaiconfig.properties +++ b/src/main/resources/etc/appprops/aaiconfig.properties @@ -1,5 +1,7 @@ -aai.truststore.filename=aai_keystore -aai.truststore.passwd.x=OBF:1vn21ugu1saj1v9i1v941sar1ugw1vo0 +aai.truststore.filename=tomcat_keystore +aai.truststore.passwd.x=OBF:1i9a1u2a1unz1lr61wn51wn11lss1unz1u301i6o +aai.keystore.filename=aai-client-cert.p12 +aai.keystore.passwd.x=OBF:1i9a1u2a1unz1lr61wn51wn11lss1unz1u301i6o aai.logging.maxStackTraceEntries=10 -aai.cacher.dmaap.consumer.enableEventProcessing=false +aai.cacher.dmaap.consumer.enableEventProcessing=true aai.cacher.dmaap.consumer.delayCheck=2 diff --git a/src/main/resources/etc/appprops/error.properties b/src/main/resources/etc/appprops/error.properties index 896df00..48e375d 100644 --- a/src/main/resources/etc/appprops/error.properties +++ b/src/main/resources/etc/appprops/error.properties @@ -166,7 +166,6 @@ AAI_9101=5:0:WARN:9101:403:3300:User is not authorized to perform function #AAI_9106=5:0:WARN:9106:403:3300:Invalid AppId #AAI_9107=5:0:WARN:9107:403:3300:No Username in Request AAI_9107=5:0:WARN:9107:403:3300:SSL is not provided in request, please contact admin -AAI_9108=5:0:WARN:9107:403:3300:Basic auth credentials is not provided in the request #--- aaiinstar: 9201-9299 #AAI_9201=5:4:ERROR:9201:500:3002:Unable to send notification diff --git a/src/main/resources/etc/appprops/initialcachekeyconfig.json b/src/main/resources/etc/appprops/initialcachekeyconfig.json index f1fdabe..d2bb66d 100644 --- a/src/main/resources/etc/appprops/initialcachekeyconfig.json +++ b/src/main/resources/etc/appprops/initialcachekeyconfig.json @@ -4,7 +4,7 @@ { "cacheKey": "cloud-region", "baseUrl": "https://AAI:AAI@localhost:8447", - "module": "/aai/v13/", + "module": "/aai/v14/", "URI": "cloud-infrastructure/cloud-regions?depth=0&resultIndex=1&resultSize=3", "timingIndicator": "onInit", "httpMethod": "GET", @@ -13,7 +13,7 @@ { "cacheKey": "complex", "baseUrl": "https://AAI:AAI@localhost:8447", - "module": "/aai/v13/", + "module": "/aai/v14/", "URI": "cloud-infrastructure/complexes?resultIndex=1&resultSize=3", "timingIndicator": "onInit", "httpMethod": "GET", @@ -22,7 +22,7 @@ { "cacheKey": "pserver", "baseUrl": "https://AAI:AAI@localhost:8447", - "module": "/aai/v13/", + "module": "/aai/v14/", "URI": "cloud-infrastructure/pservers?depth=5807c3c3-92cd-44d7-a508-8539cd36ecda&resultIndex=1&resultSize=3", "timingIndicator": "onInit", "httpMethod": "GET", @@ -31,7 +31,7 @@ { "cacheKey": "generic-vnf", "baseUrl": "https://AAI:AAI@localhost:8447", - "module": "/aai/v13/", + "module": "/aai/v14/", "URI": "network/generic-vnfs?depth=5807c3c3-92cd-44d7-a508-8539cd36ecda&resultIndex=1&resultSize=3", "timingIndicator": "onInit", "httpMethod": "GET", diff --git a/src/main/resources/etc/auth/aai-client-cert.p12 b/src/main/resources/etc/auth/aai-client-cert.p12 Binary files differdeleted file mode 100644 index 292efb7..0000000 --- a/src/main/resources/etc/auth/aai-client-cert.p12 +++ /dev/null diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 9caabe6..3bcb0ba 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -170,7 +170,12 @@ <pattern>${eelfLogPattern}</pattern> </encoder> </appender> - <logger name="org.onap.aai.cacher" level="DEBUG" additivity="false"> + + <logger name="mongo" level="DEBUG" additivity="false"> + <appender-ref ref="external" /> + </logger> + + <logger name="org.onap.aai" level="DEBUG" additivity="false"> <appender-ref ref="asyncDEBUG" /> <appender-ref ref="asyncERROR" /> <appender-ref ref="asyncMETRIC" /> |