From f3cbb41773aac52da6c1763fb25a6e0f749229ac Mon Sep 17 00:00:00 2001 From: Zlatko Murgoski Date: Wed, 19 Dec 2018 15:00:14 +0100 Subject: Fix bug with processing event list DMaapEventPublisher processing of eventList Issue-ID: DCAEGEN2-1035 Change-Id: If62ec51d1dc1f0e501fda624dae23c7a83d3727b Signed-off-by: Zlatko Murgoski --- src/main/java/org/onap/dcae/VesApplication.java | 4 +- .../commonFunction/ConfigProcessorAdapter.java | 45 +++++++ .../onap/dcae/commonFunction/EventProcessor.java | 140 +++------------------ .../org/onap/dcae/commonFunction/EventSender.java | 125 ++++++++++++++++++ 4 files changed, 187 insertions(+), 127 deletions(-) create mode 100644 src/main/java/org/onap/dcae/commonFunction/ConfigProcessorAdapter.java create mode 100644 src/main/java/org/onap/dcae/commonFunction/EventSender.java (limited to 'src/main/java') diff --git a/src/main/java/org/onap/dcae/VesApplication.java b/src/main/java/org/onap/dcae/VesApplication.java index a70d0935..b57caf3c 100644 --- a/src/main/java/org/onap/dcae/VesApplication.java +++ b/src/main/java/org/onap/dcae/VesApplication.java @@ -29,6 +29,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.json.JSONObject; import org.onap.dcae.commonFunction.EventProcessor; +import org.onap.dcae.commonFunction.EventSender; import org.onap.dcae.commonFunction.event.publishing.DMaaPConfigurationParser; import org.onap.dcae.commonFunction.event.publishing.EventPublisher; import org.onap.dcae.commonFunction.event.publishing.PublisherConfig; @@ -66,7 +67,8 @@ public class VesApplication { .parseToDomainMapping(Paths.get(properties.dMaaPConfigurationFileLocation())) .get()); spawnDynamicConfigUpdateThread(publisher, properties); - EventProcessor ep = new EventProcessor(EventPublisher.createPublisher(oplog, getDmapConfig()), properties); + EventProcessor ep = new EventProcessor( + new EventSender(EventPublisher.createPublisher(oplog, getDmapConfig()), properties)); ExecutorService executor = Executors.newFixedThreadPool(MAX_THREADS); for (int i = 0; i < MAX_THREADS; ++i) { diff --git a/src/main/java/org/onap/dcae/commonFunction/ConfigProcessorAdapter.java b/src/main/java/org/onap/dcae/commonFunction/ConfigProcessorAdapter.java new file mode 100644 index 00000000..3df412ea --- /dev/null +++ b/src/main/java/org/onap/dcae/commonFunction/ConfigProcessorAdapter.java @@ -0,0 +1,45 @@ +/*- + * ============LICENSE_START======================================================= + * org.onap.dcaegen2.collectors.ves + * ================================================================================ + * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2018 Nokia. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.dcae.commonFunction; + +import java.lang.reflect.Method; +import org.json.JSONObject; + +class ConfigProcessorAdapter { + + private final ConfigProcessors configProcessors; + + ConfigProcessorAdapter(ConfigProcessors configProcessors) { + this.configProcessors = configProcessors; + } + + boolean isFilterMet(JSONObject parameter) { + return configProcessors.isFilterMet(parameter); + } + + void runConfigProcessorFunctionByName(String functionName, JSONObject parameter) + throws ReflectiveOperationException { + Method method = configProcessors.getClass() + .getDeclaredMethod(functionName, parameter.getClass()); + method.invoke(configProcessors, parameter); + } +} diff --git a/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java b/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java index 7d27399d..bd830456 100644 --- a/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java +++ b/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java @@ -23,151 +23,39 @@ package org.onap.dcae.commonFunction; import com.att.nsa.clock.SaClock; import com.att.nsa.logging.LoggingContext; import com.att.nsa.logging.log4j.EcompFields; -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import io.vavr.collection.Map; import org.json.JSONObject; -import org.onap.dcae.ApplicationSettings; import org.onap.dcae.VesApplication; -import org.onap.dcae.commonFunction.event.publishing.EventPublisher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileReader; -import java.io.IOException; -import java.lang.reflect.Method; -import java.lang.reflect.Type; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; - public class EventProcessor implements Runnable { - static final Type EVENT_LIST_TYPE = new TypeToken>() { - }.getType(); private static final Logger log = LoggerFactory.getLogger(EventProcessor.class); - private static final String EVENT_LITERAL = "event"; - private static final String COMMON_EVENT_HEADER = "commonEventHeader"; - private final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, MM dd yyyy hh:mm:ss z"); - - public JSONObject event; - private EventPublisher eventPublisher; - private Map streamidHash; - private ApplicationSettings properties; + private EventSender eventSender; - - public EventProcessor(EventPublisher eventPublisher, ApplicationSettings properties) { - this.eventPublisher = eventPublisher; - this.properties = properties; - this.streamidHash = properties.dMaaPStreamsMapping(); + public EventProcessor(EventSender eventSender) { + this.eventSender = eventSender; } @Override public void run() { try { - while (true) { - event = VesApplication.fProcessingInputQueue.take(); - // As long as the producer is running we remove elements from - // the queue. - log.info("QueueSize:" + VesApplication.fProcessingInputQueue.size() + "\tEventProcessor\tRemoving element: " + event); - - String uuid = event.get("VESuniqueId").toString(); - LoggingContext localLC = VESLogger.getLoggingContextForThread(uuid); - localLC.put(EcompFields.kBeginTimestampMs, SaClock.now()); - - String domain = event.getJSONObject(EVENT_LITERAL).getJSONObject(COMMON_EVENT_HEADER).getString("domain"); - log.debug("event.VESuniqueId" + event.get("VESuniqueId") + "event.commonEventHeader.domain:" + domain); - streamidHash.get(domain) - .onEmpty(() -> { - log.error("No StreamID defined for publish - Message dropped" + event); - }).forEach(streamIds -> { - sendEventsToStreams(streamIds); - }); - log.debug("Message published" + event); - } + while (true){ + JSONObject event = VesApplication.fProcessingInputQueue.take(); + log.info("QueueSize:" + VesApplication.fProcessingInputQueue.size() + "\tEventProcessor\tRemoving element: " + event); + setLoggingContext(event); + log.debug("event.VESuniqueId" + event.get("VESuniqueId") + "event.commonEventHeader.domain:" + eventSender.getDomain(event)); + eventSender.send(event); + log.debug("Message published" + event); + } } catch (InterruptedException e) { log.error("EventProcessor InterruptedException" + e.getMessage()); Thread.currentThread().interrupt(); } } - public void overrideEvent() { - // Set collector timestamp in event payload before publish - addCurrentTimeToEvent(event); - - if (properties.eventTransformingEnabled()) { - // read the mapping json file - try (FileReader fr = new FileReader("./etc/eventTransform.json")) { - log.info("parse eventTransform.json"); - List events = new Gson().fromJson(fr, EVENT_LIST_TYPE); - parseEventsJson(events, new ConfigProcessorAdapter(new ConfigProcessors(event))); - } catch (IOException e) { - log.error("Couldn't find file ./etc/eventTransform.json" + e.toString()); - } - } - // Remove VESversion from event. This field is for internal use and must be removed after use. - if (event.has("VESversion")) - event.remove("VESversion"); - - log.debug("Modified event:" + event); - } - - private void sendEventsToStreams(String[] streamIdList) { - for (String aStreamIdList : streamIdList) { - log.info("Invoking publisher for streamId:" + aStreamIdList); - this.overrideEvent(); - eventPublisher.sendEvent(event, aStreamIdList); - } - } - - private void addCurrentTimeToEvent(JSONObject event) { - final Date currentTime = new Date(); - JSONObject collectorTimeStamp = new JSONObject().put("collectorTimeStamp", dateFormat.format(currentTime)); - JSONObject commonEventHeaderkey = event.getJSONObject(EVENT_LITERAL).getJSONObject(COMMON_EVENT_HEADER); - commonEventHeaderkey.put("internalHeaderFields", collectorTimeStamp); - event.getJSONObject(EVENT_LITERAL).put(COMMON_EVENT_HEADER, commonEventHeaderkey); - } - - void parseEventsJson(List eventsTransform, ConfigProcessorAdapter configProcessorAdapter) { - // load VESProcessors class at runtime - for (Event eventTransform : eventsTransform) { - JSONObject filterObj = new JSONObject(eventTransform.filter.toString()); - if (configProcessorAdapter.isFilterMet(filterObj)) { - callProcessorsMethod(configProcessorAdapter, eventTransform.processors); - } - } - } - - private void callProcessorsMethod(ConfigProcessorAdapter configProcessorAdapter, List processors) { - // call the processor method - for (Processor processor : processors) { - final String functionName = processor.functionName; - final JSONObject args = new JSONObject(processor.args.toString()); - - log.info(String.format("functionName==%s | args==%s", functionName, args)); - // reflect method call - try { - configProcessorAdapter.runConfigProcessorFunctionByName(functionName, args); - } catch (ReflectiveOperationException e) { - log.error("EventProcessor Exception" + e.getMessage() + e + e.getCause()); - } - } - } - - static class ConfigProcessorAdapter { - private final ConfigProcessors configProcessors; - - ConfigProcessorAdapter(ConfigProcessors configProcessors) { - this.configProcessors = configProcessors; - } - - boolean isFilterMet(JSONObject parameter) { - return configProcessors.isFilterMet(parameter); - } - - void runConfigProcessorFunctionByName(String functionName, JSONObject parameter) throws ReflectiveOperationException { - Method method = configProcessors.getClass().getDeclaredMethod(functionName, parameter.getClass()); - method.invoke(configProcessors, parameter); - } + private void setLoggingContext(JSONObject event) { + LoggingContext localLC = VESLogger.getLoggingContextForThread(event.get("VESuniqueId").toString()); + localLC.put(EcompFields.kBeginTimestampMs, SaClock.now()); } } \ No newline at end of file diff --git a/src/main/java/org/onap/dcae/commonFunction/EventSender.java b/src/main/java/org/onap/dcae/commonFunction/EventSender.java new file mode 100644 index 00000000..8a9c1eca --- /dev/null +++ b/src/main/java/org/onap/dcae/commonFunction/EventSender.java @@ -0,0 +1,125 @@ +/* + * ============LICENSE_START======================================================= + * PROJECT + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2018 Nokia. All rights reserved.s + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.dcae.commonFunction; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import io.vavr.collection.Map; +import io.vavr.control.Option; +import java.io.FileReader; +import java.io.IOException; +import java.lang.reflect.Type; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import org.json.JSONObject; +import org.onap.dcae.ApplicationSettings; +import org.onap.dcae.commonFunction.event.publishing.EventPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EventSender { + + private Map streamidHash; + private ApplicationSettings properties; + private EventPublisher eventPublisher; + + static final Type EVENT_LIST_TYPE = new TypeToken>() {}.getType(); + private static final Logger log = LoggerFactory.getLogger(EventSender.class); + private static final String EVENT_LITERAL = "event"; + private static final String COMMON_EVENT_HEADER = "commonEventHeader"; + private final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, MM dd yyyy hh:mm:ss z"); + + public EventSender( EventPublisher eventPublisher, ApplicationSettings properties) { + this.eventPublisher = eventPublisher; + this.streamidHash = properties.dMaaPStreamsMapping(); + this.properties = properties; + + } + + public void send(JSONObject event) { + streamidHash.get(getDomain(event)) + .onEmpty(() -> log.error("No StreamID defined for publish - Message dropped" + event)) + .forEach(streamIds -> sendEventsToStreams(event, streamIds)); + } + + public static String getDomain(JSONObject event) { + return event.getJSONObject(EVENT_LITERAL).getJSONObject(COMMON_EVENT_HEADER).getString("domain"); + } + + private void sendEventsToStreams(JSONObject event, String[] streamIdList) { + for (String aStreamIdList : streamIdList) { + log.info("Invoking publisher for streamId:" + aStreamIdList); + eventPublisher.sendEvent(overrideEvent(event), aStreamIdList); + } + } + + private JSONObject overrideEvent(JSONObject event) { + JSONObject jsonObject = addCurrentTimeToEvent(event); + if (properties.eventTransformingEnabled()) { + try (FileReader fr = new FileReader("./etc/eventTransform.json")) { + log.info("parse eventTransform.json"); + List events = new Gson().fromJson(fr, EVENT_LIST_TYPE); + parseEventsJson(events, new ConfigProcessorAdapter(new ConfigProcessors(jsonObject))); + } catch (IOException e) { + log.error("Couldn't find file ./etc/eventTransform.json" + e.toString()); + } + } + if (jsonObject.has("VESversion")) + jsonObject.remove("VESversion"); + + log.debug("Modified event:" + jsonObject); + return jsonObject; + } + + private JSONObject addCurrentTimeToEvent(JSONObject event) { + final Date currentTime = new Date(); + JSONObject collectorTimeStamp = new JSONObject().put("collectorTimeStamp", dateFormat.format(currentTime)); + JSONObject commonEventHeaderkey = event.getJSONObject(EVENT_LITERAL).getJSONObject(COMMON_EVENT_HEADER); + commonEventHeaderkey.put("internalHeaderFields", collectorTimeStamp); + event.getJSONObject(EVENT_LITERAL).put(COMMON_EVENT_HEADER, commonEventHeaderkey); + return event; + } + + private void parseEventsJson(List eventsTransform, ConfigProcessorAdapter configProcessorAdapter) { + for (Event eventTransform : eventsTransform) { + JSONObject filterObj = new JSONObject(eventTransform.filter.toString()); + if (configProcessorAdapter.isFilterMet(filterObj)) { + callProcessorsMethod(configProcessorAdapter, eventTransform.processors); + } + } + } + + private void callProcessorsMethod(ConfigProcessorAdapter configProcessorAdapter, List processors) { + for (Processor processor : processors) { + final String functionName = processor.functionName; + final JSONObject args = new JSONObject(processor.args.toString()); + log.info(String.format("functionName==%s | args==%s", functionName, args)); + try { + configProcessorAdapter.runConfigProcessorFunctionByName(functionName, args); + } catch (ReflectiveOperationException e) { + log.error("EventProcessor Exception" + e.getMessage() + e + e.getCause()); + } + } + } +} -- cgit 1.2.3-korg