From 7672276206d44d8dbc17bf73ef950b17191b6257 Mon Sep 17 00:00:00 2001 From: vagrant Date: Sat, 20 Jan 2018 14:55:43 +0000 Subject: Optimize collector publish mechanism Change-Id: I6d5fb2ad45257831e8e0ff62812df1fd1401464d Issue-ID: DCAEGEN2-281 Signed-off-by: Vijay VK --- .../onap/dcae/commonFunction/CommonStartup.java | 5 +- .../onap/dcae/commonFunction/ConfigProcessors.java | 61 ++----- .../onap/dcae/commonFunction/EventProcessor.java | 5 +- .../dcae/commonFunction/EventPublisherHash.java | 179 +++++++++++++++++++++ .../org/onap/dcae/vestest/TestEventProcessor.java | 71 +++++++- 5 files changed, 272 insertions(+), 49 deletions(-) create mode 100644 src/main/java/org/onap/dcae/commonFunction/EventPublisherHash.java (limited to 'src') diff --git a/src/main/java/org/onap/dcae/commonFunction/CommonStartup.java b/src/main/java/org/onap/dcae/commonFunction/CommonStartup.java index b4fae2a6..d1dbca92 100644 --- a/src/main/java/org/onap/dcae/commonFunction/CommonStartup.java +++ b/src/main/java/org/onap/dcae/commonFunction/CommonStartup.java @@ -195,7 +195,10 @@ public class CommonStartup extends NsaBaseEndpoint implements Runnable { // Thread epThread=new Thread(ep); // epThread.start(); executor = Executors.newFixedThreadPool(20); - executor.execute(ep); + //executor.execute(ep); + for (int i = 0; i < 20; ++i) { + executor.execute(ep); + } } catch (loadException | missingReqdSetting | IOException | invalidSettingValue | ServletException | InterruptedException e) { diff --git a/src/main/java/org/onap/dcae/commonFunction/ConfigProcessors.java b/src/main/java/org/onap/dcae/commonFunction/ConfigProcessors.java index 7e42bb57..4fb5adf0 100644 --- a/src/main/java/org/onap/dcae/commonFunction/ConfigProcessors.java +++ b/src/main/java/org/onap/dcae/commonFunction/ConfigProcessors.java @@ -41,9 +41,7 @@ public class ConfigProcessors { event = eventJson; } - /** - * - */ + public void getValue(JSONObject J) { // log.info("addAttribute"); final String field = J.getString(FIELD); @@ -57,9 +55,7 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + public void setValue(JSONObject J) { // log.info("addAttribute"); final String field = J.getString(FIELD); @@ -73,9 +69,8 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + + public String evaluate(String str) { String value = str; if (str.startsWith("$")) { @@ -85,9 +80,7 @@ public class ConfigProcessors { return value; } - /** - * { "functionName":"suppressEvent", "args":{} } - */ + public void suppressEvent(JSONObject J) { // log.info("addAttribute"); final JSONObject filter = J.optJSONObject(FILTER); @@ -100,9 +93,7 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + public void addAttribute(JSONObject J) { // log.info("addAttribute begin"); final String field = J.getString(FIELD); @@ -119,9 +110,7 @@ public class ConfigProcessors { // log.info("addAttribute End"); } - /** - * - */ + public void updateAttribute(JSONObject J) { // log.info("updateAttribute"); final String field = J.getString(FIELD); @@ -135,9 +124,7 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + public void removeAttribute(JSONObject J) { // log.info("removeAttribute"); final String field = J.getString(FIELD); @@ -149,9 +136,7 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + public void renameArrayInArray(JSONObject J) // map { log.info("renameArrayInArray"); @@ -185,9 +170,7 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + public void map(JSONObject J) { // log.info("mapAttribute"); final String field = J.getString(FIELD); @@ -200,9 +183,6 @@ public class ConfigProcessors { mapAttribute(J); } - /** - * - */ public String performOperation(String operation, String value) { log.info("performOperation"); if (operation != null) { @@ -214,9 +194,7 @@ public class ConfigProcessors { return value; } - /** - * - */ + // public void mapAttributeToArrayAttribute(JSONObject J) public void mapAttribute(JSONObject J) { // log.info("mapAttribute"); @@ -241,9 +219,7 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + public void mapToJArray(JSONObject J) { log.info("mapToJArray"); String field = J.getString(FIELD); @@ -355,9 +331,7 @@ public class ConfigProcessors { log.info("Filter not met"); } - /** - * - */ + private void removeEventKey(String field) { String[] keySet = field.split("\\.", field.length()); JSONObject keySeries = event; @@ -371,9 +345,7 @@ public class ConfigProcessors { } - /** - * - */ + private boolean checkFilter(JSONObject jo, String key, String logicKey) { String filterValue = jo.getString(key); boolean retVal = true; @@ -431,9 +403,7 @@ public class ConfigProcessors { return retVal; } - /** - * - */ + public boolean isFilterMet(JSONObject jo) { boolean retval = true; // log.info("Filter==" + jo.toString()); @@ -579,3 +549,4 @@ public class ConfigProcessors { private JSONObject event = new JSONObject(); } + diff --git a/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java b/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java index 79dea790..6811c672 100644 --- a/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java +++ b/src/main/java/org/onap/dcae/commonFunction/EventProcessor.java @@ -70,7 +70,7 @@ public class EventProcessor implements Runnable { try { event = CommonStartup.fProcessingInputQueue.take(); - log.info("EventProcessor\tRemoving element: " + event); + log.info("QueueSize:" + CommonStartup.fProcessingInputQueue.size()+ "\tEventProcessor\tRemoving element: " + event ); // EventPublisher Ep=new EventPublisher(); while (event != null) { @@ -95,7 +95,8 @@ public class EventProcessor implements Runnable { for (String aStreamIdList : streamIdList) { log.info("Invoking publisher for streamId:" + aStreamIdList); this.overrideEvent(); - EventPublisher.getInstance(aStreamIdList).sendEvent(event); + //EventPublisher.getInstance(aStreamIdList).sendEvent(event); + EventPublisherHash.getInstance().sendEvent(event, aStreamIdList); } } diff --git a/src/main/java/org/onap/dcae/commonFunction/EventPublisherHash.java b/src/main/java/org/onap/dcae/commonFunction/EventPublisherHash.java new file mode 100644 index 00000000..f8bdcaa2 --- /dev/null +++ b/src/main/java/org/onap/dcae/commonFunction/EventPublisherHash.java @@ -0,0 +1,179 @@ +/*- + * ============LICENSE_START======================================================= + * PROJECT + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.dcae.commonFunction; + +import com.att.nsa.cambria.client.CambriaBatchingPublisher; +import com.att.nsa.cambria.client.CambriaClientBuilders; +import com.att.nsa.clock.SaClock; +import com.att.nsa.logging.LoggingContext; +import com.att.nsa.logging.log4j.EcompFields; +import com.google.gson.JsonArray; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class EventPublisherHash { + + private static final String VES_UNIQUE_ID = "VESuniqueId"; + private static EventPublisherHash instance; + private CambriaBatchingPublisher pub; + + private String streamid = ""; + private String ueburl = ""; + private String topic = ""; + private String authuser = ""; + private String authpwd = ""; + + private static Logger log = LoggerFactory.getLogger(EventPublisherHash.class); + + protected static HashMap map = new HashMap(); + + + + public CambriaBatchingPublisher Dmaaphash(String newstreamid) { + pub = null; + streamid = newstreamid; + if (map != null && map.containsKey(streamid)){ + pub = map.get(streamid); + + } + else + { + + try { + ueburl = DmaapPropertyReader.getInstance(CommonStartup.cambriaConfigFile).dmaap_hash + .get(streamid + ".cambria.url"); + + if (ueburl == null) { + ueburl = DmaapPropertyReader.getInstance(CommonStartup.cambriaConfigFile).dmaap_hash + .get(streamid + ".cambria.hosts"); + } + topic = DmaapPropertyReader.getInstance(CommonStartup.cambriaConfigFile) + .getKeyValue(streamid + ".cambria.topic"); + authuser = DmaapPropertyReader.getInstance(CommonStartup.cambriaConfigFile) + .getKeyValue(streamid + ".basicAuthUsername"); + + if (authuser != null) { + authpwd = DmaapPropertyReader.getInstance(CommonStartup.cambriaConfigFile).dmaap_hash + .get(streamid + ".basicAuthPassword"); + } + + + if (authuser != null) { + log.debug(String.format("URL:%sTOPIC:%sAuthUser:%sAuthpwd:%s", ueburl, topic, authuser, authpwd)); + pub = new CambriaClientBuilders.PublisherBuilder().usingHosts(ueburl).onTopic(topic).usingHttps() + .authenticatedByHttp(authuser, authpwd).logSendFailuresAfter(5) + // .logTo(log) + // .limitBatch(100, 10) + .build(); + } else { + + log.debug(String.format("URL:%sTOPIC:%s", ueburl, topic)); + pub = new CambriaClientBuilders.PublisherBuilder().usingHosts(ueburl).onTopic(topic) + // .logTo(log) + .logSendFailuresAfter(5) + // .limitBatch(100, 10) + .build(); + + } + + map.put(streamid, pub); + + + } catch ( Exception e) { + log.error("CambriaClientBuilders connection reader exception : streamID - " + newstreamid + " " + e.getMessage()); + } + + } + return pub; + + } + + /** + * Returns event publisher + * @return event publisher + */ + public static synchronized EventPublisherHash getInstance() { + if (instance == null) { + instance = new EventPublisherHash(); + } + return instance; + + } + + + public synchronized void sendEvent(JSONObject event, String streamid) { + + log.debug("EventPublisher.sendEvent: instance for publish is ready"); + + if (event.has(VES_UNIQUE_ID)) { + String uuid = event.get(VES_UNIQUE_ID).toString(); + LoggingContext localLC = VESLogger.getLoggingContextForThread(uuid); + localLC.put(EcompFields.kBeginTimestampMs, SaClock.now()); + log.debug("Removing VESuniqueid object from event"); + event.remove(VES_UNIQUE_ID); + } + + try { + + + int pendingMsgs = Dmaaphash(streamid).send("MyPartitionKey", event.toString()); + // this.wait(2000); + + if (pendingMsgs > 100) { + log.info("Pending Message Count=" + pendingMsgs); + } + + log.info("pub.send invoked - no error"); + //CommonStartup.oplog.info(String.format("URL:%sTOPIC:%sEvent Published:%s", ueburl, topic, event)); + CommonStartup.oplog.info(String.format("StreamID:%s Event Published:%s ", streamid, event)); + } catch (IOException | IllegalArgumentException e) { + log.error("Unable to publish event: {} streamid: {}. Exception: {}", event, streamid, e); + Dmaaphash(streamid).close(); + map.remove(streamid); + } + + } + + public synchronized void closePublisher() { + + try { + if (pub != null) { + + final List stuck = pub.close(20, TimeUnit.SECONDS); + if (!stuck.isEmpty()) { + log.error(stuck.size() + " messages unsent"); + } + } + } catch (InterruptedException | IOException e) { + log.error("Caught Exception on Close event: {}", e); + } + + } +} + diff --git a/src/test/java/org/onap/dcae/vestest/TestEventProcessor.java b/src/test/java/org/onap/dcae/vestest/TestEventProcessor.java index 31807dba..f11cc905 100644 --- a/src/test/java/org/onap/dcae/vestest/TestEventProcessor.java +++ b/src/test/java/org/onap/dcae/vestest/TestEventProcessor.java @@ -36,10 +36,11 @@ import org.junit.Test; import org.onap.dcae.commonFunction.CommonStartup; import org.onap.dcae.commonFunction.EventProcessor; import org.onap.dcae.commonFunction.EventPublisher; +import org.onap.dcae.commonFunction.EventPublisherHash; import org.onap.dcae.controller.LoadDynamicConfig; import org.onap.dcae.commonFunction.DmaapPropertyReader; - +import com.att.nsa.cambria.client.CambriaBatchingPublisher; import com.google.gson.JsonParser; public class TestEventProcessor { @@ -99,6 +100,74 @@ public class TestEventProcessor { assertEquals(true, flag); } + + @Test + public void testpublisherhashclass() { + + DmaapPropertyReader dr; + EventPublisherHash eph = null; + Boolean flag = false; + eph = EventPublisherHash.getInstance(); + + + if (eph.equals(null)) + { + flag = false; + } + else + { + flag = true; + } + assertEquals(true, flag); + + + } + + @Test + public void testpublisherhashclassload() { + DmaapPropertyReader dr; + EventPublisherHash eph = null; + String testinput = "src/test/resources/testDmaapConfig.json"; + Boolean flag = false; + dr = new DmaapPropertyReader(testinput); + eph = EventPublisherHash.getInstance(); + EventProcessor ec = new EventProcessor(); + ec.event=new org.json.JSONObject(ev); + CommonStartup.cambriaConfigFile="src/test/resources/testDmaapConfig.json"; + CambriaBatchingPublisher pub = eph.Dmaaphash("sec_fault_ueb"); + + if (pub == null || pub.equals(null)) + { + flag = false; + } + else + { + flag = true; + } + assertEquals(true, flag); + + } + + @Test + public void testpublisherhashSend() { + + DmaapPropertyReader dr; + EventPublisherHash eph = null; + String testinput = "src/test/resources/testDmaapConfig.json"; + Boolean flag = true; + dr = new DmaapPropertyReader(testinput); + eph = EventPublisherHash.getInstance(); + + + EventProcessor ec = new EventProcessor(); + ec.event=new org.json.JSONObject(ev); + CommonStartup.cambriaConfigFile="src/test/resources/testDmaapConfig.json"; + eph.sendEvent(ec.event, "sec_fault_ueb"); + + assertEquals(true, flag); + + } } + -- cgit 1.2.3-korg