From 572693a58c108d6174ed6e138e989cd2f1a6b1b0 Mon Sep 17 00:00:00 2001 From: amshegokar Date: Wed, 26 Sep 2018 12:17:33 +0530 Subject: Config from Docker ENV parameters Taking Adapter Config from Docker ENV parameters if CONSUL_HOST flag is present Change-Id: Id097e433112fbdcc0d403ca3f87abee123e76693 Issue-ID: DCAEGEN2-335 Signed-off-by: amshegokar --- .../adapter/UniversalEventAdapter.java | 8 +- .../configs/UniversalEventConfiguration.java | 68 ------- .../onap/universalvesadapter/dmaap/Creator.java | 15 +- .../universalvesadapter/service/DMaapService.java | 8 +- .../service/VESAdapterInitializer.java | 226 +++++++++++++++------ .../universalvesadapter/utils/DmaapConfig.java | 27 ++- 6 files changed, 209 insertions(+), 143 deletions(-) delete mode 100644 UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/configs/UniversalEventConfiguration.java (limited to 'UniversalVesAdapter/src/main/java/org') diff --git a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/adapter/UniversalEventAdapter.java b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/adapter/UniversalEventAdapter.java index 65c7b9c..cd47619 100644 --- a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/adapter/UniversalEventAdapter.java +++ b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/adapter/UniversalEventAdapter.java @@ -57,8 +57,8 @@ public class UniversalEventAdapter implements GenericAdapter { private final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); private String enterpriseId; - @Value("${defaultMappingFileName}") - private String defaulMappingFileName; + @Value("${defaultEnterpriseId}") + private String defaultEnterpriseId; private Map eventToSmooksMapping = new ConcurrentHashMap<>(); public UniversalEventAdapter() { @@ -88,10 +88,10 @@ public class UniversalEventAdapter implements GenericAdapter { if (VESAdapterInitializer.getMappingFiles().containsKey(enterpriseId)) { configFileData = VESAdapterInitializer.getMappingFiles().get(enterpriseId); - LOGGER.debug("Using Mapping file as Mapping file is not available for Enterprise Id:{}",enterpriseId); + LOGGER.debug("Using Mapping file as Mapping file is available for Enterprise Id:{}",enterpriseId); } else { - configFileData = VESAdapterInitializer.getMappingFiles().get(defaulMappingFileName); + configFileData = VESAdapterInitializer.getMappingFiles().get(defaultEnterpriseId); LOGGER.debug("Using Default Mapping file as Mapping file is not available for Enterprise Id:{}",enterpriseId); } diff --git a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/configs/UniversalEventConfiguration.java b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/configs/UniversalEventConfiguration.java deleted file mode 100644 index 6f85ef3..0000000 --- a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/configs/UniversalEventConfiguration.java +++ /dev/null @@ -1,68 +0,0 @@ -/* -* ============LICENSE_START======================================================= -* ONAP : DCAE -* ================================================================================ -* Copyright 2018 TechMahindra -*================================================================================= -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* ============LICENSE_END========================================================= -*/ -package org.onap.universalvesadapter.configs; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * Configuration for universal adapter service - * - * @author kmalbari - * - */ -@Component -public class UniversalEventConfiguration{ - - private final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); - - @Value("${universal.configFiles}") - private String configFiles; - - private Map eventToConfigFileMap = new ConcurrentHashMap<>(); - - - /** - * Returns config file for a particular event from the mapping. - * If mapping is empty, populates its - * - * @param eventType - * - * @return config file name - */ - public String getConfigForEvent(String eventType){ - LOGGER.debug("Getting config file name for event:" + eventType); - if(null != configFiles && eventToConfigFileMap.isEmpty()){ - for(String entry : configFiles.split(",")){ - eventToConfigFileMap.put(entry.split(":")[0], entry.split(":")[1]); - } - LOGGER.debug("Populated mappings for event type to config files"); - } - - return (null == eventToConfigFileMap.get(eventType)) ? "default" : eventToConfigFileMap.get(eventType) ; - - } - - -} diff --git a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/dmaap/Creator.java b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/dmaap/Creator.java index cd76f79..bfd3c6d 100644 --- a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/dmaap/Creator.java +++ b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/dmaap/Creator.java @@ -36,7 +36,7 @@ public class Creator { private final Logger LOGGER = LoggerFactory.getLogger(Creator.class); private DMaaPMRFactory dMaaPMRFactoryInstance; - private String hostname; + private String dmaaphost; private String publisherTopic; private String subcriberTopic; private DmaapConfig dmaapConfig; @@ -46,6 +46,11 @@ public class Creator { this.dmaapConfig = dmaapConfig; } + public DmaapConfig getDmaapConfig() { + return dmaapConfig; + } + + public Creator() { } @@ -54,11 +59,11 @@ public class Creator { public void propertyFileInitializer() { - this.hostname = dmaapConfig.getHostname(); + this.dmaaphost = dmaapConfig.getDmaaphost(); this.publisherTopic = dmaapConfig.getPublisherTopic(); this.subcriberTopic = dmaapConfig.getSubscriberTopic(); this.dMaaPMRFactoryInstance = DMaaPMRFactory.create(); - LOGGER.info("The Hostname of DMaap is :" + hostname); + LOGGER.info("The Hostname of DMaap is :" + dmaaphost); } @@ -68,7 +73,7 @@ public class Creator { propertyFileInitializer(); DMaaPMRPublisherConfig dMaaPMRPublisherConfig = null; try { - dMaaPMRPublisherConfig = new DMaaPMRPublisherConfig.Builder(hostname, publisherTopic,dmaapConfig).build(); + dMaaPMRPublisherConfig = new DMaaPMRPublisherConfig.Builder(dmaaphost, publisherTopic,dmaapConfig).build(); } catch (IOException e) { LOGGER.error("failed or interrupted I/O operations while creating publisher config:{}",e.getCause()); } @@ -80,7 +85,7 @@ public class Creator { propertyFileInitializer(); DMaaPMRSubscriberConfig dMaaPMRSubscriberConfig = null; try { - dMaaPMRSubscriberConfig = new DMaaPMRSubscriberConfig.Builder(hostname, subcriberTopic, dmaapConfig).build(); + dMaaPMRSubscriberConfig = new DMaaPMRSubscriberConfig.Builder(dmaaphost, subcriberTopic, dmaapConfig).build(); } catch (IOException e) { LOGGER.error("failed or interrupted I/O operations while creating subcriber config:{}",e.getCause()); diff --git a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/DMaapService.java b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/DMaapService.java index 04f333e..d07dd04 100644 --- a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/DMaapService.java +++ b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/DMaapService.java @@ -29,9 +29,11 @@ import org.onap.universalvesadapter.dmaap.MRPublisher.DMaaPMRPublisher; import org.onap.universalvesadapter.dmaap.MRSubcriber.DMaaPMRSubscriber; import org.onap.universalvesadapter.exception.DMaapException; import org.onap.universalvesadapter.exception.VesException; +import org.onap.universalvesadapter.utils.DmaapConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component @@ -41,6 +43,8 @@ public class DMaapService { private static List list = new LinkedList(); @Autowired private UniversalEventAdapter eventAdapter; + @Autowired + private DmaapConfig dmaapConfig; /** * It fetches events from DMaap in JSON, transforms JSON to VES format and @@ -53,6 +57,8 @@ public class DMaapService { throws InterruptedException { LOGGER.info("fetch and publish from and to Dmaap started"); + int pollingInternalInt=dmaapConfig.getPollingInterval(); + LOGGER.info("The Polling Interval in Milli Second is :" +pollingInternalInt); while (true) { synchronized (this) { for (String incomingJsonString : dMaaPMRSubscriber.fetchMessages().getFetchedMessages()) { @@ -61,7 +67,7 @@ public class DMaapService { } if (list.size() == 0) { - Thread.sleep(2000); + Thread.sleep(pollingInternalInt); } LOGGER.debug("number of messages to be converted :{}", list.size()); diff --git a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/VESAdapterInitializer.java b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/VESAdapterInitializer.java index b63eb1e..424d40e 100644 --- a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/VESAdapterInitializer.java +++ b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/service/VESAdapterInitializer.java @@ -20,105 +20,136 @@ package org.onap.universalvesadapter.service; import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.HashMap; import java.util.Map; - import org.apache.commons.codec.binary.Hex; +import org.onap.universalvesadapter.dmaap.Creator; +import org.onap.universalvesadapter.utils.DmaapConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ApplicationContext; import org.springframework.core.Ordered; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; //AdapterInitializer @Component public class VESAdapterInitializer implements CommandLineRunner, Ordered { private static final Logger LOGGER = LoggerFactory.getLogger(VESAdapterInitializer.class); + + @Autowired + private Creator creator; + @Autowired + private DmaapConfig dmaapConfig; @Value("${spring.datasource.url}") String dBurl; @Value("${spring.datasource.username}") String user; @Value("${spring.datasource.password}") String pwd; + @Value("${defaultMappingFilelocation}") + String defaultMappingFileLocation; + @Value("${MappingFileTableName}") + String MappingFileTableName; + @Value("${defaultEnterpriseId}") + String defaultEnterpriseId; + @Value("${server.port}") + String serverPort; + @Autowired + private JdbcTemplate jdbcTemplate; + private static Map mappingFiles = new HashMap(); private static Map env; private static String url; public static String retString; public static String retCBSString; - + public static String configFile = "/opt/app/KV-Configuration.json"; + byte[] bytesArray = null; + @Autowired private ApplicationContext applicationContext; + @Override public void run(String... args) throws Exception { - - fetchMappingFile(); - getconsul(); - - } - - private void getconsul() { - env = System.getenv(); for (Map.Entry entry : env.entrySet()) { LOGGER.info(entry.getKey() + ":" + entry.getValue()); } - if (env.containsKey("CONSUL_HOST") && env.containsKey("CONFIG_BINDING_SERVICE")) { - LOGGER.info(">>>Dynamic configuration to be fetched from ConfigBindingService"); - url = env.get("CONSUL_HOST") + ":8500/v1/catalog/service/" + env.get("CONFIG_BINDING_SERVICE"); - - retString = executecurl(url); - - } else { - - LOGGER.info(">>>Static configuration to be used"); - final String url = "http://localhost:8085/start"; - final String USER_AGENT = "Mozilla/5.0"; - - try { - URL obj = new URL(url); - HttpURLConnection httpURLConnection = (HttpURLConnection) obj.openConnection(); - - // optional default is GET - httpURLConnection.setRequestMethod("GET"); - - // add request header - httpURLConnection.setRequestProperty("User-Agent", USER_AGENT); - - int responseCode = httpURLConnection.getResponseCode(); - LOGGER.info("Sending 'GET' request to URL : " + url); - LOGGER.info("Response Code : " + responseCode); - BufferedReader in = new BufferedReader(new InputStreamReader(httpURLConnection.getInputStream())); - String inputLine; - StringBuffer response = new StringBuffer(); - - while ((inputLine = in.readLine()) != null) { - response.append(inputLine); - } - in.close(); - - // print result - LOGGER.info("The result is :" + response.toString()); - - } catch (Exception e) { - LOGGER.error("Error occured due to :" + e.getMessage()); - e.printStackTrace(); + if (env.containsKey("CONSUL_HOST") && env.containsKey("CONFIG_BINDING_SERVICE") && env.containsKey("HOSTNAME")) { + //TODO - Add logic to talk to Consul and CBS to get the configuration. For now, we will refer to configuration coming from docker env parameters + + LOGGER.info(">>>Dynamic configuration to be used"); + + if( (env.get("MR_DMAAPHOST")==null || + (env.get("MR_DMAAPHOST")==null || + (env.get("MR_DEFAULT_PORT_NUMBER")==null || + (env.get("URL_JDBC")==null || + (env.get("JDBC_USERNAME")==null || + (env.get("JDBC_PASSWORD")==null ))))))) { + + + LOGGER.error("Some docker environment parameter is missing. Sample Usage is -\n sudo docker run -d -p 8085:8085/tcp --env URL_JDBC=jdbc:postgresql://10.53.172.129:5432/dummy --env JDBC_USERNAME=ngpuser --env JDBC_PASSWORD=root --env MR_DMAAPHOST=10.10.10.10 --env MR_DEFAULT_PORT_NUMBER=3904 --env CONSUL_HOST=10.53.172.109 --env HOSTNAME=mvp-dcaegen2-collectors-ves --env CONFIG_BINDING_SERVICE=config_binding_service -e DMAAPHOST='10.53.172.156' onap/org.onap.dcaegen2.services.mapper.vesadapter.universalvesadaptor:latest"); + System.exit(SpringApplication.exit(applicationContext, () -> {LOGGER.error("Application stoped due to missing default mapping file");return-1;})); + + }else { + + + + dmaapConfig.setDmaaphost(env.get("MR_DMAAPHOST")); + dmaapConfig.setDEFAULT_PORT_NUMBER(Integer.parseInt(env.get("MR_DEFAULT_PORT_NUMBER"))); + creator.setDmaapConfig(dmaapConfig); + + dBurl=env.get("URL_JDBC"); + user=env.get("JDBC_USERNAME"); + pwd=env.get("JDBC_PASSWORD"); } + + + } else { + LOGGER.info(">>>Static configuration to be used"); } + prepareDatabase(); + fetchMappingFile(); + + LOGGER.info("Triggering controller's start url "); + executecurl("http://localhost:"+serverPort+"/start"); + } + private void getconsul() { + + //TODO + } + + public static boolean verifyConfigChange() { + //TODO + return false; } - private String executecurl(String url) { + public static void getCBS() { + //TODO + } + public static void writefile(String retCBSString) { + //TODO + } + + private static String executecurl(String url) { + + LOGGER.info("Running curl command for url:"+url); String[] command = { "curl", "-v", url }; ProcessBuilder process = new ProcessBuilder(command); Process p; @@ -147,9 +178,11 @@ public class VESAdapterInitializer implements CommandLineRunner, Ordered { } public void fetchMappingFile() { - String query = "SELECT * FROM mapping_file"; - try (Connection con = DriverManager.getConnection(dBurl, user, pwd);PreparedStatement pstmt = con.prepareStatement(query);ResultSet rs = pstmt.executeQuery();) { + + try (Connection con = DriverManager.getConnection(dBurl, user, pwd)) { LOGGER.info("Retrieving data from DB"); + PreparedStatement pstmt = con.prepareStatement("SELECT * FROM mapping_file"); + ResultSet rs = pstmt.executeQuery(); // parsing the column each time is a linear search int column1Pos = rs.findColumn("enterpriseid"); int column2Pos = rs.findColumn("mappingfilecontents"); @@ -162,14 +195,91 @@ public class VESAdapterInitializer implements CommandLineRunner, Ordered { String data = new String(bytes, "UTF-8"); mappingFiles.put(column1, data); } - LOGGER.info("DB Initialization Completed..." + mappingFiles.size()); + LOGGER.info("DB Initialization Completed, Total # Mappingfiles are" + mappingFiles.size()); } catch (Exception e) { LOGGER.error("Error occured due to :" + e.getMessage()); e.printStackTrace(); } } - + + + private void prepareDatabase() throws IOException { + + + LOGGER.info("The Default Mapping file Location:"+defaultMappingFileLocation.trim()); + + if(ClassLoader.getSystemResource(defaultMappingFileLocation.trim())==null){ + LOGGER.error("Default mapping file " + defaultMappingFileLocation.trim() + " is missing"); + System.exit(SpringApplication.exit(applicationContext, () -> {LOGGER.error("Application stoped due to missing default mapping file");return-1;})); + } + + File file = new File(ClassLoader.getSystemResource(defaultMappingFileLocation.trim()).getFile()); + FileInputStream fileInputStream=null; + + try + { + bytesArray = new byte[(int) file.length()]; + fileInputStream = new FileInputStream(file); + fileInputStream.read(bytesArray); + + } catch (IOException e1) { + LOGGER.error("Exception Occured while reading the default mapping file ,Cause: " + e1.getMessage(), e1); + //exit on missing default mapping file + System.exit(SpringApplication.exit(applicationContext, () -> {LOGGER.error("Application stoped due to missing default mapping file");return-1;})); + }finally { + try { + fileInputStream.close(); + }catch (IOException e) { + LOGGER.error("Exception while closing file inputstream" + e.getMessage(), e); + } + } + + try (Connection con = DriverManager.getConnection(dBurl, user, pwd)) { + LOGGER.info("Postgresql Connection successful..."); + LOGGER.debug("1Connection object:"+con.toString()); + //creating table if not exist + PreparedStatement pstmt11=con.prepareStatement("CREATE TABLE IF NOT EXISTS public."+MappingFileTableName+"\r\n" + + "(\r\n" + + " enterpriseid character varying COLLATE pg_catalog.\"default\" NOT NULL,\r\n" + + " mappingfilecontents bytea,\r\n" + + " mimetype character varying COLLATE pg_catalog.\"default\",\r\n" + + " file_name character varying COLLATE pg_catalog.\"default\",\r\n" + + " CONSTRAINT mapping_file_pkey5 PRIMARY KEY (enterpriseid)\r\n" + + ")\r\n" + + "WITH (\r\n" + + " OIDS = FALSE\r\n" + + ")\r\n" + + "TABLESPACE pg_default;"); + pstmt11.executeUpdate(); + LOGGER.info("CREATE TABLE IF NOT EXISTS executed successfully...."); + + if((bytesArray.length>0)&&(!bytesArray.toString().equals(""))) { + LOGGER.debug("2Connection object:"+con.toString()); + PreparedStatement pstmt=con.prepareStatement("INSERT INTO "+MappingFileTableName+"(enterpriseid, mappingfilecontents, mimetype, File_Name) VALUES (?, ?, ?, ?) ON CONFLICT (enterpriseid) DO NOTHING;"); + pstmt.setString(1,defaultEnterpriseId); + pstmt.setBytes(2,bytesArray ); + pstmt.setString(3,"text/xml"); + pstmt.setString(4, file.getName()); + + pstmt.executeUpdate(); + LOGGER.info("Made sure that default mapping file is present in table"); + } + else { + LOGGER.error(file.getName()+" is empty"); + //exit on empty mapping file + System.exit(SpringApplication.exit(applicationContext, () -> {LOGGER.error("Application stoped beacuase default mapping file is empty..");return-1;})); + } + + + + } catch (SQLException e) { + LOGGER.error("Received exception : " + e.getMessage(), e); + //exit on SqlException + System.exit(SpringApplication.exit(applicationContext, () -> {LOGGER.error("Application Stoped due to ",e.getCause());return-1;})); + } + + } public static Map getMappingFiles() { return mappingFiles; } @@ -180,7 +290,7 @@ public class VESAdapterInitializer implements CommandLineRunner, Ordered { @Override public int getOrder() { - return 1; + return 0; } } diff --git a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/utils/DmaapConfig.java b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/utils/DmaapConfig.java index 34bbc8e..11bba61 100644 --- a/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/utils/DmaapConfig.java +++ b/UniversalVesAdapter/src/main/java/org/onap/universalvesadapter/utils/DmaapConfig.java @@ -30,15 +30,19 @@ import org.springframework.stereotype.Component; @ConfigurationProperties public class DmaapConfig { - @Value("${mr.hostname}") + @Value("${mr.dmaaphost}") @NotEmpty - private String hostname; + private String dmaaphost; // default port number @Value("${mr.DEFAULT_PORT_NUMBER}") @NotEmpty private int DEFAULT_PORT_NUMBER; + @Value("${mr.POLLING_INTERVAL}") + @NotEmpty + private int pollingInterval; + // default to no username @Value("${mr.DEFAULT_USER_NAME}") private String DEFAULT_USER_NAME; @@ -124,12 +128,12 @@ public class DmaapConfig { @NotEmpty private String subcriberSUBSCRIBER_MSG_LIMIT_QUERY_PARAM_NAME; - public void setHostname(String hostname) { - this.hostname = hostname; + public void setDmaaphost(String dmaaphost) { + this.dmaaphost = dmaaphost; } - - public String getHostname() { - return hostname; + + public String getDmaaphost() { + return dmaaphost; } public int getDEFAULT_PORT_NUMBER() { @@ -285,4 +289,13 @@ public class DmaapConfig { this.subcriberSUBSCRIBER_MSG_LIMIT_QUERY_PARAM_NAME = subcriberSUBSCRIBER_MSG_LIMIT_QUERY_PARAM_NAME; } + public int getPollingInterval() { + return pollingInterval; + } + + public void setPollingInterval(int pollingInterval) { + this.pollingInterval = pollingInterval; + } + + } -- cgit 1.2.3-korg