diff options
Diffstat (limited to 'components/datalake-handler/des')
11 files changed, 227 insertions, 258 deletions
diff --git a/components/datalake-handler/des/Dockerfile b/components/datalake-handler/des/Dockerfile index e5a0af27..584ba9d6 100644 --- a/components/datalake-handler/des/Dockerfile +++ b/components/datalake-handler/des/Dockerfile @@ -1,5 +1,6 @@ # ============LICENSE_START=================================================== # Copyright (C) 2020 China Mobile. +# Copyright (C) 2021 Wipro Limited # ============================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,15 +16,17 @@ # # SPDX-License-Identifier: Apache-2.0 # ============LICENSE_END===================================================== -FROM openjdk:11-jre-slim + +FROM nexus3.onap.org:10001/onap/integration-java11:8.0.0 MAINTAINER Kai Lu <lukai@chinamobile.com> EXPOSE 1681 -RUN groupadd -r datalake && useradd -r -g datalake datalake -RUN mkdir /home/datalake/ && \ - mkdir /home/datalake/db_init +USER root + +RUN addgroup datalake && adduser -G datalake -h /home/datalake -D datalake +RUN mkdir /home/datalake/db_init WORKDIR /home/datalake @@ -41,10 +44,8 @@ WORKDIR /home/datalake RUN chmod -R 0755 ./* && \ chown -R datalake:datalake /home/datalake -RUN apt update && \ - apt install -y mariadb-client && \ - apt install -y curl +RUN apk --update add postgresql-client curl -#USER datalake -CMD ["sh", "run.sh"] +USER datalake +ENTRYPOINT /home/datalake/run.sh diff --git a/components/datalake-handler/des/pom.xml b/components/datalake-handler/des/pom.xml index 012c346f..e23c0c72 100644 --- a/components/datalake-handler/des/pom.xml +++ b/components/datalake-handler/des/pom.xml @@ -25,7 +25,7 @@ <parent> <groupId>org.onap.dcaegen2.services.components</groupId> <artifactId>datalake-handler</artifactId> - <version>1.1.0-SNAPSHOT</version> + <version>1.1.1-SNAPSHOT</version> </parent> <artifactId>des</artifactId> @@ -60,9 +60,9 @@ </dependency> <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - <version>2.4.1</version> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>42.2.18</version> </dependency> <dependency> @@ -260,6 +260,9 @@ <goals> <goal>build</goal> </goals> + <configuration> + <useMavenSettingsForAuth>true</useMavenSettingsForAuth> + </configuration> </execution> <execution> <id>tag-and-push-image-latest</id> diff --git a/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db.sql b/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db.sql index e71093aa..9b391d0f 100644 --- a/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db.sql +++ b/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db.sql @@ -1,9 +1,9 @@ - /* * ============LICENSE_START======================================================= * ONAP : DATALAKE * ================================================================================ * Copyright 2020 China Mobile +* Copyright (C) 2021 Wipro Limited *================================================================================= * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,126 +18,146 @@ * limitations under the License. * ============LICENSE_END========================================================= */ -drop DATABASE datalake; -create database datalake; -use datalake; -CREATE TABLE `topic_name` ( - `id` varchar(255) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `db_type` ( - `id` varchar(255) NOT NULL, - `default_port` int(11) DEFAULT NULL, - `name` varchar(255) NOT NULL, - `tool` bit(1) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `db` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `database_name` varchar(255) DEFAULT NULL, - `enabled` bit(1) NOT NULL, - `encrypt` bit(1) DEFAULT NULL, - `host` varchar(255) DEFAULT NULL, - `login` varchar(255) DEFAULT NULL, - `name` varchar(255) DEFAULT NULL, - `pass` varchar(255) DEFAULT NULL, - `port` int(11) DEFAULT NULL, - `property1` varchar(255) DEFAULT NULL, - `property2` varchar(255) DEFAULT NULL, - `property3` varchar(255) DEFAULT NULL, - `db_type_id` varchar(255) NOT NULL, - `presto_catalog` varchar(255) DEFAULT NULL, - PRIMARY KEY (`id`), - KEY `FK3njadtw43ieph7ftt4kxdhcko` (`db_type_id`), - CONSTRAINT `FK3njadtw43ieph7ftt4kxdhcko` FOREIGN KEY (`db_type_id`) REFERENCES `db_type` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `design_type` ( - `id` varchar(255) NOT NULL, - `name` varchar(255) DEFAULT NULL, - `note` varchar(255) DEFAULT NULL, - `db_type_id` varchar(255) NOT NULL, - PRIMARY KEY (`id`), - KEY `FKm8rkv2qkq01gsmeq1c3y4w02x` (`db_type_id`), - CONSTRAINT `FKm8rkv2qkq01gsmeq1c3y4w02x` FOREIGN KEY (`db_type_id`) REFERENCES `db_type` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `design` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `body` text DEFAULT NULL, - `name` varchar(255) DEFAULT NULL, - `note` varchar(255) DEFAULT NULL, - `submitted` bit(1) DEFAULT NULL, - `design_type_id` varchar(255) NOT NULL, - `topic_name_id` varchar(255) NOT NULL, - PRIMARY KEY (`id`), - KEY `FKo43yi6aputq6kwqqu8eqbspm5` (`design_type_id`), - KEY `FKabb8e74230glxpaiai4aqsr34` (`topic_name_id`), - CONSTRAINT `FKabb8e74230glxpaiai4aqsr34` FOREIGN KEY (`topic_name_id`) REFERENCES `topic_name` (`id`), - CONSTRAINT `FKo43yi6aputq6kwqqu8eqbspm5` FOREIGN KEY (`design_type_id`) REFERENCES `design_type` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `kafka` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `broker_list` varchar(255) NOT NULL, - `consumer_count` int(11) DEFAULT 3, - `enabled` bit(1) NOT NULL, - `excluded_topic` varchar(1023) DEFAULT '__consumer_offsets,__transaction_state', - `group` varchar(255) DEFAULT 'datalake', - `included_topic` varchar(255) DEFAULT NULL, - `login` varchar(255) DEFAULT NULL, - `name` varchar(255) NOT NULL, - `pass` varchar(255) DEFAULT NULL, - `secure` bit(1) DEFAULT b'0', - `security_protocol` varchar(255) DEFAULT NULL, - `timeout_sec` int(11) DEFAULT 10, - `zk` varchar(255) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `topic` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `aggregate_array_path` varchar(255) DEFAULT NULL, - `correlate_cleared_message` bit(1) NOT NULL DEFAULT b'0', - `data_format` varchar(255) DEFAULT NULL, - `enabled` bit(1) NOT NULL, - `flatten_array_path` varchar(255) DEFAULT NULL, - `login` varchar(255) DEFAULT NULL, - `message_id_path` varchar(255) DEFAULT NULL, - `pass` varchar(255) DEFAULT NULL, - `save_raw` bit(1) NOT NULL DEFAULT b'0', - `ttl_day` int(11) DEFAULT NULL, - `topic_name_id` varchar(255) NOT NULL, - PRIMARY KEY (`id`), - KEY `FKj3pldlfaokdhqjfva8n3pkjca` (`topic_name_id`), - CONSTRAINT `FKj3pldlfaokdhqjfva8n3pkjca` FOREIGN KEY (`topic_name_id`) REFERENCES `topic_name` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `map_db_design` ( - `design_id` int(11) NOT NULL, - `db_id` int(11) NOT NULL, - PRIMARY KEY (`design_id`,`db_id`), - KEY `FKhpn49r94k05mancjtn301m2p0` (`db_id`), - CONSTRAINT `FKfli240v96cfjbnmjqc0fvvd57` FOREIGN KEY (`design_id`) REFERENCES `design` (`id`), - CONSTRAINT `FKhpn49r94k05mancjtn301m2p0` FOREIGN KEY (`db_id`) REFERENCES `db` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `map_db_topic` ( - `topic_id` int(11) NOT NULL, - `db_id` int(11) NOT NULL, - PRIMARY KEY (`db_id`,`topic_id`), - KEY `FKq1jon185jnrr7dv1dd8214uw0` (`topic_id`), - CONSTRAINT `FKirro29ojp7jmtqx9m1qxwixcc` FOREIGN KEY (`db_id`) REFERENCES `db` (`id`), - CONSTRAINT `FKq1jon185jnrr7dv1dd8214uw0` FOREIGN KEY (`topic_id`) REFERENCES `topic` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `map_kafka_topic` ( - `kafka_id` int(11) NOT NULL, - `topic_id` int(11) NOT NULL, - PRIMARY KEY (`topic_id`,`kafka_id`), - KEY `FKtdrme4h7rxfh04u2i2wqu23g5` (`kafka_id`), - CONSTRAINT `FK5q7jdxy54au5rcrhwa4a5igqi` FOREIGN KEY (`topic_id`) REFERENCES `topic` (`id`), - CONSTRAINT `FKtdrme4h7rxfh04u2i2wqu23g5` FOREIGN KEY (`kafka_id`) REFERENCES `kafka` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE `data_exposure` ( - `id` varchar(255) NOT NULL, - `note` varchar(255) DEFAULT NULL, - `sql_template` varchar(10000) NOT NULL, - `db_id` int(11) NOT NULL, - PRIMARY KEY (`id`), - KEY `FKf5ps4jxauwawk4ac86t5t6xev` (`db_id`), - CONSTRAINT `FKf5ps4jxauwawk4ac86t5t6xev` FOREIGN KEY (`db_id`) REFERENCES `db` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +SELECT 'CREATE DATABASE datalake' +WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'datalake')\gexec + +\c datalake + +CREATE TABLE topic_name ( + id varchar(255) NOT NULL, + PRIMARY KEY (id) +); +CREATE TABLE db_type ( + id varchar(255) NOT NULL, + default_port int DEFAULT NULL, + name varchar(255) NOT NULL, + tool bit(1) NOT NULL, + PRIMARY KEY (id) +); +CREATE SEQUENCE db_seq; + +CREATE TABLE db ( + id int NOT NULL DEFAULT NEXTVAL ('db_seq'), + database_name varchar(255) DEFAULT NULL, + enabled bit(1) NOT NULL, + encrypt bit(1) DEFAULT NULL, + host varchar(255) DEFAULT NULL, + login varchar(255) DEFAULT NULL, + name varchar(255) DEFAULT NULL, + pass varchar(255) DEFAULT NULL, + port int DEFAULT NULL, + property1 varchar(255) DEFAULT NULL, + property2 varchar(255) DEFAULT NULL, + property3 varchar(255) DEFAULT NULL, + db_type_id varchar(255) NOT NULL, + presto_catalog varchar(255) DEFAULT NULL, + PRIMARY KEY (id), + CONSTRAINT FK3njadtw43ieph7ftt4kxdhcko FOREIGN KEY (db_type_id) REFERENCES db_type (id) +); + +CREATE INDEX FK3njadtw43ieph7ftt4kxdhcko ON db (db_type_id); +CREATE TABLE design_type ( + id varchar(255) NOT NULL, + name varchar(255) DEFAULT NULL, + note varchar(255) DEFAULT NULL, + db_type_id varchar(255) NOT NULL, + PRIMARY KEY (id), + CONSTRAINT FKm8rkv2qkq01gsmeq1c3y4w02x FOREIGN KEY (db_type_id) REFERENCES db_type (id) +); + +CREATE INDEX FKm8rkv2qkq01gsmeq1c3y4w02x ON design_type (db_type_id); +CREATE SEQUENCE design_seq; + +CREATE TABLE design ( + id int NOT NULL DEFAULT NEXTVAL ('design_seq'), + body text DEFAULT NULL, + name varchar(255) DEFAULT NULL, + note varchar(255) DEFAULT NULL, + submitted bit(1) DEFAULT NULL, + design_type_id varchar(255) NOT NULL, + topic_name_id varchar(255) NOT NULL, + PRIMARY KEY (id), + CONSTRAINT FKabb8e74230glxpaiai4aqsr34 FOREIGN KEY (topic_name_id) REFERENCES topic_name (id), + CONSTRAINT FKo43yi6aputq6kwqqu8eqbspm5 FOREIGN KEY (design_type_id) REFERENCES design_type (id) +); + +CREATE INDEX FKo43yi6aputq6kwqqu8eqbspm5 ON design (design_type_id); +CREATE INDEX FKabb8e74230glxpaiai4aqsr34 ON design (topic_name_id); +CREATE SEQUENCE kafka_seq; + +CREATE TABLE kafka ( + id int NOT NULL DEFAULT NEXTVAL ('kafka_seq'), + broker_list varchar(255) NOT NULL, + consumer_count int DEFAULT 3, + enabled bit(1) NOT NULL, + excluded_topic varchar(1023) DEFAULT '__consumer_offsets,__transaction_state', + "group" varchar(255) DEFAULT 'datalake', + included_topic varchar(255) DEFAULT NULL, + login varchar(255) DEFAULT NULL, + name varchar(255) NOT NULL, + pass varchar(255) DEFAULT NULL, + secure bit(1) DEFAULT b'0', + security_protocol varchar(255) DEFAULT NULL, + timeout_sec int DEFAULT 10, + zk varchar(255) NOT NULL, + PRIMARY KEY (id) +); +CREATE SEQUENCE topic_seq; + +CREATE TABLE topic ( + id int NOT NULL DEFAULT NEXTVAL ('topic_seq'), + aggregate_array_path varchar(255) DEFAULT NULL, + correlate_cleared_message bit(1) NOT NULL DEFAULT b'0', + data_format varchar(255) DEFAULT NULL, + enabled bit(1) NOT NULL, + flatten_array_path varchar(255) DEFAULT NULL, + login varchar(255) DEFAULT NULL, + message_id_path varchar(255) DEFAULT NULL, + pass varchar(255) DEFAULT NULL, + save_raw bit(1) NOT NULL DEFAULT b'0', + ttl_day int DEFAULT NULL, + topic_name_id varchar(255) NOT NULL, + PRIMARY KEY (id), + CONSTRAINT FKj3pldlfaokdhqjfva8n3pkjca FOREIGN KEY (topic_name_id) REFERENCES topic_name (id) +); + +CREATE INDEX FKj3pldlfaokdhqjfva8n3pkjca ON topic (topic_name_id); +CREATE TABLE map_db_design ( + design_id int NOT NULL, + db_id int NOT NULL, + PRIMARY KEY (design_id,db_id), + CONSTRAINT FKfli240v96cfjbnmjqc0fvvd57 FOREIGN KEY (design_id) REFERENCES design (id), + CONSTRAINT FKhpn49r94k05mancjtn301m2p0 FOREIGN KEY (db_id) REFERENCES db (id) +); + +CREATE INDEX FKhpn49r94k05mancjtn301m2p0 ON map_db_design (db_id); +CREATE TABLE map_db_topic ( + topic_id int NOT NULL, + db_id int NOT NULL, + PRIMARY KEY (db_id,topic_id), + CONSTRAINT FKirro29ojp7jmtqx9m1qxwixcc FOREIGN KEY (db_id) REFERENCES db (id), + CONSTRAINT FKq1jon185jnrr7dv1dd8214uw0 FOREIGN KEY (topic_id) REFERENCES topic (id) +); + +CREATE INDEX FKq1jon185jnrr7dv1dd8214uw0 ON map_db_topic (topic_id); +CREATE TABLE map_kafka_topic ( + kafka_id int NOT NULL, + topic_id int NOT NULL, + PRIMARY KEY (topic_id,kafka_id), + CONSTRAINT FK5q7jdxy54au5rcrhwa4a5igqi FOREIGN KEY (topic_id) REFERENCES topic (id), + CONSTRAINT FKtdrme4h7rxfh04u2i2wqu23g5 FOREIGN KEY (kafka_id) REFERENCES kafka (id) +); + +CREATE INDEX FKtdrme4h7rxfh04u2i2wqu23g5 ON map_kafka_topic (kafka_id); +CREATE TABLE data_exposure ( + id varchar(255) NOT NULL, + note varchar(255) DEFAULT NULL, + sql_template varchar(10000) NOT NULL, + db_id int NOT NULL, + PRIMARY KEY (id), + CONSTRAINT FKf5ps4jxauwawk4ac86t5t6xev FOREIGN KEY (db_id) REFERENCES db (id) +); + +CREATE INDEX FKf5ps4jxauwawk4ac86t5t6xev ON data_exposure (db_id); + diff --git a/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db_data.sql b/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db_data.sql index 234351fb..0e80e088 100644 --- a/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db_data.sql +++ b/components/datalake-handler/des/src/assembly/init_db/db_scripts/init_db_data.sql @@ -4,6 +4,7 @@ * ONAP : DATALAKE * ================================================================================ * Copyright 2020 China Mobile +* Copyright (C) 2021 Wipro Limited *================================================================================= * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,12 +19,13 @@ * limitations under the License. * ============LICENSE_END========================================================= */ -INSERT INTO datalake.kafka( +\c datalake +INSERT INTO kafka( id ,name ,consumer_count ,enabled - ,`group` + ,"group" ,broker_list ,included_topic ,login @@ -34,62 +36,62 @@ INSERT INTO datalake.kafka( ,zk ) VALUES ( 1 - ,'main Kafka cluster' -- name - IN varchar(255) + ,'main Kafka cluster' -- name - IN varchar (255) ,3 -- consumer_count - IN int(11) - ,1 -- enabled - IN bit(1) + ,B'1' -- enabled - IN bit(1) ,'dlgroup' -- group - IN varchar(255) ,'message-router-kafka:9092' -- host_port - IN varchar(255) ,'' -- included_topic - IN varchar(255) ,'admin' -- login - IN varchar(255) ,'admin-secret' -- pass - IN varchar(255) - ,0 -- secure - IN bit(1) + ,B'0' -- secure - IN bit(1) ,'SASL_PLAINTEXT' -- security_protocol - IN varchar(255) ,10 -- timeout_sec - IN int(11) ,'message-router-zookeeper:2181' -- zk - IN varchar(255) ); -insert into db_type (`id`, `name`, tool) values ('CB', 'Couchbase', false); -insert into db_type (`id`, `name`, tool) values ('ES', 'Elasticsearch', false); -insert into db_type (`id`, `name`, tool,`default_port`) values ('MONGO', 'MongoDB', false, 27017); -insert into db_type (`id`, `name`, tool) values ('DRUID', 'Druid', false); -insert into db_type (`id`, `name`, tool) values ('HDFS', 'HDFS', false); -insert into db_type (`id`, `name`, tool) values ('KIBANA', 'Kibana', true); -insert into db_type (`id`, `name`, tool) values ('SUPERSET', 'Apache Superset', true); -insert into db (id, db_type_id, enabled, encrypt, `name`,`host`,`login`,`pass`,`database_name`) values (1, 'CB', true, true, 'Couchbase 1','dl-couchbase','dl','dl1234','datalake'); -insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (2, 'ES', true, true, 'Elasticsearch','dl-es'); -insert into db (id, db_type_id, enabled, encrypt, `name`,`host`,`port`,`database_name`,`presto_catalog`) values (3, 'MONGO', true, true, 'MongoDB 1','dl-mongodb',27017,'datalake','mongodb'); -insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (4, 'DRUID', true, true, 'Druid','dl-druid'); -insert into db (id, db_type_id, enabled, encrypt, `name`,`host`,`login`) values (5, 'HDFS', true, true, 'Hadoop Cluster','dl-hdfs','dl'); -insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (6, 'KIBANA', true, false, 'Kibana demo','dl-es'); -insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (7, 'SUPERSET', true, false, 'Superset demo','dl-druid'); +insert into db_type (id, name, tool) values ('CB', 'Couchbase', B'0'); +insert into db_type (id, name, tool) values ('ES', 'Elasticsearch', B'0'); +insert into db_type (id, name, tool,default_port) values ('MONGO', 'MongoDB', B'0', 27017); +insert into db_type (id, name, tool) values ('DRUID', 'Druid', B'0'); +insert into db_type (id, name, tool) values ('HDFS', 'HDFS', B'0'); +insert into db_type (id, name, tool) values ('KIBANA', 'Kibana', B'1'); +insert into db_type (id, name, tool) values ('SUPERSET', 'Apache Superset', B'1'); +insert into db (id, db_type_id, enabled, encrypt, name,host,login,pass,database_name) values (1, 'CB', B'1', B'1', 'Couchbase 1','dl-couchbase','dl','dl1234','datalake'); +insert into db (id, db_type_id, enabled, encrypt, name,host) values (2, 'ES', B'1', B'1', 'Elasticsearch','dl-es'); +insert into db (id, db_type_id, enabled, encrypt, name,host,port,database_name,presto_catalog) values (3, 'MONGO', B'1', B'1', 'MongoDB 1','dl-mongodb',27017,'datalake','mongodb'); +insert into db (id, db_type_id, enabled, encrypt, name,host) values (4, 'DRUID', B'1',B'1', 'Druid','dl-druid'); +insert into db (id, db_type_id, enabled, encrypt, name,host,login) values (5, 'HDFS', B'1', B'1', 'Hadoop Cluster','dl-hdfs','dl'); +insert into db (id, db_type_id, enabled, encrypt, name,host) values (6, 'KIBANA', B'1', B'0', 'Kibana demo','dl-es'); +insert into db (id, db_type_id, enabled, encrypt, name,host) values (7, 'SUPERSET', B'1', B'0', 'Superset demo','dl-druid'); insert into topic_name (id) values ('_DL_DEFAULT_'); insert into topic_name (id) values ('unauthenticated.SEC_FAULT_OUTPUT'); insert into topic_name (id) values ('unauthenticated.VES_MEASUREMENT_OUTPUT'); insert into topic_name (id) values ('EPC'); insert into topic_name (id) values ('HW'); -- in production, default enabled should be off -insert into `topic`(id, `topic_name_id`,`enabled`,`save_raw`,`ttl_day`,`data_format`) values (1, '_DL_DEFAULT_',1,0,3650,'JSON'); -insert into `topic`(id, `topic_name_id`,correlate_cleared_message,`enabled`, message_id_path,`data_format`) -values (2, 'unauthenticated.SEC_FAULT_OUTPUT',1,1,'/event/commonEventHeader/eventName,/event/commonEventHeader/reportingEntityName,/event/faultFields/specificProblem,/event/commonEventHeader/eventId','JSON'); -insert into `topic`(id, `topic_name_id`,`enabled`, aggregate_array_path,flatten_array_path,`data_format`) -values (3, 'unauthenticated.VES_MEASUREMENT_OUTPUT',1, +insert into topic(id, topic_name_id,enabled,save_raw,ttl_day,data_format) values (1, '_DL_DEFAULT_',B'1',B'0',3650,'JSON'); +insert into topic(id, topic_name_id,correlate_cleared_message,enabled, message_id_path,data_format) +values (2, 'unauthenticated.SEC_FAULT_OUTPUT',B'1',B'1','/event/commonEventHeader/eventName,/event/commonEventHeader/reportingEntityName,/event/faultFields/specificProblem,/event/commonEventHeader/eventId','JSON'); +insert into topic(id, topic_name_id,enabled, aggregate_array_path,flatten_array_path,data_format) +values (3, 'unauthenticated.VES_MEASUREMENT_OUTPUT',B'1', '/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray', '/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface', 'JSON'); -insert into `topic`(id, `topic_name_id`,`enabled`, flatten_array_path,`data_format`) -values (4, 'EPC',1, '/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface', 'JSON'); -insert into `topic`(id, `topic_name_id`,`enabled`, aggregate_array_path,`data_format`) -values (5, 'HW',1, +insert into topic(id, topic_name_id,enabled, flatten_array_path,data_format) +values (4, 'EPC',B'1', '/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface', 'JSON'); +insert into topic(id, topic_name_id,enabled, aggregate_array_path,data_format) +values (5, 'HW',B'1', '/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray', -'JSON'); -insert into `map_db_topic`(`db_id`,`topic_id`) select db.id, topic.id from db_type, db, topic where db.db_type_id=db_type.id and db_type.tool=0; -insert into `map_kafka_topic`(`kafka_id`,`topic_id`) select kafka.id, topic.id from kafka, topic; -insert into design_type (id, `name`, `db_type_id`) values ('KIBANA_DB', 'Kibana Dashboard', 'KIBANA'); -insert into design_type (id, `name`, `db_type_id`) values ('KIBANA_SEARCH', 'Kibana Search', 'KIBANA'); -insert into design_type (id, `name`, `db_type_id`) values ('KIBANA_VISUAL', 'Kibana Visualization', 'KIBANA'); -insert into design_type (id, `name`, `db_type_id`) values ('ES_MAPPING', 'Elasticsearch Field Mapping Template', 'ES'); -insert into design_type (id, `name`, `db_type_id`) values ('DRUID_KAFKA_SPEC', 'Druid Kafka Indexing Service Supervisor Spec', 'DRUID'); -insert into design (id, `name`,topic_name_id, `submitted`,`body`, design_type_id) values (1, 'Kibana Dashboard on EPC test1', 'EPC', 0, 'body here', 'KIBANA_DB'); -insert into map_db_design (`design_id`,`db_id` ) values (1, 6); -insert into `data_exposure`(`id`,`note`,`sql_template`,`db_id`) values ('totalBandwidth','KPI bandwidth history','select from_unixtime(commonEventHeader.lastEpochMicrosec/1000) as timeStamp, sum(measurementFields.additionalFields."UPF.N3IncPkt._Dnn"+measurementFields.additionalFields."UPF.N3OgPkt._Dnn") as bandwidth from upf where commonEventHeader.sourceId = ''${id}'' and ( from_unixtime(commonEventHeader.lastEpochMicrosec/1000) between from_iso8601_timestamp( ''${timeStamp}'') - interval ''${hour}'' hour and from_iso8601_timestamp( ''${timeStamp}'') ) group by commonEventHeader.lastEpochMicrosec order by commonEventHeader.lastEpochMicrosec desc ',3); -insert into `data_exposure`(`id`,`note`,`sql_template`,`db_id`) values ('totalTraffic','KPI sum over history','select commonEventHeader.sourceId as id, sum(measurementFields.additionalFields."UPF.N3IncPkt._Dnn"+measurementFields.additionalFields."UPF.N3OgPkt._Dnn") as totalTraffic from upf where commonEventHeader.sourceId = ''${id}'' and from_unixtime(commonEventHeader.lastEpochMicrosec/1000) <= from_iso8601_timestamp( ''${timeStamp}'') ',3); -insert into `data_exposure`(`id`,`note`,`sql_template`,`db_id`) values ('userNumber','KPI',' select from_unixtime(commonEventHeader.lastEpochMicrosec/1000) as timeStamp, sum(measurementFields.additionalFields."AMF.RegSub._NS") as userNumber from amf where commonEventHeader.sourceId = ''${id}'' and ( from_unixtime(commonEventHeader.lastEpochMicrosec/1000) between from_iso8601_timestamp( ''${timeStamp}'') - interval ''${hour}'' hour and from_iso8601_timestamp( ''${timeStamp}'') ) group by commonEventHeader.lastEpochMicrosec, commonEventHeader.sourceId order by commonEventHeader.lastEpochMicrosec desc ',3); +'JSON'); +insert into map_db_topic(db_id,topic_id) select db.id, topic.id from db_type, db, topic where db.db_type_id=db_type.id and db_type.tool=B'0'; +insert into map_kafka_topic(kafka_id,topic_id) select kafka.id, topic.id from kafka, topic; +insert into design_type (id, name, db_type_id) values ('KIBANA_DB', 'Kibana Dashboard', 'KIBANA'); +insert into design_type (id, name, db_type_id) values ('KIBANA_SEARCH', 'Kibana Search', 'KIBANA'); +insert into design_type (id, name, db_type_id) values ('KIBANA_VISUAL', 'Kibana Visualization', 'KIBANA'); +insert into design_type (id, name, db_type_id) values ('ES_MAPPING', 'Elasticsearch Field Mapping Template', 'ES'); +insert into design_type (id, name, db_type_id) values ('DRUID_KAFKA_SPEC', 'Druid Kafka Indexing Service Supervisor Spec', 'DRUID'); +insert into design (id, name,topic_name_id, submitted,body, design_type_id) values (1, 'Kibana Dashboard on EPC test1', 'EPC', B'0', 'body here', 'KIBANA_DB'); +insert into map_db_design (design_id,db_id ) values (1, 6); +insert into data_exposure(id,note,sql_template,db_id) values ('totalBandwidth','KPI bandwidth history','select from_unixtime(commonEventHeader.lastEpochMicrosec/1000) as timeStamp, sum(measurementFields.additionalFields."UPF.N3IncPkt._Dnn"+measurementFields.additionalFields."UPF.N3OgPkt._Dnn") as bandwidth from upf where commonEventHeader.sourceId = ''${id}'' and ( from_unixtime(commonEventHeader.lastEpochMicrosec/1000) between from_iso8601_timestamp( ''${timeStamp}'') - interval ''${hour}'' hour and from_iso8601_timestamp( ''${timeStamp}'') ) group by commonEventHeader.lastEpochMicrosec order by commonEventHeader.lastEpochMicrosec desc ',3); +insert into data_exposure(id,note,sql_template,db_id) values ('totalTraffic','KPI sum over history','select commonEventHeader.sourceId as id, sum(measurementFields.additionalFields."UPF.N3IncPkt._Dnn"+measurementFields.additionalFields."UPF.N3OgPkt._Dnn") as totalTraffic from upf where commonEventHeader.sourceId = ''${id}'' and from_unixtime(commonEventHeader.lastEpochMicrosec/1000) <= from_iso8601_timestamp( ''${timeStamp}'') ',3); +insert into data_exposure(id,note,sql_template,db_id) values ('userNumber','KPI',' select from_unixtime(commonEventHeader.lastEpochMicrosec/1000) as timeStamp, sum(measurementFields.additionalFields."AMF.RegSub._NS") as userNumber from amf where commonEventHeader.sourceId = ''${id}'' and ( from_unixtime(commonEventHeader.lastEpochMicrosec/1000) between from_iso8601_timestamp( ''${timeStamp}'') - interval ''${hour}'' hour and from_iso8601_timestamp( ''${timeStamp}'') ) group by commonEventHeader.lastEpochMicrosec, commonEventHeader.sourceId order by commonEventHeader.lastEpochMicrosec desc ',3); diff --git a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/10_check-db-exist b/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/10_check-db-exist deleted file mode 100644 index 4b117fd4..00000000 --- a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/10_check-db-exist +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# ============LICENSE_START========================================== -# =================================================================== -# Copyright © 2020 China Mobile. All rights reserved. -# =================================================================== -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END============================================ -# -result=`mysql -qfsB -uroot -p"$MYSQL_ROOT_PASSWORD" -h "$MYSQL_HOST" -P "$MYSQL_PORT" -e "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='datalake'" 2>&1` - -if [[ ! -z $result ]]; -then - echo "DATABASE ALREADY EXISTS" - touch /tmp/db_exist - exit 1 -else - echo "DATABASE DOES NOT EXIST" - exit 0 -fi
\ No newline at end of file diff --git a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/11_create-database b/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/11_create-database index 773f9c48..f69f1ab1 100644 --- a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/11_create-database +++ b/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/11_create-database @@ -3,6 +3,7 @@ # ============LICENSE_START========================================== # =================================================================== # Copyright © 2020 China Mobile. All rights reserved. +# Copyright (C) 2021 Wipro Limited. # =================================================================== # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +19,4 @@ # ============LICENSE_END============================================ # -if [ -f "/tmp/db_exist" ]; then - exit 0 -fi - -cat /home/datalake/init_db.sql | mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -h "$MYSQL_HOST" -P "$MYSQL_PORT" || exit 1 +psql -h $PG_HOST -U $PG_USER -d $PG_DB -f /home/datalake/init_db.sql diff --git a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/15_db-adduser b/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/15_db-adduser deleted file mode 100644 index 92d2c730..00000000 --- a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/15_db-adduser +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# ============LICENSE_START========================================== -# =================================================================== -# Copyright © 2020 China Mobile. All rights reserved. -# =================================================================== -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END============================================ -# - -if [ -f "/tmp/db_exist" ]; then - exit 0 -fi - -echo "Creating datalake user" 1>/tmp/mariadb-datalake-user.log 2>&1 - -mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -h "$MYSQL_HOST" -P "$MYSQL_PORT" << 'EOF' || exit 1 -CREATE USER IF NOT EXISTS 'dl'; -GRANT USAGE ON *.* TO 'dl'@'%' IDENTIFIED BY 'dl1234'; -GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, SHOW VIEW ON `datalake`.* TO 'dl'@'%'; -FLUSH PRIVILEGES; -EOF - -echo "Created so user . . ." 1>>/tmp/mariadb-datalake-user.log 2>&1 diff --git a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/20_db-initdb b/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/20_db-initdb index abc98552..32986cda 100644 --- a/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/20_db-initdb +++ b/components/datalake-handler/des/src/assembly/init_db/scripts/db_init/20_db-initdb @@ -3,6 +3,7 @@ # ============LICENSE_START========================================== # =================================================================== # Copyright © 2020 China Mobile. All rights reserved. +# Copyright (C) 2021 Wipro Limited. # =================================================================== # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +19,4 @@ # ============LICENSE_END============================================ # -if [ -f "/tmp/db_exist" ]; then - exit 0 -fi - -cat /home/datalake/init_db_data.sql | mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -h "$MYSQL_HOST" -P "$MYSQL_PORT" || exit 1 +psql -h $PG_HOST -U $PG_USER -d $PG_DB -f /home/datalake/init_db_data.sql diff --git a/components/datalake-handler/des/src/assembly/run.sh b/components/datalake-handler/des/src/assembly/run.sh index decd97e0..e6df159d 100644 --- a/components/datalake-handler/des/src/assembly/run.sh +++ b/components/datalake-handler/des/src/assembly/run.sh @@ -1,5 +1,6 @@ # ============LICENSE_START=================================================== # Copyright (C) 2020 China Mobile. +# Copyright (C) 2021 Wipro Limited. # ============================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,12 +21,14 @@ echo "start init db ..." -/bin/run-parts /home/datalake/db_init +export PGPASSWORD=$PG_PASSWORD + +sh db_init/11_create-database +sh db_init/20_db-initdb echo "finish init db" cmd=`find . -name des*-execute.jar` - if [ -n "$cmd" ]; then java -jar "$cmd" else diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java index 7689b279..4ed5d99c 100644 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java @@ -3,6 +3,7 @@ * ONAP : Data Extraction Service * ================================================================================ * Copyright 2020 China Mobile +* Copyright (C) 2021 Wipro Limited *================================================================================= * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +21,12 @@ package org.onap.datalake.des; +import javax.sql.DataSource; + import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; import springfox.documentation.swagger2.annotations.EnableSwagger2; @@ -40,4 +45,12 @@ public class DesApplication { SpringApplication.run(DesApplication.class, args); } + @Bean + public DataSource dataSource() { + + String url = "jdbc:postgresql://" + System.getenv("PG_HOST").trim() + ":" + System.getenv("PG_PORT").trim() + "/datalake"; + return DataSourceBuilder.create().url(url).username(System.getenv("PG_USER").trim()) + .password(System.getenv("PG_PASSWORD").trim()).build(); + } + } diff --git a/components/datalake-handler/des/src/main/resources/application.properties b/components/datalake-handler/des/src/main/resources/application.properties index 5c0a9f02..7672f1b1 100644 --- a/components/datalake-handler/des/src/main/resources/application.properties +++ b/components/datalake-handler/des/src/main/resources/application.properties @@ -19,10 +19,7 @@ defaultTopicName=_DL_DEFAULT_ spring.jpa.hibernate.ddl-auto=none spring.jpa.show-sql=false -#spring.datasource.driver-class-name=com.mysql.jdbc.Driver -spring.datasource.url=jdbc:mariadb://mariadb-galera:3306/datalake?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8 -spring.datasource.username=dl -spring.datasource.password=dl1234 +spring.datasource.driver-class-name=org.hibernate.dialect.PostgreSQL9Dialect #####################Logging logging.level.org.springframework.web=ERROR |