diff options
| -rw-r--r-- | components/datalake-handler/packages/init_db/Dockerfile | 48 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/db_scripts/init_db.sql (renamed from components/datalake-handler/feeder/src/assembly/scripts/init_db.sql) | 285 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/db_scripts/init_db_data.sql (renamed from components/datalake-handler/feeder/src/assembly/scripts/init_db_data.sql) | 218 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/pom.xml | 45 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/scripts/db_init/10_check-db-exist | 30 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/scripts/db_init/11_create-database | 25 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/scripts/db_init/15_db-adduser | 34 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/scripts/db_init/20_db-initdb | 25 | ||||
| -rw-r--r-- | components/datalake-handler/packages/init_db/scripts/run.sh | 28 |
9 files changed, 488 insertions, 250 deletions
diff --git a/components/datalake-handler/packages/init_db/Dockerfile b/components/datalake-handler/packages/init_db/Dockerfile new file mode 100644 index 00000000..cf6aed2d --- /dev/null +++ b/components/datalake-handler/packages/init_db/Dockerfile @@ -0,0 +1,48 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright (c) 2020 QCT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============LICENSE_END============================================ + +FROM ubuntu:16.04 AS builder +MAINTAINER Kate Hsuan <kate.hsuan@qct.io> + +LABEL org.label-schema.docker.build="docker build -t dl-db-init . --no-cache" \ + org.label-schema.docker.run="docker run dl-init" + +WORKDIR /app +WORKDIR /app/db_init +ADD scripts/db_init/ . +RUN chmod 0755 ./* +WORKDIR /app +COPY scripts/run.sh . +COPY db_scripts/init_db.sql . +COPY db_scripts/init_db_data.sql . + +RUN chmod 0755 /app/run.sh + +ARG mysqlpasswd="0000" +ENV MYSQL_ROOT_PASSWORD=$mysqlpasswd + +ARG mysqlhost="mariadb-galera" +ENV MYSQL_HOST=$mysqlhost + +ARG mysqlport="3306" +ENV MYSQL_PORT=${mysqlport} + +RUN apt update && \ + apt install -y mariadb-client-core-10.0 && \ + apt install -y curl + +CMD ["sh", "run.sh"] diff --git a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql b/components/datalake-handler/packages/init_db/db_scripts/init_db.sql index 3f495e2d..b363988d 100644 --- a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql +++ b/components/datalake-handler/packages/init_db/db_scripts/init_db.sql @@ -1,143 +1,142 @@ -/*
-* ============LICENSE_START=======================================================
-* ONAP : DATALAKE
-* ================================================================================
-* Copyright 2019-2020 China Mobile
-*=================================================================================
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* ============LICENSE_END=========================================================
-*/
-
-drop DATABASE datalake;
-create database datalake;
-use datalake;
-
-CREATE TABLE `topic_name` (
- `id` varchar(255) NOT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `db_type` (
- `id` varchar(255) NOT NULL,
- `default_port` int(11) DEFAULT NULL,
- `name` varchar(255) NOT NULL,
- `tool` bit(1) NOT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `db` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `database_name` varchar(255) DEFAULT NULL,
- `enabled` bit(1) NOT NULL,
- `encrypt` bit(1) DEFAULT NULL,
- `host` varchar(255) DEFAULT NULL,
- `login` varchar(255) DEFAULT NULL,
- `name` varchar(255) DEFAULT NULL,
- `pass` varchar(255) DEFAULT NULL,
- `port` int(11) DEFAULT NULL,
- `property1` varchar(255) DEFAULT NULL,
- `property2` varchar(255) DEFAULT NULL,
- `property3` varchar(255) DEFAULT NULL,
- `db_type_id` varchar(255) NOT NULL,
- PRIMARY KEY (`id`),
- KEY `FK3njadtw43ieph7ftt4kxdhcko` (`db_type_id`),
- CONSTRAINT `FK3njadtw43ieph7ftt4kxdhcko` FOREIGN KEY (`db_type_id`) REFERENCES `db_type` (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `design_type` (
- `id` varchar(255) NOT NULL,
- `name` varchar(255) DEFAULT NULL,
- `note` varchar(255) DEFAULT NULL,
- `db_type_id` varchar(255) NOT NULL,
- PRIMARY KEY (`id`),
- KEY `FKm8rkv2qkq01gsmeq1c3y4w02x` (`db_type_id`),
- CONSTRAINT `FKm8rkv2qkq01gsmeq1c3y4w02x` FOREIGN KEY (`db_type_id`) REFERENCES `db_type` (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `design` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `body` text DEFAULT NULL,
- `name` varchar(255) DEFAULT NULL,
- `note` varchar(255) DEFAULT NULL,
- `submitted` bit(1) DEFAULT NULL,
- `design_type_id` varchar(255) NOT NULL,
- `topic_name_id` varchar(255) NOT NULL,
- PRIMARY KEY (`id`),
- KEY `FKo43yi6aputq6kwqqu8eqbspm5` (`design_type_id`),
- KEY `FKabb8e74230glxpaiai4aqsr34` (`topic_name_id`),
- CONSTRAINT `FKabb8e74230glxpaiai4aqsr34` FOREIGN KEY (`topic_name_id`) REFERENCES `topic_name` (`id`),
- CONSTRAINT `FKo43yi6aputq6kwqqu8eqbspm5` FOREIGN KEY (`design_type_id`) REFERENCES `design_type` (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `kafka` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `broker_list` varchar(255) NOT NULL,
- `consumer_count` int(11) DEFAULT 3,
- `enabled` bit(1) NOT NULL,
- `excluded_topic` varchar(1023) DEFAULT '__consumer_offsets,__transaction_state',
- `group` varchar(255) DEFAULT 'datalake',
- `included_topic` varchar(255) DEFAULT NULL,
- `login` varchar(255) DEFAULT NULL,
- `name` varchar(255) NOT NULL,
- `pass` varchar(255) DEFAULT NULL,
- `secure` bit(1) DEFAULT b'0',
- `security_protocol` varchar(255) DEFAULT NULL,
- `timeout_sec` int(11) DEFAULT 10,
- `zk` varchar(255) NOT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `topic` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `aggregate_array_path` varchar(255) DEFAULT NULL,
- `correlate_cleared_message` bit(1) NOT NULL DEFAULT b'0',
- `data_format` varchar(255) DEFAULT NULL,
- `enabled` bit(1) NOT NULL,
- `flatten_array_path` varchar(255) DEFAULT NULL,
- `login` varchar(255) DEFAULT NULL,
- `message_id_path` varchar(255) DEFAULT NULL,
- `pass` varchar(255) DEFAULT NULL,
- `save_raw` bit(1) NOT NULL DEFAULT b'0',
- `ttl_day` int(11) DEFAULT NULL,
- `topic_name_id` varchar(255) NOT NULL,
- PRIMARY KEY (`id`),
- KEY `FKj3pldlfaokdhqjfva8n3pkjca` (`topic_name_id`),
- CONSTRAINT `FKj3pldlfaokdhqjfva8n3pkjca` FOREIGN KEY (`topic_name_id`) REFERENCES `topic_name` (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `map_db_design` (
- `design_id` int(11) NOT NULL,
- `db_id` int(11) NOT NULL,
- PRIMARY KEY (`design_id`,`db_id`),
- KEY `FKhpn49r94k05mancjtn301m2p0` (`db_id`),
- CONSTRAINT `FKfli240v96cfjbnmjqc0fvvd57` FOREIGN KEY (`design_id`) REFERENCES `design` (`id`),
- CONSTRAINT `FKhpn49r94k05mancjtn301m2p0` FOREIGN KEY (`db_id`) REFERENCES `db` (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `map_db_topic` (
- `topic_id` int(11) NOT NULL,
- `db_id` int(11) NOT NULL,
- PRIMARY KEY (`db_id`,`topic_id`),
- KEY `FKq1jon185jnrr7dv1dd8214uw0` (`topic_id`),
- CONSTRAINT `FKirro29ojp7jmtqx9m1qxwixcc` FOREIGN KEY (`db_id`) REFERENCES `db` (`id`),
- CONSTRAINT `FKq1jon185jnrr7dv1dd8214uw0` FOREIGN KEY (`topic_id`) REFERENCES `topic` (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `map_kafka_topic` (
- `kafka_id` int(11) NOT NULL,
- `topic_id` int(11) NOT NULL,
- PRIMARY KEY (`topic_id`,`kafka_id`),
- KEY `FKtdrme4h7rxfh04u2i2wqu23g5` (`kafka_id`),
- CONSTRAINT `FK5q7jdxy54au5rcrhwa4a5igqi` FOREIGN KEY (`topic_id`) REFERENCES `topic` (`id`),
- CONSTRAINT `FKtdrme4h7rxfh04u2i2wqu23g5` FOREIGN KEY (`kafka_id`) REFERENCES `kafka` (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+/* +* ============LICENSE_START======================================================= +* ONAP : DATALAKE +* ================================================================================ +* Copyright 2019-2020 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +create database if not exists datalake; +use datalake; + +CREATE TABLE `topic_name` ( + `id` varchar(255) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `db_type` ( + `id` varchar(255) NOT NULL, + `default_port` int(11) DEFAULT NULL, + `name` varchar(255) NOT NULL, + `tool` bit(1) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `db` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `database_name` varchar(255) DEFAULT NULL, + `enabled` bit(1) NOT NULL, + `encrypt` bit(1) DEFAULT NULL, + `host` varchar(255) DEFAULT NULL, + `login` varchar(255) DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `pass` varchar(255) DEFAULT NULL, + `port` int(11) DEFAULT NULL, + `property1` varchar(255) DEFAULT NULL, + `property2` varchar(255) DEFAULT NULL, + `property3` varchar(255) DEFAULT NULL, + `db_type_id` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `FK3njadtw43ieph7ftt4kxdhcko` (`db_type_id`), + CONSTRAINT `FK3njadtw43ieph7ftt4kxdhcko` FOREIGN KEY (`db_type_id`) REFERENCES `db_type` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `design_type` ( + `id` varchar(255) NOT NULL, + `name` varchar(255) DEFAULT NULL, + `note` varchar(255) DEFAULT NULL, + `db_type_id` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `FKm8rkv2qkq01gsmeq1c3y4w02x` (`db_type_id`), + CONSTRAINT `FKm8rkv2qkq01gsmeq1c3y4w02x` FOREIGN KEY (`db_type_id`) REFERENCES `db_type` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `design` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `body` text DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `note` varchar(255) DEFAULT NULL, + `submitted` bit(1) DEFAULT NULL, + `design_type_id` varchar(255) NOT NULL, + `topic_name_id` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `FKo43yi6aputq6kwqqu8eqbspm5` (`design_type_id`), + KEY `FKabb8e74230glxpaiai4aqsr34` (`topic_name_id`), + CONSTRAINT `FKabb8e74230glxpaiai4aqsr34` FOREIGN KEY (`topic_name_id`) REFERENCES `topic_name` (`id`), + CONSTRAINT `FKo43yi6aputq6kwqqu8eqbspm5` FOREIGN KEY (`design_type_id`) REFERENCES `design_type` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `kafka` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `broker_list` varchar(255) NOT NULL, + `consumer_count` int(11) DEFAULT 3, + `enabled` bit(1) NOT NULL, + `excluded_topic` varchar(1023) DEFAULT '__consumer_offsets,__transaction_state', + `group` varchar(255) DEFAULT 'datalake', + `included_topic` varchar(255) DEFAULT NULL, + `login` varchar(255) DEFAULT NULL, + `name` varchar(255) NOT NULL, + `pass` varchar(255) DEFAULT NULL, + `secure` bit(1) DEFAULT b'0', + `security_protocol` varchar(255) DEFAULT NULL, + `timeout_sec` int(11) DEFAULT 10, + `zk` varchar(255) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `topic` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `aggregate_array_path` varchar(255) DEFAULT NULL, + `correlate_cleared_message` bit(1) NOT NULL DEFAULT b'0', + `data_format` varchar(255) DEFAULT NULL, + `enabled` bit(1) NOT NULL, + `flatten_array_path` varchar(255) DEFAULT NULL, + `login` varchar(255) DEFAULT NULL, + `message_id_path` varchar(255) DEFAULT NULL, + `pass` varchar(255) DEFAULT NULL, + `save_raw` bit(1) NOT NULL DEFAULT b'0', + `ttl_day` int(11) DEFAULT NULL, + `topic_name_id` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `FKj3pldlfaokdhqjfva8n3pkjca` (`topic_name_id`), + CONSTRAINT `FKj3pldlfaokdhqjfva8n3pkjca` FOREIGN KEY (`topic_name_id`) REFERENCES `topic_name` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `map_db_design` ( + `design_id` int(11) NOT NULL, + `db_id` int(11) NOT NULL, + PRIMARY KEY (`design_id`,`db_id`), + KEY `FKhpn49r94k05mancjtn301m2p0` (`db_id`), + CONSTRAINT `FKfli240v96cfjbnmjqc0fvvd57` FOREIGN KEY (`design_id`) REFERENCES `design` (`id`), + CONSTRAINT `FKhpn49r94k05mancjtn301m2p0` FOREIGN KEY (`db_id`) REFERENCES `db` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `map_db_topic` ( + `topic_id` int(11) NOT NULL, + `db_id` int(11) NOT NULL, + PRIMARY KEY (`db_id`,`topic_id`), + KEY `FKq1jon185jnrr7dv1dd8214uw0` (`topic_id`), + CONSTRAINT `FKirro29ojp7jmtqx9m1qxwixcc` FOREIGN KEY (`db_id`) REFERENCES `db` (`id`), + CONSTRAINT `FKq1jon185jnrr7dv1dd8214uw0` FOREIGN KEY (`topic_id`) REFERENCES `topic` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `map_kafka_topic` ( + `kafka_id` int(11) NOT NULL, + `topic_id` int(11) NOT NULL, + PRIMARY KEY (`topic_id`,`kafka_id`), + KEY `FKtdrme4h7rxfh04u2i2wqu23g5` (`kafka_id`), + CONSTRAINT `FK5q7jdxy54au5rcrhwa4a5igqi` FOREIGN KEY (`topic_id`) REFERENCES `topic` (`id`), + CONSTRAINT `FKtdrme4h7rxfh04u2i2wqu23g5` FOREIGN KEY (`kafka_id`) REFERENCES `kafka` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/components/datalake-handler/feeder/src/assembly/scripts/init_db_data.sql b/components/datalake-handler/packages/init_db/db_scripts/init_db_data.sql index 1328279a..7857d3ee 100644 --- a/components/datalake-handler/feeder/src/assembly/scripts/init_db_data.sql +++ b/components/datalake-handler/packages/init_db/db_scripts/init_db_data.sql @@ -1,107 +1,111 @@ -/*
-* ============LICENSE_START=======================================================
-* ONAP : DATALAKE
-* ================================================================================
-* Copyright 2019 China Mobile
-*=================================================================================
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* ============LICENSE_END=========================================================
-*/
-
-INSERT INTO datalake.kafka(
- id
- ,name
- ,consumer_count
- ,enabled
- ,`group`
- ,broker_list
- ,included_topic
- ,login
- ,pass
- ,secure
- ,security_protocol
- ,timeout_sec
- ,zk
-) VALUES (
- 1
- ,'main Kafka cluster' -- name - IN varchar(255)
- ,3 -- consumer_count - IN int(11)
- ,1 -- enabled - IN bit(1)
- ,'dlgroup' -- group - IN varchar(255)
- ,'message-router-kafka:9092' -- host_port - IN varchar(255)
- ,'' -- included_topic - IN varchar(255)
- ,'admin' -- login - IN varchar(255)
- ,'admin-secret' -- pass - IN varchar(255)
- ,0 -- secure - IN bit(1)
- ,'SASL_PLAINTEXT' -- security_protocol - IN varchar(255)
- ,10 -- timeout_sec - IN int(11)
- ,'message-router-zookeeper:2181' -- zk - IN varchar(255)
-);
-
-insert into db_type (`id`, `name`, tool) values ('CB', 'Couchbase', false);
-insert into db_type (`id`, `name`, tool) values ('ES', 'Elasticsearch', false);
-insert into db_type (`id`, `name`, tool,`default_port`) values ('MONGO', 'MongoDB', false, 27017);
-insert into db_type (`id`, `name`, tool) values ('DRUID', 'Druid', false);
-insert into db_type (`id`, `name`, tool) values ('HDFS', 'HDFS', false);
-insert into db_type (`id`, `name`, tool) values ('KIBANA', 'Kibana', true);
-insert into db_type (`id`, `name`, tool) values ('SUPERSET', 'Apache Superset', true);
-
-insert into db (id, db_type_id, enabled, encrypt, `name`,`host`,`login`,`pass`,`database_name`) values (1, 'CB', true, true, 'Couchbase 1','dl-couchbase','dl','dl1234','datalake');
-insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (2, 'ES', true, true, 'Elasticsearch','dl-es');
-insert into db (id, db_type_id, enabled, encrypt, `name`,`host`,`port`,`database_name`) values (3, 'MONGO', true, true, 'MongoDB 1','dl-mongodb',27017,'datalake');
-insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (4, 'DRUID', true, true, 'Druid','dl-druid');
-insert into db (id, db_type_id, enabled, encrypt, `name`,`host`,`login`) values (5, 'HDFS', true, true, 'Hadoop Cluster','dl-hdfs','dl');
-insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (6, 'KIBANA', true, false, 'Kibana demo','dl-es');
-insert into db (id, db_type_id, enabled, encrypt, `name`,`host`) values (7, 'SUPERSET', true, false, 'Superset demo','dl-druid');
-
-insert into topic_name (id) values ('_DL_DEFAULT_');
-insert into topic_name (id) values ('unauthenticated.SEC_FAULT_OUTPUT');
-insert into topic_name (id) values ('unauthenticated.VES_MEASUREMENT_OUTPUT');
-insert into topic_name (id) values ('EPC');
-insert into topic_name (id) values ('HW');
-
--- in production, default enabled should be off
-insert into `topic`(id, `topic_name_id`,`enabled`,`save_raw`,`ttl_day`,`data_format`) values (1, '_DL_DEFAULT_',1,0,3650,'JSON');
-
-insert into `topic`(id, `topic_name_id`,correlate_cleared_message,`enabled`, message_id_path,`data_format`)
-values (2, 'unauthenticated.SEC_FAULT_OUTPUT',1,1,'/event/commonEventHeader/eventName,/event/commonEventHeader/reportingEntityName,/event/faultFields/specificProblem,/event/commonEventHeader/eventId','JSON');
-
-insert into `topic`(id, `topic_name_id`,`enabled`, aggregate_array_path,flatten_array_path,`data_format`)
-values (3, 'unauthenticated.VES_MEASUREMENT_OUTPUT',1,
-'/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray',
-'/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface',
-'JSON');
-
-insert into `topic`(id, `topic_name_id`,`enabled`, flatten_array_path,`data_format`)
-values (4, 'EPC',1, '/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface', 'JSON');
-
-insert into `topic`(id, `topic_name_id`,`enabled`, aggregate_array_path,`data_format`)
-values (5, 'HW',1,
-'/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray',
-'JSON');
-
-
-insert into `map_db_topic`(`db_id`,`topic_id`) select db.id, topic.id from db_type, db, topic where db.db_type_id=db_type.id and db_type.tool=0;
-insert into `map_kafka_topic`(`kafka_id`,`topic_id`) select kafka.id, topic.id from kafka, topic;
-
-
-insert into design_type (id, `name`, `db_type_id`) values ('KIBANA_DB', 'Kibana Dashboard', 'KIBANA');
-insert into design_type (id, `name`, `db_type_id`) values ('KIBANA_SEARCH', 'Kibana Search', 'KIBANA');
-insert into design_type (id, `name`, `db_type_id`) values ('KIBANA_VISUAL', 'Kibana Visualization', 'KIBANA');
-insert into design_type (id, `name`, `db_type_id`) values ('ES_MAPPING', 'Elasticsearch Field Mapping Template', 'ES');
-insert into design_type (id, `name`, `db_type_id`) values ('DRUID_KAFKA_SPEC', 'Druid Kafka Indexing Service Supervisor Spec', 'DRUID');
-
-
-insert into design (id, `name`,topic_name_id, `submitted`,`body`, design_type_id) values (1, 'Kibana Dashboard on EPC test1', 'EPC', 0, 'body here', 'KIBANA_DB');
-
-insert into map_db_design (`design_id`,`db_id` ) values (1, 6);
+/* +* ============LICENSE_START======================================================= +* ONAP : DATALAKE +* ================================================================================ +* Copyright 2019 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +use datalake; + +INSERT INTO kafka( + `id` + ,`name` + ,`consumer_count` + ,`enabled` + ,`group` + ,`broker_list` + ,`included_topic` + ,`login` + ,`pass` + ,`secure` + ,`security_protocol` + ,`timeout_sec` + ,`zk` +) VALUES ( + 1 + ,'main Kafka cluster' -- name - IN varchar(255) + ,3 -- consumer_count - IN int(11) + ,1 -- enabled - IN bit(1) + ,'dlgroup' -- group - IN varchar(255) + ,'message-router-kafka:9092' -- host_port - IN varchar(255) + ,'' -- included_topic - IN varchar(255) + ,'admin' -- login - IN varchar(255) + ,'admin-secret' -- pass - IN varchar(255) + ,0 -- secure - IN bit(1) + ,'SASL_PLAINTEXT' -- security_protocol - IN varchar(255) + ,10 -- timeout_sec - IN int(11) + ,'message-router-zookeeper:2181' -- zk - IN varchar(255) +); + +use datalake; + +insert into db_type (`id`, `name`, tool) values ('CB', 'Couchbase', false); +insert into db_type (`id`, `name`, tool) values ('ES', 'Elasticsearch', false); +insert into db_type (`id`, `name`, tool,`default_port`) values ('MONGO', 'MongoDB', false, 27017); +insert into db_type (`id`, `name`, tool) values ('DRUID', 'Druid', false); +insert into db_type (`id`, `name`, tool) values ('HDFS', 'HDFS', false); +insert into db_type (`id`, `name`, tool) values ('KIBANA', 'Kibana', true); +insert into db_type (`id`, `name`, tool) values ('SUPERSET', 'Apache Superset', true); + +insert into db (`id`, `db_type_id`, `enabled`, `encrypt`, `name`,`host`,`login`,`pass`,`database_name`) values (1, 'CB', true, true, 'Couchbase 1','dl-couchbase','dl','dl1234','datalake'); +insert into db (`id`, `db_type_id`, `enabled`, `encrypt`, `name`,`host`) values (2, 'ES', true, true, 'Elasticsearch','dl-es'); +insert into db (`id`, `db_type_id`, `enabled`, `encrypt`, `name`,`host`,`port`,`database_name`) values (3, 'MONGO', true, true, 'MongoDB 1','dl-mongodb',27017,'datalake'); +insert into db (`id`, `db_type_id`, `enabled`, `encrypt`, `name`,`host`) values (4, 'DRUID', true, true, 'Druid','dl-druid'); +insert into db (`id`, `db_type_id`, `enabled`, `encrypt`, `name`,`host`,`login`) values (5, 'HDFS', true, true, 'Hadoop Cluster','dl-hdfs','dl'); +insert into db (`id`, `db_type_id`, `enabled`, `encrypt`, `name`,`host`) values (6, 'KIBANA', true, false, 'Kibana demo','dl-es'); +insert into db (`id`, `db_type_id`, `enabled`, `encrypt`, `name`,`host`) values (7, 'SUPERSET', true, false, 'Superset demo','dl-druid'); + +insert into topic_name (`id`) values ('_DL_DEFAULT_'); +insert into topic_name (`id`) values ('unauthenticated.SEC_FAULT_OUTPUT'); +insert into topic_name (`id`) values ('unauthenticated.VES_MEASUREMENT_OUTPUT'); +insert into topic_name (`id`) values ('EPC'); +insert into topic_name (`id`) values ('HW'); + +-- in production, default enabled should be off +insert into `topic`(`id`, `topic_name_id`,`enabled`,`save_raw`,`ttl_day`,`data_format`) values (1, '_DL_DEFAULT_',1,0,3650,'JSON'); + +insert into `topic`(`id`, `topic_name_id`,`correlate_cleared_message`,`enabled`, `message_id_path`,`data_format`) +values (2, 'unauthenticated.SEC_FAULT_OUTPUT',1,1,'/event/commonEventHeader/eventName,/event/commonEventHeader/reportingEntityName,/event/faultFields/specificProblem,/event/commonEventHeader/eventId','JSON'); + +insert into `topic`(id, `topic_name_id`,`enabled`, `aggregate_array_path`,`flatten_array_path`,`data_format`) +values (3, 'unauthenticated.VES_MEASUREMENT_OUTPUT',1, +'/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray', +'/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface', +'JSON'); + +insert into `topic`(`id`, `topic_name_id`,`enabled`, `flatten_array_path`,`data_format`) +values (4, 'EPC',1, '/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface', 'JSON'); + +insert into `topic`(`id`, `topic_name_id`,`enabled`, `aggregate_array_path`,`data_format`) +values (5, 'HW',1, +'/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray', +'JSON'); + + +insert into `map_db_topic`(`db_id`,`topic_id`) select db.id, topic.id from db_type, db, topic where db.db_type_id=db_type.id and db_type.tool=0; +insert into `map_kafka_topic`(`kafka_id`,`topic_id`) select kafka.id, topic.id from kafka, topic; + + +insert into design_type (`id`, `name`, `db_type_id`) values ('KIBANA_DB', 'Kibana Dashboard', 'KIBANA'); +insert into design_type (`id`, `name`, `db_type_id`) values ('KIBANA_SEARCH', 'Kibana Search', 'KIBANA'); +insert into design_type (`id`, `name`, `db_type_id`) values ('KIBANA_VISUAL', 'Kibana Visualization', 'KIBANA'); +insert into design_type (`id`, `name`, `db_type_id`) values ('ES_MAPPING', 'Elasticsearch Field Mapping Template', 'ES'); +insert into design_type (`id`, `name`, `db_type_id`) values ('DRUID_KAFKA_SPEC', 'Druid Kafka Indexing Service Supervisor Spec', 'DRUID'); + + +insert into design (`id`, `name`,`topic_name_id`, `submitted`,`body`, `design_type_id`) values (1, 'Kibana Dashboard on EPC test1', 'EPC', 0, 'body here', 'KIBANA_DB'); + +insert into map_db_design (`design_id`,`db_id` ) values (1, 6); diff --git a/components/datalake-handler/packages/init_db/pom.xml b/components/datalake-handler/packages/init_db/pom.xml new file mode 100644 index 00000000..fc74e6a7 --- /dev/null +++ b/components/datalake-handler/packages/init_db/pom.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.onap.dcaegen2.services.components</groupId> + <artifactId>datalake-handler</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + + <groupId>org.onap.dcaegen2.services.components.datalake-handler</groupId> + <artifactId>db_init</artifactId> + <packaging>pom</packaging> + <properties> + <swagger.version>2.9.2</swagger.version> + <dockerfile-maven.version>1.4.5</dockerfile-maven.version> + <docker.image.path>onap/org.onap.dcaegen2.services.datalake.dbinit</docker.image.path> + </properties> + <name>DataLake db init</name> + <build> + <plugins> + + <plugin> + <groupId>com.spotify</groupId> + <artifactId>dockerfile-maven-plugin</artifactId> + <version>${dockerfile-maven.version}</version> + <configuration> + <repository>${onap.nexus.dockerregistry.daily}/${docker.image.path}</repository> + <tag>${project.version}</tag> + <dockerfile>Dockerfile</dockerfile> + <noCache>true</noCache> + </configuration> + <dependencies> + <dependency> + <groupId>javax.activation</groupId> + <artifactId>javax.activation-api</artifactId> + <version>1.2.0</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </build> +</project> diff --git a/components/datalake-handler/packages/init_db/scripts/db_init/10_check-db-exist b/components/datalake-handler/packages/init_db/scripts/db_init/10_check-db-exist new file mode 100644 index 00000000..cdf8af50 --- /dev/null +++ b/components/datalake-handler/packages/init_db/scripts/db_init/10_check-db-exist @@ -0,0 +1,30 @@ +#!/bin/bash +# +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2020 QCT Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# +result=`mysql -qfsB -uroot -p$MYSQL_ROOT_PASSWORD -h $MYSQL_HOST -P $MYSQL_PORT -e "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='datalake'" 2>&1` + +if [[ ! -z $result ]]; +then + echo "DATABASE ALREADY EXISTS" + touch /tmp/db_exist + exit 1 +else + echo "DATABASE DOES NOT EXIST" + exit 0 +fi
\ No newline at end of file diff --git a/components/datalake-handler/packages/init_db/scripts/db_init/11_create-database b/components/datalake-handler/packages/init_db/scripts/db_init/11_create-database new file mode 100644 index 00000000..3e34828f --- /dev/null +++ b/components/datalake-handler/packages/init_db/scripts/db_init/11_create-database @@ -0,0 +1,25 @@ +#!/bin/bash +# +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2020 QCT Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# + +if [ -f "/tmp/db_exist" ]; then + exit 0 +fi + +cat /app/init_db.sql | mysql -uroot -p$MYSQL_ROOT_PASSWORD -h $MYSQL_HOST -P $MYSQL_PORT || exit 1 diff --git a/components/datalake-handler/packages/init_db/scripts/db_init/15_db-adduser b/components/datalake-handler/packages/init_db/scripts/db_init/15_db-adduser new file mode 100644 index 00000000..c691751c --- /dev/null +++ b/components/datalake-handler/packages/init_db/scripts/db_init/15_db-adduser @@ -0,0 +1,34 @@ +#!/bin/bash +# +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2020 QCT Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# + +if [ -f "/tmp/db_exist" ]; then + exit 0 +fi + +echo "Creating datalake user" 1>/tmp/mariadb-datalake-user.log 2>&1 + +mysql -uroot -p$MYSQL_ROOT_PASSWORD -p$MYSQL_ROOT_PASSWORD -h $MYSQL_HOST -P $MYSQL_PORT << 'EOF' || exit 1 +CREATE USER IF NOT EXISTS 'dl'; +GRANT USAGE ON *.* TO 'dl'@'%' IDENTIFIED BY 'dl1234'; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, SHOW VIEW ON `datalake`.* TO 'dl'@'%'; +FLUSH PRIVILEGES; +EOF + +echo "Created so user . . ." 1>>/tmp/mariadb-datalake-user.log 2>&1 diff --git a/components/datalake-handler/packages/init_db/scripts/db_init/20_db-initdb b/components/datalake-handler/packages/init_db/scripts/db_init/20_db-initdb new file mode 100644 index 00000000..3b6d340f --- /dev/null +++ b/components/datalake-handler/packages/init_db/scripts/db_init/20_db-initdb @@ -0,0 +1,25 @@ +#!/bin/bash +# +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2020 QCT Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# + +if [ -f "/tmp/db_exist" ]; then + exit 0 +fi + +cat /app/init_db_data.sql | mysql -uroot -p$MYSQL_ROOT_PASSWORD -h $MYSQL_HOST -P $MYSQL_PORT || exit 1 diff --git a/components/datalake-handler/packages/init_db/scripts/run.sh b/components/datalake-handler/packages/init_db/scripts/run.sh new file mode 100644 index 00000000..e1e59252 --- /dev/null +++ b/components/datalake-handler/packages/init_db/scripts/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# ============LICENSE_START========================================== +# =================================================================== +# Copyright (c) 2020 QCT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============LICENSE_END============================================ + + +echo "start init db ..." + +/bin/run-parts /app/db_init + +echo "finish init db" + + +curl -X PUT -H "contain-type:application/json" http://consul:8500/v1/kv/k8s-datalake -d "{\"mysql_host\": \"$MYSQL_HOST\", \"mysql_password\": \"$MYSQL_ROOT_PASSWORD\", \"mysql_port\": \"$MYSQL_PORT\", \"presto_host\": \"$PRESTO_HOST\"}" + |