diff options
12 files changed, 618 insertions, 249 deletions
diff --git a/components/datalake-handler/feeder/pom.xml b/components/datalake-handler/feeder/pom.xml index 9c1bb785..2af2af86 100644 --- a/components/datalake-handler/feeder/pom.xml +++ b/components/datalake-handler/feeder/pom.xml @@ -1,186 +1,197 @@ <?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.onap.dcaegen2.services.components</groupId> - <artifactId>datalake-handler</artifactId> - <version>1.0.0-SNAPSHOT</version> - </parent> - - <groupId>org.onap.dcaegen2.services.components.datalake-handler</groupId> - <artifactId>feeder</artifactId> - <packaging>jar</packaging> - <name>DataLake Feeder</name> - - - <dependencies> - - <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - </dependency> - - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-web</artifactId> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-actuator</artifactId> - </dependency> - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.onap.dcaegen2.services.components</groupId> + <artifactId>datalake-handler</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + + <groupId>org.onap.dcaegen2.services.components.datalake-handler</groupId> + <artifactId>feeder</artifactId> + <packaging>jar</packaging> + <name>DataLake Feeder</name> + + + <dependencies> + + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + </dependency> + + <dependency> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-actuator</artifactId> + </dependency> + <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-data-couchbase</artifactId> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-test</artifactId> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-configuration-processor</artifactId> - </dependency> - - <dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>elasticsearch-rest-high-level-client</artifactId> - </dependency> - - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-yaml</artifactId> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-xml</artifactId> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </dependency> - - <dependency> - <groupId>org.projectlombok</groupId> - <artifactId>lombok</artifactId> - </dependency> - - <dependency> - <groupId>io.druid</groupId> - <artifactId>tranquility-core_2.11</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.velocity</groupId> - <artifactId>velocity-engine-core</artifactId> - </dependency> - - - <dependency> - <groupId>org.hibernate</groupId> - <artifactId>hibernate-core</artifactId> - <version>5.3.7.Final</version> - </dependency> - - <!-- jsr303 validation --> - <dependency> - <groupId>javax.validation</groupId> - <artifactId>validation-api</artifactId> - <version>2.0.1.Final</version> - </dependency> - - <dependency> - <groupId>org.hibernate</groupId> - <artifactId>hibernate-validator</artifactId> - <version>6.0.10.Final</version> - </dependency> - - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-swagger2</artifactId> - <version>2.9.2</version> - <scope>compile</scope> - </dependency> - - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-swagger-ui</artifactId> - <version>2.9.2</version> - <scope>compile</scope> - </dependency> - - <dependency> - <groupId>org.mongodb</groupId> - <artifactId>mongo-java-driver</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-maven-plugin</artifactId> - <version>${springboot.version}</version> - <executions> - <execution> - <goals> - <goal>repackage</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> - <artifactId>maven-failsafe-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>integration-test</goal> - <goal>verify</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-couchbase</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-configuration-processor</artifactId> + </dependency> + + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-high-level-client</artifactId> + </dependency> + + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-yaml</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-xml</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + </dependency> + + <dependency> + <groupId>io.druid</groupId> + <artifactId>tranquility-core_2.11</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity-engine-core</artifactId> + </dependency> + + + <dependency> + <groupId>org.hibernate</groupId> + <artifactId>hibernate-core</artifactId> + <version>5.3.7.Final</version> + </dependency> + + <!-- jsr303 validation --> + <dependency> + <groupId>javax.validation</groupId> + <artifactId>validation-api</artifactId> + <version>2.0.1.Final</version> + </dependency> + + <dependency> + <groupId>org.hibernate</groupId> + <artifactId>hibernate-validator</artifactId> + <version>6.0.10.Final</version> + </dependency> + + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-swagger2</artifactId> + <version>2.9.2</version> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-swagger-ui</artifactId> + <version>2.9.2</version> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.mongodb</groupId> + <artifactId>mongo-java-driver</artifactId> + </dependency> + <dependency> + <groupId>com.couchbase.mock</groupId> + <artifactId>CouchbaseMock</artifactId> + <version>1.5.22</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.couchbase.client</groupId> + <artifactId>core-io</artifactId> + <version>1.7.4</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <version>${springboot.version}</version> + <executions> + <execution> + <goals> + <goal>repackage</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> </project> diff --git a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql b/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql index 32b9268c..a349d146 100644 --- a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql +++ b/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql @@ -22,7 +22,7 @@ CREATE TABLE `db` ( `name` varchar(255) NOT NULL,
`host` varchar(255) DEFAULT NULL,
`port` int(11) DEFAULT NULL,
- `database` varchar(255) DEFAULT NULL,
+ `database_name` varchar(255) DEFAULT NULL,
`encrypt` bit(1) DEFAULT NULL,
`login` varchar(255) DEFAULT NULL,
`pass` varchar(255) DEFAULT NULL,
@@ -43,10 +43,10 @@ CREATE TABLE `map_db_topic` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-insert into db (name,host,login,pass,`database`) values ('Couchbase','dl_couchbase','dl','dl1234','datalake');
-insert into db (name,host) values ('Elasticsearch','dl_es');
-insert into db (name,host,port,`database`) values ('MongoDB','dl_mongodb',27017,'datalake');
-insert into db (name,host) values ('Druid','dl_druid');
+insert into db (`name`,`host`,`login`,`pass`,`database`) values ('Couchbase','dl_couchbase','dl','dl1234','datalake');
+insert into db (`name`,`host`) values ('Elasticsearch','dl_es');
+insert into db (`name`,`host`,`port`,`database`) values ('MongoDB','dl_mongodb',27017,'datalake');
+insert into db (`name`,`host`) values ('Druid','dl_druid');
-- in production, default enabled should be off
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/DbController.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/DbController.java index c4288d93..4bfe7aa3 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/DbController.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/DbController.java @@ -20,27 +20,24 @@ package org.onap.datalake.feeder.controller; import java.io.IOException; -import java.util.Set; +import java.util.*; import javax.servlet.http.HttpServletResponse; +import io.swagger.annotations.*; import org.onap.datalake.feeder.domain.Db; import org.onap.datalake.feeder.domain.Topic; import org.onap.datalake.feeder.repository.DbRepository; +import org.onap.datalake.feeder.repository.TopicRepository; import org.onap.datalake.feeder.service.DbService; +import org.onap.datalake.feeder.controller.domain.DbConfig; +import org.onap.datalake.feeder.controller.domain.PostReturnBody; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.validation.BindingResult; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -57,6 +54,7 @@ import io.swagger.annotations.ApiResponses; @RestController @RequestMapping(value = "/dbs", produces = { MediaType.APPLICATION_JSON_VALUE }) + //@Api(value = "db", consumes = "application/json", produces = "application/json") public class DbController { @@ -66,80 +64,180 @@ public class DbController { private DbRepository dbRepository; @Autowired + private TopicRepository topicRepository; + + @Autowired private DbService dbService; //list all dbs - @GetMapping("/") + @GetMapping("") @ResponseBody - @ApiOperation(value="Get all databases' details.") - public Iterable<Db> list() throws IOException { + @ApiOperation(value="Gat all databases name") + //public Iterable<Db> list() throws IOException { + public List<String> list() throws IOException { Iterable<Db> ret = dbRepository.findAll(); - return ret; + List<String> retString = new ArrayList<>(); + for(Db db : ret) + { + log.info(db.getName()); + retString.add(db.getName()); + + } + return retString; } - //Read a db + //Create a DB + @PostMapping("") + @ResponseBody + @ApiOperation(value="Create a new database.") + public PostReturnBody<DbConfig> createDb(@RequestBody DbConfig dbConfig, BindingResult result, HttpServletResponse response) throws IOException { + if (result.hasErrors()) { + sendError(response, 400, "Malformed format of Post body: " + result.toString()); + return null; + } + + Db oldDb = dbService.getDb(dbConfig.getName()); + if (oldDb != null) { + sendError(response, 400, "Db already exists: " + dbConfig.getName()); + return null; + } else { + Db newdb = new Db(); + newdb.setName(dbConfig.getName()); + newdb.setHost(dbConfig.getHost()); + newdb.setPort(dbConfig.getPort()); + newdb.setLogin(dbConfig.getLogin()); + newdb.setPass(dbConfig.getPassword()); + newdb.setEncrypt(false); + + if(dbConfig.getName().equals("Elecsticsearch") || dbConfig.getName().equals("Druid")) + { + newdb.setDatabase(new String(dbConfig.getDatabase())); + } + dbRepository.save(newdb); + DbConfig retMsg; + PostReturnBody<DbConfig> retBody = new PostReturnBody<>(); + retMsg = new DbConfig(); + composeRetMessagefromDbConfig(newdb, retMsg); + retBody.setReturnBody(retMsg); + retBody.setStatusCode(200); + return retBody; + } + } + + //Show a db //the topics are missing in the return, since in we use @JsonBackReference on Db's topics //need to the the following method to retrieve the topic list @GetMapping("/{dbName}") @ResponseBody @ApiOperation(value="Get a database's details.") public Db getDb(@PathVariable("dbName") String dbName, HttpServletResponse response) throws IOException { - Db db = dbService.getDb(dbName); + /*Db db = dbService.getDb(dbName); + if (db == null) { + sendError(response, 404, "Db not found: " + dbName); + }*/ + Db db = dbRepository.findByName(dbName); if (db == null) { sendError(response, 404, "Db not found: " + dbName); } return db; } - //Read topics in a DB - @GetMapping("/{dbName}/topics") - @ResponseBody - @ApiOperation(value="Get a database's all topics.") - public Set<Topic> getDbTopics(@PathVariable("dbName") String dbName) throws IOException { - Db db = dbService.getDb(dbName); - Set<Topic> topics = db.getTopics(); - return topics; - } //Update Db - @PutMapping("/") + @PutMapping("/{dbName}") @ResponseBody @ApiOperation(value="Update a database.") - public Db updateDb(@RequestBody Db db, BindingResult result, HttpServletResponse response) throws IOException { + public PostReturnBody<DbConfig> updateDb(@PathVariable("dbName") String dbName, @RequestBody DbConfig dbConfig, BindingResult result, HttpServletResponse response) throws IOException { if (result.hasErrors()) { sendError(response, 400, "Error parsing DB: " + result.toString()); return null; } - Db oldDb = dbService.getDb(db.getName()); + Db oldDb = dbService.getDb(dbConfig.getName()); if (oldDb == null) { - sendError(response, 404, "Db not found: " + db.getName()); + sendError(response, 404, "Db not found: " + dbConfig.getName()); return null; } else { - dbRepository.save(db); - return db; + oldDb.setName(dbConfig.getName()); + oldDb.setHost(dbConfig.getHost()); + oldDb.setPort(dbConfig.getPort()); + oldDb.setLogin(dbConfig.getLogin()); + oldDb.setPass(dbConfig.getPassword()); + oldDb.setEncrypt(false); + + if(oldDb.getName().equals("Elecsticsearch") || oldDb.getName().equals("Druid")) + { + oldDb.setDatabase(dbConfig.getDatabase()); + } + dbRepository.save(oldDb); + DbConfig retMsg; + PostReturnBody<DbConfig> retBody = new PostReturnBody<>(); + retMsg = new DbConfig(); + composeRetMessagefromDbConfig(oldDb, retMsg); + retBody.setReturnBody(retMsg); + retBody.setStatusCode(200); + return retBody; } } - @PostMapping("/") + //Delete a db + //the topics are missing in the return, since in we use @JsonBackReference on Db's topics + //need to the the following method to retrieve the topic list + @DeleteMapping("/{dbName}") @ResponseBody - @ApiOperation(value="Create a new database.") - public Db createDb(@RequestBody Db db, BindingResult result, HttpServletResponse response) throws IOException { + @ApiOperation(value="Delete a database.") + public void deleteDb(@PathVariable("dbName") String dbName, HttpServletResponse response) throws IOException { - if (result.hasErrors()) { - sendError(response, 400, "Error parsing DB: " + result.toString()); - return null; + Db delDb = dbRepository.findByName(dbName); + if (delDb == null) { + sendError(response, 404, "Db not found: " + dbName); } + Set<Topic> topicRelation = delDb.getTopics(); + topicRelation.clear(); + dbRepository.save(delDb); + dbRepository.delete(delDb); + response.setStatus(204); + } - Db oldDb = dbService.getDb(db.getName()); - if (oldDb != null) { - sendError(response, 400, "Db already exists: " + db.getName()); + //Read topics in a DB + @GetMapping("/{dbName}/topics") + @ResponseBody + @ApiOperation(value="Get a database's all topics.") + public Set<Topic> getDbTopics(@PathVariable("dbName") String dbName, HttpServletResponse response) throws IOException { + //Db db = dbService.getDb(dbName); + Set<Topic> topics; + try { + Db db = dbRepository.findByName(dbName); + topics = db.getTopics(); + }catch(Exception ex) + { + sendError(response, 404, "DB: " + dbName + " or Topics not found"); return null; - } else { - dbRepository.save(db); - return db; + } + return topics; + } + + + @PostMapping("/verify") + @ResponseBody + @ApiOperation(value="Database connection verification") + public PostReturnBody<DbConfig> verifyDbConnection(@RequestBody DbConfig dbConfig, HttpServletResponse response) throws IOException { + + /* + Not implemented yet. + */ + + response.setStatus(501); + return null; + } + + private void composeRetMessagefromDbConfig(Db db, DbConfig dbConfigMsg) + { + dbConfigMsg.setName(db.getName()); + dbConfigMsg.setHost(db.getHost()); + dbConfigMsg.setPort(db.getPort()); } private void sendError(HttpServletResponse response, int sc, String msg) throws IOException { diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/domain/DbConfig.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/domain/DbConfig.java new file mode 100644 index 00000000..63de2196 --- /dev/null +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/domain/DbConfig.java @@ -0,0 +1,52 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2019 QCT + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.datalake.feeder.controller.domain; + +import java.util.Set; + +import javax.persistence.CascadeType; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.ManyToMany; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonBackReference; + +import lombok.Getter; +import lombok.Setter; + +/** + * JSON request body for DB manipulation. + * + * @author Kate Hsuan + * + */ + +@Getter +@Setter +public class DbConfig { + private String name; + private String host; + private String login; + private String password; + private String database; + private int port; + private String poperties; +} diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/domain/PostReturnBody.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/domain/PostReturnBody.java new file mode 100644 index 00000000..107f494d --- /dev/null +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/domain/PostReturnBody.java @@ -0,0 +1,45 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2019 QCT + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.feeder.controller.domain; + +import lombok.Getter; +import lombok.Setter; + + +/** + * Unified POST return format + * { + * statusCode: int, + * message: { + * return body ... + * } + * } + * + * @author Kate Hsuan + * + */ + +@Setter +@Getter +public class PostReturnBody<T> { + private int statusCode; + private T returnBody; +} diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Db.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Db.java index 3fe14e33..7aaf4e35 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Db.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Db.java @@ -21,17 +21,19 @@ package org.onap.datalake.feeder.domain; import java.util.Set; -import javax.persistence.CascadeType; +import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.FetchType; import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; import javax.persistence.ManyToMany; import javax.persistence.Table; - import com.fasterxml.jackson.annotation.JsonBackReference; - import lombok.Getter; import lombok.Setter; + /** * Domain class representing bid data storage * @@ -44,28 +46,42 @@ import lombok.Setter; @Table(name = "db") public class Db { @Id + @Column(name="`name`") private String name; + @Column(name="`host`") private String host; + + @Column(name="`port`") private Integer port; + + @Column(name="`login`") private String login; + + @Column(name="`pass`") private String pass; + @Column(name="`database_name`") private String database; + + @Column(name="`encrypt`") private Boolean encrypt; - + + @Column(name="`property1`") private String property1; + + @Column(name="`property2`") private String property2; + + @Column(name="`property3`") private String property3; @JsonBackReference - @ManyToMany(mappedBy = "dbs", cascade = CascadeType.ALL) - /* - @ManyToMany(cascade=CascadeType.ALL)//, fetch=FetchType.EAGER) + @ManyToMany(fetch = FetchType.EAGER) @JoinTable( name = "map_db_topic", - joinColumns = { @JoinColumn(name="db_name") }, - inverseJoinColumns = { @JoinColumn(name="topic_name") } - ) */ + joinColumns = { @JoinColumn(name="db_name") }, + inverseJoinColumns = { @JoinColumn(name="topic_name") } + ) protected Set<Topic> topics; public Db() { diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java index 65478702..b7f89fcd 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java @@ -67,7 +67,7 @@ public class Topic { //@ManyToMany(mappedBy = "topics", cascade=CascadeType.ALL) @JsonBackReference //@JsonManagedReference - @ManyToMany(cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @ManyToMany(fetch = FetchType.EAGER) @JoinTable( name = "map_db_topic", joinColumns = { @JoinColumn(name="topic_name") }, inverseJoinColumns = { @JoinColumn(name="db_name") } diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/repository/DbRepository.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/repository/DbRepository.java index ae03f469..b09dcdca 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/repository/DbRepository.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/repository/DbRepository.java @@ -33,4 +33,6 @@ import org.springframework.data.repository.CrudRepository; public interface DbRepository extends CrudRepository<Db, String> {
+ Db findByName(String Name);
+
}
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/CouchbaseService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/CouchbaseService.java index 01c908ef..a717e10a 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/CouchbaseService.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/CouchbaseService.java @@ -59,21 +59,26 @@ public class CouchbaseService { @Autowired private DbService dbService; - Bucket bucket; + Bucket bucket; + private boolean isReady = false; @PostConstruct private void init() { // Initialize Couchbase Connection - - Db couchbase = dbService.getCouchbase(); - Cluster cluster = CouchbaseCluster.create(couchbase.getHost()); - cluster.authenticate(couchbase.getLogin(), couchbase.getPass()); - bucket = cluster.openBucket(couchbase.getDatabase()); - - log.info("Connect to Couchbase {}", couchbase.getHost()); - - // Create a N1QL Primary Index (but ignore if it exists) - bucket.bucketManager().createN1qlPrimaryIndex(true, false); + try { + Db couchbase = dbService.getCouchbase(); + Cluster cluster = CouchbaseCluster.create(couchbase.getHost()); + cluster.authenticate(couchbase.getLogin(), couchbase.getPass()); + bucket = cluster.openBucket(couchbase.getDatabase()); + log.info("Connect to Couchbase {}", couchbase.getHost()); + // Create a N1QL Primary Index (but ignore if it exists) + bucket.bucketManager().createN1qlPrimaryIndex(true, false); + } + catch(Exception ex) + { + isReady = false; + } + isReady = true; } @PreDestroy diff --git a/components/datalake-handler/feeder/src/main/resources/application.properties b/components/datalake-handler/feeder/src/main/resources/application.properties index 42b13459..842d01b3 100644 --- a/components/datalake-handler/feeder/src/main/resources/application.properties +++ b/components/datalake-handler/feeder/src/main/resources/application.properties @@ -1,6 +1,6 @@ server.port = 1680 - +server.servlet.context-path = /datalake/v1 # Spring connection to MariaDB for ORM #spring.jpa.hibernate.ddl-auto=update diff --git a/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java index b94211e9..afaede76 100644 --- a/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java +++ b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * org.onap.holmes.common.aai + * ONAP : DATALAKE * ================================================================================ * Copyright (C) 2018-2019 Huawei. All rights reserved. * ================================================================================ diff --git a/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/service/CouchbaseServiceTest.java b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/service/CouchbaseServiceTest.java new file mode 100755 index 00000000..9e40a2b1 --- /dev/null +++ b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/service/CouchbaseServiceTest.java @@ -0,0 +1,140 @@ +/*- + * ============LICENSE_START======================================================= + * ONAP : DATALAKE + * ================================================================================ + * Copyright (C) 2018-2019 Huawei. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.feeder.service; + +import com.couchbase.client.java.Cluster; +import com.couchbase.client.java.CouchbaseCluster; +import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; +import com.couchbase.mock.Bucket; +import com.couchbase.mock.BucketConfiguration; +import com.couchbase.mock.CouchbaseMock; +import com.couchbase.mock.client.MockClient; +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.onap.datalake.feeder.domain.Topic; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class CouchbaseServiceTest { + protected final BucketConfiguration bucketConfiguration = new BucketConfiguration(); + protected MockClient mockClient; + protected CouchbaseMock couchbaseMock; + protected Cluster cluster; + protected com.couchbase.client.java.Bucket bucket; + protected int carrierPort; + protected int httpPort; + + protected void getPortInfo(String bucket) throws Exception { + httpPort = couchbaseMock.getHttpPort(); + carrierPort = couchbaseMock.getCarrierPort(bucket); + } + + protected void createMock(@NotNull String name, @NotNull String password) throws Exception { + bucketConfiguration.numNodes = 1; + bucketConfiguration.numReplicas = 1; + bucketConfiguration.numVBuckets = 1024; + bucketConfiguration.name = name; + bucketConfiguration.type = Bucket.BucketType.COUCHBASE; + bucketConfiguration.password = password; + ArrayList<BucketConfiguration> configList = new ArrayList<BucketConfiguration>(); + configList.add(bucketConfiguration); + couchbaseMock = new CouchbaseMock(0, configList); + couchbaseMock.start(); + couchbaseMock.waitForStartup(); + } + + protected void createClient() { + cluster = CouchbaseCluster.create(DefaultCouchbaseEnvironment.builder() + .bootstrapCarrierDirectPort(carrierPort) + .bootstrapHttpDirectPort(httpPort) + .build(), "couchbase://127.0.0.1"); + bucket = cluster.openBucket("default"); + } + + @Before + public void setUp() throws Exception { + createMock("default", ""); + getPortInfo("default"); + createClient(); + } + + @After + public void tearDown() { + if (cluster != null) { + cluster.disconnect(); + } + if (couchbaseMock != null) { + couchbaseMock.stop(); + } + if (mockClient != null) { + mockClient.shutdown(); + } + } + + @Test + public void testSaveJsonsWithTopicId() { + + String text = "{ data: { data2 : { value : 'hello'}}}"; + + JSONObject json = new JSONObject(text); + + Topic topic = new Topic("test getMessageId"); + topic.setMessageIdPath("/data/data2/value"); + List<JSONObject> jsons = new ArrayList<>(); + json.put("_ts", 1234); + jsons.add(json); + CouchbaseService couchbaseService = new CouchbaseService(); + couchbaseService.bucket = bucket; + couchbaseService.saveJsons(topic, jsons); + + } + + @Test + public void testSaveJsonsWithOutTopicId() { + + String text = "{ data: { data2 : { value : 'hello'}}}"; + + JSONObject json = new JSONObject(text); + + Topic topic = new Topic("test getMessageId"); + List<JSONObject> jsons = new ArrayList<>(); + json.put("_ts", 1234); + jsons.add(json); + CouchbaseService couchbaseService = new CouchbaseService(); + couchbaseService.bucket = bucket; + couchbaseService.saveJsons(topic, jsons); + } + + @Test + public void testCleanupBucket() { + CouchbaseService couchbaseService = new CouchbaseService(); + couchbaseService.bucket = bucket; + couchbaseService.cleanUp(); + } + +}
\ No newline at end of file |