From 90ca9ee68591d24d9fc6797d69135f05f4332b2c Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 3 Sep 2020 16:36:54 +0800 Subject: data extraction service Issue-ID: DCAEGEN2-2258 Signed-off-by: Kai Lu Change-Id: I9089acd786dfcf4d9b0c77e64f5deca3b7c4e6f6 --- components/datalake-handler/des/Dockerfile | 2 +- components/datalake-handler/des/pom.xml | 112 ++++--- .../datalake-handler/des/src/assembly/run.sh | 15 +- .../java/org/onap/datalake/des/DesApplication.java | 10 +- .../org/onap/datalake/des/DesSwaggerConfig.java | 67 ++++ .../java/org/onap/datalake/des/SwaggerConfig.java | 67 ---- .../des/controller/DataExposureController.java | 342 ++++++++++----------- .../datalake/des/controller/PostReturnBody.java | 45 +++ .../org/onap/datalake/des/domain/DataExposure.java | 65 ++-- .../main/java/org/onap/datalake/des/domain/Db.java | 146 +++++++++ .../java/org/onap/datalake/des/domain/DbType.java | 98 ++++++ .../onap/datalake/des/dto/DataExposureConfig.java | 4 +- .../java/org/onap/datalake/des/dto/DbConfig.java | 48 +++ .../des/repository/DataExposureRepository.java | 7 +- .../onap/datalake/des/repository/DbRepository.java | 39 +++ .../datalake/des/repository/DbTypeRepository.java | 35 +++ .../datalake/des/service/DataExposureService.java | 121 ++++---- .../des/src/main/resources/application.properties | 42 +-- .../org/onap/datalake/des/ApplicationTest.java | 31 ++ .../onap/datalake/des/DesSwaggerConfigTest.java | 47 +++ .../des/controller/DataExposureControllerTest.java | 75 +++++ .../onap/datalake/des/domain/DataExposureTest.java | 52 ++++ .../java/org/onap/datalake/des/domain/DbTest.java | 57 ++++ .../org/onap/datalake/des/domain/DbTypeTest.java | 62 ++++ .../java/org/onap/datalake/des/util/TestUtil.java | 50 +++ .../des/src/test/resources/application.properties | 50 +++ 26 files changed, 1264 insertions(+), 425 deletions(-) create mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesSwaggerConfig.java delete mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/SwaggerConfig.java create mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/PostReturnBody.java create mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/Db.java create mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DbType.java create mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DbConfig.java create mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbRepository.java create mode 100644 components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbTypeRepository.java create mode 100644 components/datalake-handler/des/src/test/java/org/onap/datalake/des/ApplicationTest.java create mode 100644 components/datalake-handler/des/src/test/java/org/onap/datalake/des/DesSwaggerConfigTest.java create mode 100644 components/datalake-handler/des/src/test/java/org/onap/datalake/des/controller/DataExposureControllerTest.java create mode 100644 components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DataExposureTest.java create mode 100644 components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTest.java create mode 100644 components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTypeTest.java create mode 100644 components/datalake-handler/des/src/test/java/org/onap/datalake/des/util/TestUtil.java create mode 100644 components/datalake-handler/des/src/test/resources/application.properties diff --git a/components/datalake-handler/des/Dockerfile b/components/datalake-handler/des/Dockerfile index a5d590a3..622a45cd 100644 --- a/components/datalake-handler/des/Dockerfile +++ b/components/datalake-handler/des/Dockerfile @@ -17,7 +17,7 @@ # ============LICENSE_END===================================================== FROM openjdk:11-jre-slim -MAINTAINER Guobiao Mo +MAINTAINER Kai Lu EXPOSE 1681 diff --git a/components/datalake-handler/des/pom.xml b/components/datalake-handler/des/pom.xml index 9554052f..e62fcb6f 100644 --- a/components/datalake-handler/des/pom.xml +++ b/components/datalake-handler/des/pom.xml @@ -1,23 +1,16 @@ - + 1.1.0-SNAPSHOT - org.onap.dcaegen2.services.components.datalake-handler des jar DataLake Extraction Service @@ -53,7 +45,7 @@ com.facebook.presto presto-jdbc - 0.229 + 0.240 @@ -225,12 +217,6 @@ test - - org.onap.dcaegen2.services.components.datalake-handler - feeder - ${project.version} - - @@ -239,6 +225,16 @@ org.springframework.boot spring-boot-maven-plugin + + execute + + + + repackage + + + + com.spotify @@ -257,20 +253,54 @@ ${project.build.finalName}.jar - + diff --git a/components/datalake-handler/des/src/assembly/run.sh b/components/datalake-handler/des/src/assembly/run.sh index 363daf6e..42e6f134 100644 --- a/components/datalake-handler/des/src/assembly/run.sh +++ b/components/datalake-handler/des/src/assembly/run.sh @@ -23,12 +23,19 @@ echo "start init db ..." echo "finish init db" -cmd=`find . -regex '\./feeder-[0-9]+\.[0-9]+\.[0-9]+[-SNAPSHOT]+\.jar'` -cmd1=`find . -regex '\./feeder-[0-9]+\.[0-9]+\.[0-9]+\.jar'` +cmd=`find . -regex '\./des-[0-9]+\.[0-9]+\.[0-9]+[-SNAPSHOT]+\.jar'` +cmd1=`find . -regex '\./des-[0-9]+\.[0-9]+\.[0-9]+\.jar'` +cmd2=`find . -regex '\./des-[0-9]+\.[0-9]+\.[0-9]+[-execute]+\.jar'` +cmd3=`find . -regex '\./des-[0-9]+\.[0-9]+\.[0-9]+[-SNAPSHOT]+[-execute]+\.jar'` + if [ -n "$cmd" ]; then - java -jar $cmd + java -jar "$cmd" elif [ -n "$cmd1" ]; then - java -jar $cmd1 + java -jar "$cmd1" +elif [ -n "$cmd2" ]; then + java -jar "$cmd2" +elif [ -n "$cmd3" ]; then + java -jar "$cmd3" else echo "STRING is empty" sleep 10000 diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java index afb0fef9..7689b279 100644 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesApplication.java @@ -26,18 +26,18 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import springfox.documentation.swagger2.annotations.EnableSwagger2; /** - * Entry point of the Data Extraction Service application + * Entry point of the Data Extraction Service application. * * @author Kai Lu * */ -@SpringBootApplication +@SpringBootApplication(scanBasePackages = { "org.onap.datalake.*" }) @EnableSwagger2 public class DesApplication { - public static void main(String[] args) { - SpringApplication.run(DesApplication.class, args); - } + public static void main(String[] args) { + SpringApplication.run(DesApplication.class, args); + } } diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesSwaggerConfig.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesSwaggerConfig.java new file mode 100644 index 00000000..78443481 --- /dev/null +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/DesSwaggerConfig.java @@ -0,0 +1,67 @@ +/* +* ============LICENSE_START======================================================= +* ONAP : Data Extraction Service +* ================================================================================ +* Copyright 2020 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +package org.onap.datalake.des; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +/** + * For Swagger integration. + * + * @author Kai Lu + * + */ + +@Configuration +@EnableSwagger2 +public class DesSwaggerConfig { + + /** + * des produceApi. + * + * @return Docket Docket + * + */ + @Bean + public Docket desProduceApi() { + return new Docket(DocumentationType.SWAGGER_2).apiInfo(desApiInfo()).select() + .apis(RequestHandlerSelectors.basePackage("org.onap.datalake.des")).paths(PathSelectors.any()).build(); + } + + /** + * des Api description. + * + * @return ApiInfo des api Info + * + */ + private ApiInfo desApiInfo() { + return new ApiInfoBuilder().title("DataLake Rest APIs") + .description("This page lists all the rest apis for DataLake.").version("1.0.0-SNAPSHOT").build(); + } +} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/SwaggerConfig.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/SwaggerConfig.java deleted file mode 100644 index 79022e50..00000000 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/SwaggerConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -/* -* ============LICENSE_START======================================================= -* ONAP : Data Extraction Service -* ================================================================================ -* Copyright 2020 China Mobile -*================================================================================= -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* ============LICENSE_END========================================================= -*/ - -package org.onap.datalake.des; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import springfox.documentation.builders.ApiInfoBuilder; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.builders.RequestHandlerSelectors; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; - -/** - * For Swagger integration - * - * @author Kai Lu - * - */ - -@Configuration -@EnableSwagger2 -public class SwaggerConfig { - - /** - * produceApi. - * - * @return Docket Docket - * - */ - @Bean - public Docket produceApi() { - return new Docket(DocumentationType.SWAGGER_2).apiInfo(apiInfo()).select() - .apis(RequestHandlerSelectors.basePackage("org.onap.datalake.des")).paths(PathSelectors.any()).build(); - } - - /** - * Api description. - * - * @return ApiInfo api Info - * - */ - private ApiInfo apiInfo() { - return new ApiInfoBuilder().title("DataLake Rest APIs") - .description("This page lists all the rest apis for DataLake.").version("1.0.0-SNAPSHOT").build(); - } -} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/DataExposureController.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/DataExposureController.java index 925afb89..d66adb4c 100644 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/DataExposureController.java +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/DataExposureController.java @@ -20,6 +20,9 @@ package org.onap.datalake.des.controller; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; + import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; @@ -40,13 +43,11 @@ import org.onap.datalake.des.domain.DataExposure; import org.onap.datalake.des.dto.DataExposureConfig; import org.onap.datalake.des.repository.DataExposureRepository; import org.onap.datalake.des.service.DataExposureService; -import org.onap.datalake.feeder.controller.domain.PostReturnBody; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.validation.BindingResult; -import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -56,11 +57,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; - /** - * Data Exposure WS. + * Data Extraction WS. * * @author Kai Lu * @@ -70,211 +68,201 @@ import io.swagger.annotations.ApiOperation; @Api(value = "/exposure", consumes = "application/json", produces = "application/json") public class DataExposureController { - private final Logger log = LoggerFactory.getLogger(this.getClass()); - @Autowired - private DataExposureService dataExposureService; - @Autowired - private DataExposureRepository dataExposureRepository; + private final Logger log = LoggerFactory.getLogger(DataExposureController.class); + + @Autowired + private DataExposureService dataExposureService; + + @Autowired + private DataExposureRepository dataExposureRepository; + + private static final String PRESTO_URL = "jdbc:presto://dl-presto:9000/%s/%s"; + + private static final String PRESTO_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; /** * serve API. * - * @param serviceId serviceId - * @param requestMap requestMap + * @param serviceId serviceId + * @param requestMap requestMap * @param bindingResult bindingResult - * @param response response - * @return message that application is started - * @throws IOException - * @throws SQLException + * @param response response + * @return message that application is started + * @throws IOException IO exception + * @throws SQLException sql exception + * @throws ClassNotFoundException class not found exception * */ - @PostMapping("/{serviceId}") - @ResponseBody - @ApiOperation(value = "Datalake Data Exposure Service.") - public HashMap serve(@PathVariable String serviceId, @RequestBody Map requestMap, - BindingResult bindingResult, HttpServletResponse response) throws IOException, SQLException { - log.info("Going to start Datalake Data Exposure Service ... requestMap=" + requestMap); - HashMap ret = new HashMap<>(); - ret.put("request", requestMap); - DataExposure dataExposure = dataExposureService.getDataExposure(serviceId); - String sqlTemplate = dataExposure.getSqlTemplate(); - StringSubstitutor sub = new StringSubstitutor(requestMap); - String query = sub.replace(sqlTemplate); - log.info("Going to start Datalake Data Exposure Service ... query=" + query); - // https://prestodb.io/docs/current/installation/jdbc.html - String url = String.format("jdbc:presto://dl-presto:8080/%s/%s", dataExposure.getDb().getHost(), - dataExposure.getDb().getDatabase()); - Properties properties = new Properties(); - properties.setProperty("user", "test"); - // properties.setProperty("password", "secret"); - // properties.setProperty("SSL", "true"); - Connection connection = DriverManager.getConnection(url, properties); - Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(query); - ResultSetMetaData meta = rs.getMetaData(); - int columnCount = meta.getColumnCount(); - ArrayList> result = new ArrayList<>(); - int count = 0; - while (rs.next()) { - HashMap entry = new HashMap<>(); - for (int i = 1; i <= columnCount; i++) { - String label = meta.getColumnLabel(i); - Object value = rs.getObject(i); - entry.put(label, value); - log.info(label + "\t" + value); - } - result.add(entry); - count++; - } - ret.put("result", result); - ret.put("result_count", count); - return ret; - } + @PostMapping("/{serviceId}") + @ResponseBody + @ApiOperation(value = "Datalake Data Exposure Service.") + public HashMap serve(@PathVariable String serviceId, @RequestBody Map requestMap, + BindingResult bindingResult, HttpServletResponse response) + throws IOException, SQLException, ClassNotFoundException { + log.info("Going to start Datalake Data Exposure Service ... requestMap=" + requestMap); + HashMap ret = new HashMap<>(); + ret.put("request", requestMap); + DataExposure dataExposure = dataExposureService.getDataExposure(serviceId); + String sqlTemplate = dataExposure.getSqlTemplate(); + StringSubstitutor sub = new StringSubstitutor(requestMap); + String query = sub.replace(sqlTemplate); + log.info("Going to start Datalake Data Exposure Service ... query=" + query); + // https://prestodb.io/docs/current/installation/jdbc.html + + Class.forName(PRESTO_DRIVER); + String url = String.format(PRESTO_URL, dataExposure.getDb().getDbType().getName().toLowerCase(), + dataExposure.getDb().getDatabase().toLowerCase()); + + Properties properties = new Properties(); + properties.setProperty("user", "test"); + // properties.setProperty("password", "secret"); + // properties.setProperty("SSL", "true"); + Connection connection = DriverManager.getConnection(url, properties); + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(query); + ResultSetMetaData meta = rs.getMetaData(); + int columnCount = meta.getColumnCount(); + ArrayList> result = new ArrayList<>(); + int count = 0; + while (rs.next()) { + HashMap entry = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + String label = meta.getColumnLabel(i); + Object value = rs.getObject(i); + entry.put(label, value); + log.info(label + "\t" + value); + } + result.add(entry); + count++; + } + ret.put("result", result); + ret.put("result_count", count); + return ret; + } /** * queryAllDataExposure API. * - * @return data exposure config list + * @return data exposure config list * */ - @GetMapping("") - @ResponseBody - @ApiOperation(value = "Datalake Data Exposure list") - public List queryAllDataExposure() { - return dataExposureService.queryAllDataExposure(); - } + @GetMapping("") + @ResponseBody + @ApiOperation(value = "Datalake Data Exposure list") + public List queryAllDataExposure() { + return dataExposureService.queryAllDataExposure(); + } /** * query API. * - * @param id id - * @param response HttpServletResponse - * @return DataExposureConfig - * @throws IOException - * - */ - @GetMapping("/{id}") - @ResponseBody - @ApiOperation(value = "Get Detail of DataExposure") - public DataExposureConfig queryAllDataExposure(@PathVariable String id, HttpServletResponse response) - throws IOException { - log.info("Get Detail of DataExposure Starting....."); - DataExposure oldDataExposure = dataExposureService.getDataExposureById(id); - if (oldDataExposure == null) { - sendError(response, 400, "DataExposure not found, ID: " + id); - return null; - } else { - log.info("ResponseBody......" + oldDataExposure.getDataExposureConfig()); - return oldDataExposure.getDataExposureConfig(); - } - } - - /** - * delete Kfaka API. - * - * @param id id + * @param id id * @param response HttpServletResponse - * @throws IOException + * @return DataExposureConfig + * @throws IOException * */ - @DeleteMapping("/{id}") - @ResponseBody - @ApiOperation(value = "delete a dataExposure.") - public void deleteKafka(@PathVariable String id, HttpServletResponse response) throws IOException { - DataExposure oldDataExposure = dataExposureService.getDataExposureById(id); - if (oldDataExposure == null) { - sendError(response, 400, "DataExposure not found, ID: " + id); - } else { - dataExposureRepository.delete(oldDataExposure); - response.setStatus(204); - } - } + @GetMapping("/{id}") + @ResponseBody + @ApiOperation(value = "Get Detail of DataExposure") + public DataExposureConfig queryAllDataExposure(@PathVariable String id, HttpServletResponse response) + throws IOException { + log.info("Get Detail of DataExposure Starting....."); + DataExposure oldDataExposure = dataExposureService.getDataExposureById(id); + if (oldDataExposure == null) { + sendError(response, 400, "DataExposure not found, ID: " + id); + return null; + } else { + log.info("ResponseBody......" + oldDataExposure.getDataExposureConfig()); + return oldDataExposure.getDataExposureConfig(); + } + } /** * Create a DataExposure. * * @param dataExposureConfig dataExposureConfig - * @param result BindingResult - * @param response HttpServletResponse - * @return DataExposureConfig - * @throws IOException + * @param result BindingResult + * @param response HttpServletResponse + * @return DataExposureConfig + * @throws IOException * */ - @PostMapping("") - @ResponseBody - @ApiOperation(value = "Create a DataExposure.") - public PostReturnBody createDataExposure(@RequestBody DataExposureConfig dataExposureConfig, - BindingResult result, HttpServletResponse response) throws IOException { - if (result.hasErrors()) { - sendError(response, 400, "Error parsing DataExposureConfig : " + result.toString()); - return null; - } - DataExposure oldDataExposure = dataExposureService.getDataExposureById(dataExposureConfig.getId()); - if (oldDataExposure != null) { - sendError(response, 400, "DataExposure is exist " + dataExposureConfig.getId()); - return null; - } else { - DataExposure dataExposure = null; - try { - dataExposure = dataExposureService.fillDataExposureConfiguration(dataExposureConfig); - } catch (Exception e) { - log.debug("FillDataExposureConfiguration failed", e.getMessage()); - sendError(response, 400, "Error FillDataExposureConfiguration: " + e.getMessage()); - return null; - } - dataExposureRepository.save(dataExposure); - log.info("Kafka save successed"); - return mkPostReturnBody(200, dataExposure); - } - } + @PostMapping("") + @ResponseBody + @ApiOperation(value = "Create a DataExposure.") + public PostReturnBody createDataExposure(@RequestBody DataExposureConfig dataExposureConfig, + BindingResult result, HttpServletResponse response) throws IOException { + if (result.hasErrors()) { + sendError(response, 400, "Error parsing DataExposureConfig : " + result.toString()); + return null; + } + DataExposure oldDataExposure = dataExposureService.getDataExposureById(dataExposureConfig.getId()); + if (oldDataExposure != null) { + sendError(response, 400, "DataExposure is exist " + dataExposureConfig.getId()); + return null; + } else { + DataExposure dataExposure = null; + try { + dataExposure = dataExposureService.fillDataExposureConfiguration(dataExposureConfig); + } catch (Exception e) { + log.debug("FillDataExposureConfiguration failed", e.getMessage()); + sendError(response, 400, "Error FillDataExposureConfiguration: " + e.getMessage()); + return null; + } + dataExposureRepository.save(dataExposure); + log.info("Kafka save successed"); + return mkPostReturnBody(200, dataExposure); + } + } /** * Update a DataExposure. * * @param dataExposureConfig dataExposureConfig - * @param result BindingResult - * @param id id - * @param response HttpServletResponse - * @return DataExposureConfig - * @throws IOException + * @param result BindingResult + * @param id id + * @param response HttpServletResponse + * @return DataExposureConfig + * @throws IOException * */ - @PutMapping("/{id}") - @ResponseBody - @ApiOperation(value = "Update a DataExposure.") - public PostReturnBody updateDataExposure(@RequestBody DataExposureConfig dataExposureConfig, - BindingResult result, @PathVariable String id, HttpServletResponse response) throws IOException { - if (result.hasErrors()) { - sendError(response, 400, "Error parsing DataExposureConfig : " + result.toString()); - return null; - } - DataExposure oldDataExposure = dataExposureService.getDataExposureById(id); - if (oldDataExposure == null) { - sendError(response, 400, "DataExposure not found: " + id); - return null; - } else { - try { - dataExposureService.fillDataExposureConfiguration(dataExposureConfig, oldDataExposure); - } catch (Exception e) { - log.debug("FillDataExposureConfiguration failed", e.getMessage()); - sendError(response, 400, "Error FillDataExposureConfiguration: " + e.getMessage()); - return null; - } - dataExposureRepository.save(oldDataExposure); - log.info("DataExposure update successed"); - return mkPostReturnBody(200, oldDataExposure); - } - } + @PutMapping("/{id}") + @ResponseBody + @ApiOperation(value = "Update a DataExposure.") + public PostReturnBody updateDataExposure(@RequestBody DataExposureConfig dataExposureConfig, + BindingResult result, @PathVariable String id, HttpServletResponse response) throws IOException { + if (result.hasErrors()) { + sendError(response, 400, "Error parsing DataExposureConfig : " + result.toString()); + return null; + } + DataExposure oldDataExposure = dataExposureService.getDataExposureById(id); + if (oldDataExposure == null) { + sendError(response, 400, "DataExposure not found: " + id); + return null; + } else { + try { + dataExposureService.fillDataExposureConfiguration(dataExposureConfig, oldDataExposure); + } catch (Exception e) { + log.debug("FillDataExposureConfiguration failed", e.getMessage()); + sendError(response, 400, "Error FillDataExposureConfiguration: " + e.getMessage()); + return null; + } + dataExposureRepository.save(oldDataExposure); + log.info("DataExposure update successed"); + return mkPostReturnBody(200, oldDataExposure); + } + } - private PostReturnBody mkPostReturnBody(int statusCode, DataExposure dataExposure) { - PostReturnBody retBody = new PostReturnBody<>(); - retBody.setStatusCode(statusCode); - retBody.setReturnBody(dataExposure.getDataExposureConfig()); - return retBody; - } + private PostReturnBody mkPostReturnBody(int statusCode, DataExposure dataExposure) { + PostReturnBody retBody = new PostReturnBody<>(); + retBody.setStatusCode(statusCode); + retBody.setReturnBody(dataExposure.getDataExposureConfig()); + return retBody; + } - private void sendError(HttpServletResponse response, int sc, String msg) throws IOException { - log.info(msg); - response.sendError(sc, msg); - } + private void sendError(HttpServletResponse response, int sc, String msg) throws IOException { + log.info(msg); + response.sendError(sc, msg); + } } diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/PostReturnBody.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/PostReturnBody.java new file mode 100644 index 00000000..9f1eb369 --- /dev/null +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/controller/PostReturnBody.java @@ -0,0 +1,45 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2019 QCT + * Copyright 2020 China Mobile + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.controller; + +import lombok.Getter; +import lombok.Setter; + +/** + * Unified POST/PUT return format + * { + * statusCode: int, + * message: { + * return body ... + * } + * } + * + * @author Kate + * + */ + +@Setter +@Getter +public class PostReturnBody { + private int statusCode; + private T returnBody; +} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DataExposure.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DataExposure.java index c1347022..2857359f 100644 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DataExposure.java +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DataExposure.java @@ -21,15 +21,22 @@ package org.onap.datalake.des.domain; import com.fasterxml.jackson.annotation.JsonBackReference; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + import lombok.Getter; import lombok.Setter; -import javax.persistence.*; import org.onap.datalake.des.dto.DataExposureConfig; -import org.onap.datalake.feeder.domain.Db; /** - * Domain class representing DataExposure + * Domain class representing DataExposure. * * @author Kai Lu */ @@ -39,38 +46,38 @@ import org.onap.datalake.feeder.domain.Db; @Table(name = "data_exposure") public class DataExposure { - @Id - @Column(name = "`id`") - private String id; - @Column(name = "`sql_template`", nullable = false) - private String sqlTemplate; - @Column(name = "`note`") - private String note; - @ManyToOne(fetch = FetchType.EAGER) - @JoinColumn(name = "db_id", nullable = false) - @JsonBackReference - private Db db; + @Id + @Column(name = "`id`") + private String id; + @Column(name = "`sql_template`", nullable = false) + private String sqlTemplate; + @Column(name = "`note`") + private String note; + @ManyToOne(fetch = FetchType.EAGER) + @JoinColumn(name = "db_id", nullable = false) + @JsonBackReference + private Db db; - public DataExposure() { - } + public DataExposure() { + } - public DataExposure(String id, String sqlTemplate) { - this.id = id; - this.sqlTemplate = sqlTemplate; - } + public DataExposure(String id, String sqlTemplate) { + this.id = id; + this.sqlTemplate = sqlTemplate; + } /** * getDataExposureConfig. * - * @return data exposure config + * @return data exposure config * */ - public DataExposureConfig getDataExposureConfig() { - DataExposureConfig dataExposureConfig = new DataExposureConfig(); - dataExposureConfig.setId(getId()); - dataExposureConfig.setSqlTemplate(getSqlTemplate()); - dataExposureConfig.setNote(getNote()); - dataExposureConfig.setDbId(getDb().getId()); - return dataExposureConfig; - } + public DataExposureConfig getDataExposureConfig() { + DataExposureConfig dataExposureConfig = new DataExposureConfig(); + dataExposureConfig.setId(getId()); + dataExposureConfig.setSqlTemplate(getSqlTemplate()); + dataExposureConfig.setNote(getNote()); + dataExposureConfig.setDbId(getDb().getId()); + return dataExposureConfig; + } } diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/Db.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/Db.java new file mode 100644 index 00000000..5323c274 --- /dev/null +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/Db.java @@ -0,0 +1,146 @@ +/* +* ============LICENSE_START======================================================= +* ONAP : DataLake +* ================================================================================ +* Copyright 2020 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +package org.onap.datalake.des.domain; + +import com.fasterxml.jackson.annotation.JsonBackReference; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToMany; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import lombok.Getter; +import lombok.Setter; + +import org.onap.datalake.des.dto.DbConfig; + +/** + * Domain class representing bid data storage. + * + * @author Guobiao Mo + * + */ +@Setter +@Getter +@Entity +@Table(name = "db") +public class Db { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "`id`") + private int id; + + @Column(name = "`name`") + private String name; + + @Column(name = "`enabled`", nullable = false) + private boolean enabled; + + @Column(name = "`host`") + private String host; + + @Column(name = "`port`") + private Integer port; + + @Column(name = "`login`") + private String login; + + @Column(name = "`pass`") + private String pass; + + @Column(name = "`database_name`") + private String database; + + @Column(name = "`encrypt`") + private boolean encrypt; + + @Column(name = "`property1`") + private String property1; + + @Column(name = "`property2`") + private String property2; + + @Column(name = "`property3`") + private String property3; + + @JsonBackReference + @ManyToOne(fetch = FetchType.EAGER) + @JoinColumn(name = "db_type_id", nullable = false) + private DbType dbType; + + @JsonBackReference + @ManyToMany(fetch = FetchType.EAGER) + @JoinTable(name = "map_db_topic", joinColumns = { @JoinColumn(name = "db_id") }, inverseJoinColumns = { + @JoinColumn(name = "topic_id") }) + + @Override + public String toString() { + return String.format("Db %s (name=%s, enabled=%s)", id, name, enabled); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (this.getClass() != obj.getClass()) { + return false; + } + + return id == ((Db) obj).getId(); + } + + @Override + public int hashCode() { + return id; + } + + /** + * get db config. + * + * @return DbConfig db config + * + */ + public DbConfig getDbConfig() { + + DbConfig dbConfig = new DbConfig(); + + dbConfig.setId(getId()); + dbConfig.setName(getName()); + dbConfig.setHost(getHost()); + dbConfig.setPort(getPort()); + dbConfig.setPass(getPass()); + dbConfig.setLogin(getLogin()); + dbConfig.setEncrypt(isEncrypt()); + dbConfig.setEnabled(isEnabled()); + dbConfig.setDatabase(getDatabase()); + dbConfig.setDbTypeId(getDbType().getId()); + return dbConfig; + } +} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DbType.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DbType.java new file mode 100644 index 00000000..251a6f31 --- /dev/null +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/domain/DbType.java @@ -0,0 +1,98 @@ +/* +* ============LICENSE_START======================================================= +* ONAP : DataLake +* ================================================================================ +* Copyright 2020 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +package org.onap.datalake.des.domain; + +import com.fasterxml.jackson.annotation.JsonBackReference; + +import java.util.HashSet; +import java.util.Set; + +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.OneToMany; +import javax.persistence.Table; + +import lombok.Getter; +import lombok.Setter; + +/** + * Domain class representing bid data storage type. + * + * @author Guobiao Mo + * + */ +@Setter +@Getter +@Entity +@Table(name = "db_type") +public class DbType { + @Id + @Column(name = "`id`") + private String id; + + @Column(name = "`name`", nullable = false) + private String name; + + @Column(name = "`default_port`") + private Integer defaultPort; + + @Column(name = "`tool`", nullable = false) + private boolean tool; + + @JsonBackReference + @OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY, mappedBy = "dbType") + protected Set dbs = new HashSet<>(); + + public DbType() { + } + + public DbType(String id, String name) { + this.id = id; + this.name = name; + } + + @Override + public String toString() { + return String.format("DbType %s (name=%s)", id, name); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (this.getClass() != obj.getClass()) { + return false; + } + + return id.equals(((DbType) obj).getId()); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + +} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DataExposureConfig.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DataExposureConfig.java index 86124f77..df8fbe5f 100644 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DataExposureConfig.java +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DataExposureConfig.java @@ -19,10 +19,12 @@ */ package org.onap.datalake.des.dto; + import lombok.Getter; import lombok.Setter; + /** - * SON request body for DataExposure manipulation. + * Data Exposure Config. * * @author Kai Lu */ diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DbConfig.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DbConfig.java new file mode 100644 index 00000000..aba36b4d --- /dev/null +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/dto/DbConfig.java @@ -0,0 +1,48 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2020 China Mobile + * Copyright 2019 QCT + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.dto; + +import lombok.Getter; +import lombok.Setter; + +/** + * JSON request body for DB manipulation. + * + * @author Kate Hsuan + * + */ + +@Getter +@Setter +public class DbConfig { + private int id; + private String name; + private String host; + private boolean enabled; + private String login; + private String pass; + private boolean encrypt; + private String database; + private Integer port; + private String poperties; + private String dbTypeId; +} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DataExposureRepository.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DataExposureRepository.java index b77e5d29..b611baf4 100644 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DataExposureRepository.java +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DataExposureRepository.java @@ -24,12 +24,11 @@ import org.onap.datalake.des.domain.DataExposure; import org.springframework.data.repository.CrudRepository; /** - * - * DataExposure Repository - * + * DataExposure Repository. + * * @author Kai Lu * - */ + */ public interface DataExposureRepository extends CrudRepository { diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbRepository.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbRepository.java new file mode 100644 index 00000000..aa402d5e --- /dev/null +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbRepository.java @@ -0,0 +1,39 @@ +/* +* ============LICENSE_START======================================================= +* ONAP : DataLake +* ================================================================================ +* Copyright 2020 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +package org.onap.datalake.des.repository; + +import org.onap.datalake.des.domain.Db; +import org.springframework.data.repository.CrudRepository; + +/** + * Db Repository. + * + * @author Guobiao Mo + * + */ + +public interface DbRepository extends CrudRepository { + + Db findByName(String name); + + Iterable findByEncrypt(boolean encrypt); + +} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbTypeRepository.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbTypeRepository.java new file mode 100644 index 00000000..0b141efd --- /dev/null +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/repository/DbTypeRepository.java @@ -0,0 +1,35 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2020 China Mobile + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.repository; + +import org.onap.datalake.des.domain.DbType; +import org.springframework.data.repository.CrudRepository; + +/** + * DbType Repository. + * + * @author Guobiao Mo + */ + +public interface DbTypeRepository extends CrudRepository { + + Iterable findByTool(boolean tool); +} diff --git a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/service/DataExposureService.java b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/service/DataExposureService.java index c7d642b5..6ba41604 100644 --- a/components/datalake-handler/des/src/main/java/org/onap/datalake/des/service/DataExposureService.java +++ b/components/datalake-handler/des/src/main/java/org/onap/datalake/des/service/DataExposureService.java @@ -20,23 +20,22 @@ package org.onap.datalake.des.service; -import java.io.IOException; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Optional; + import org.onap.datalake.des.domain.DataExposure; +import org.onap.datalake.des.domain.Db; import org.onap.datalake.des.dto.DataExposureConfig; import org.onap.datalake.des.repository.DataExposureRepository; -import org.onap.datalake.feeder.domain.Db; -import org.onap.datalake.feeder.repository.DbRepository; +import org.onap.datalake.des.repository.DbRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; /** - * Service for DataExposure + * Service for DataExposure. * * @author Kai Lu * @@ -44,88 +43,100 @@ import org.springframework.stereotype.Service; @Service public class DataExposureService { - private final Logger log = LoggerFactory.getLogger(this.getClass()); - @Autowired - private DataExposureRepository dataExposureRepository; - @Autowired - private DbRepository dbRepository; + private final Logger log = LoggerFactory.getLogger(this.getClass()); + @Autowired + private DataExposureRepository dataExposureRepository; + @Autowired + private DbRepository dbRepository; /** * getDataExposure. * * @param serviceId serviceId * - * @return DataExposure + * @return DataExposure * */ - public DataExposure getDataExposure(String serviceId) { - Optional ret = dataExposureRepository.findById(serviceId); - return ret.isPresent() ? ret.get() : null; - } + public DataExposure getDataExposure(String serviceId) { + Optional ret = dataExposureRepository.findById(serviceId); + return ret.isPresent() ? ret.get() : null; + } - public List queryAllDataExposure() { - List dataExposureList = null; - List dataExposureConfigList = new ArrayList<>(); - dataExposureList = (List) dataExposureRepository.findAll(); - if (!dataExposureList.isEmpty()) { - log.info("DataExposureList is not null"); - for (DataExposure dataExposure : dataExposureList) { - dataExposureConfigList.add(dataExposure.getDataExposureConfig()); - } - } - return dataExposureConfigList; - } + /** + * getDataExposure. + * + * @return DataExposure + * + */ + public List queryAllDataExposure() { + List dataExposureList = null; + List dataExposureConfigList = new ArrayList<>(); + dataExposureList = (List) dataExposureRepository.findAll(); + if (!dataExposureList.isEmpty()) { + log.info("DataExposureList is not null"); + for (DataExposure dataExposure : dataExposureList) { + dataExposureConfigList.add(dataExposure.getDataExposureConfig()); + } + } + return dataExposureConfigList; + } /** * getDataExposureById. * * @param id id * - * @return data exposure + * @return data exposure * */ - public DataExposure getDataExposureById(String id) { - Optional ret = dataExposureRepository.findById(id); - return ret.isPresent() ? ret.get() : null; - } + public DataExposure getDataExposureById(String id) { + Optional ret = dataExposureRepository.findById(id); + return ret.isPresent() ? ret.get() : null; + } /** * fillDataExposureConfiguration. * * @param dataExposureConfig DataExposureConfig * - * @return data exposure + * @return data exposure * */ - public DataExposure fillDataExposureConfiguration(DataExposureConfig dataExposureConfig) { - DataExposure dataExposure = new DataExposure(); - fillDataExposure(dataExposureConfig, dataExposure); - return dataExposure; - } + public DataExposure fillDataExposureConfiguration(DataExposureConfig dataExposureConfig) { + DataExposure dataExposure = new DataExposure(); + fillDataExposure(dataExposureConfig, dataExposure); + return dataExposure; + } /** * fillDataExposureConfiguration. * * @param dataExposureConfig DataExposureConfig - * @param dataExposure DataExposure + * @param dataExposure DataExposure * - * @return data exposure + * @return data exposure * */ - public void fillDataExposureConfiguration(DataExposureConfig dataExposureConfig, DataExposure dataExposure) { - fillDataExposure(dataExposureConfig, dataExposure); - } + public void fillDataExposureConfiguration(DataExposureConfig dataExposureConfig, DataExposure dataExposure) { + fillDataExposure(dataExposureConfig, dataExposure); + } + + private void fillDataExposure(DataExposureConfig dataExposureConfig, DataExposure dataExposure) + throws IllegalArgumentException { + + dataExposure.setId(dataExposureConfig.getId()); + dataExposure.setNote(dataExposureConfig.getNote()); + dataExposure.setSqlTemplate(dataExposureConfig.getSqlTemplate()); + if (dataExposureConfig.getDbId() == null) { + throw new IllegalArgumentException("Can not find db_id in db, db_id: " + dataExposureConfig.getDbId()); + } + + Optional dbOptional = dbRepository.findById(dataExposureConfig.getDbId()); + + if (!dbOptional.isPresent()) { + throw new IllegalArgumentException("db_id is null " + dataExposureConfig.getDbId()); + } - private void fillDataExposure(DataExposureConfig dataExposureConfig, DataExposure dataExposure) - throws IllegalArgumentException { - dataExposure.setId(dataExposureConfig.getId()); - dataExposure.setNote(dataExposureConfig.getNote()); - dataExposure.setSqlTemplate(dataExposureConfig.getSqlTemplate()); - if (dataExposureConfig.getDbId() == null) - throw new IllegalArgumentException("Can not find db_id in db, db_id: " + dataExposureConfig.getDbId()); - Optional dbOptional = dbRepository.findById(dataExposureConfig.getDbId()); - if (!dbOptional.isPresent()) - throw new IllegalArgumentException("db_id is null " + dataExposureConfig.getDbId()); - dataExposure.setDb(dbOptional.get()); - } + dataExposure.setDb(dbOptional.get()); + } } diff --git a/components/datalake-handler/des/src/main/resources/application.properties b/components/datalake-handler/des/src/main/resources/application.properties index c0997e78..5c0a9f02 100644 --- a/components/datalake-handler/des/src/main/resources/application.properties +++ b/components/datalake-handler/des/src/main/resources/application.properties @@ -1,5 +1,5 @@ #####################App general -server.port = 16810 +server.port = 1681 server.servlet.context-path = /datalake/v1 #tolerate inconsistency when system crash, see PullThread.run() @@ -24,34 +24,6 @@ spring.datasource.url=jdbc:mariadb://mariadb-galera:3306/datalake?autoReconnect= spring.datasource.username=dl spring.datasource.password=dl1234 - -#####################DMaaP -dmaapZookeeperHostPort=message-router-zookeeper:2181 -dmaapKafkaHostPort=message-router-kafka:9092 -dmaapKafkaGroup=dlgroup44 -#dmaapKafkaLogin=admin -#dmaapKafkaPass=admin-secret -#dmaapKafkaSecurityProtocol=SASL_PLAINTEXT - -#in second -dmaapKafkaTimeout=10 -dmaapKafkaExclude[0]=__consumer_offsets -dmaapKafkaExclude[1]=__transaction_state -#dmaapKafkaExclude[2]=msgrtr.apinode.metrics.dmaap -#check for new topics , in millisecond -dmaapCheckNewTopicInterval=10000 - -kafkaConsumerCount=3 - -#####################Elasticsearch -elasticsearchType=doc - -#####################HDFS -hdfsBufferSize=4096 -#how often we flush stall updates, in millisecond -hdfsFlushInterval=30000 -hdfsBatchSize=500 - #####################Logging logging.level.org.springframework.web=ERROR logging.level.com.att.nsa.apiClient.http=ERROR @@ -59,15 +31,3 @@ logging.level.org.onap.datalake=DEBUG #####################Verison datalakeVersion=0.0.1 - -#####################KibanaDashboardImportApi -kibanaDashboardImportApi=/api/kibana/dashboards/import?exclude=index-pattern - -#####################KibanaPort -kibanaPort=5601 - -#####################Elasticsearch Template API -esTemplateMappingApi=/_template/ - -#####################Elasticsearch port -esPort=9200 diff --git a/components/datalake-handler/des/src/test/java/org/onap/datalake/des/ApplicationTest.java b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/ApplicationTest.java new file mode 100644 index 00000000..eb2f57f7 --- /dev/null +++ b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/ApplicationTest.java @@ -0,0 +1,31 @@ +/*- + * ============LICENSE_START======================================================= + * ONAP : DATALAKE + * ================================================================================ + * Copyright (C) 2020 China Mobile. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des; + +import org.junit.Test; +import org.onap.datalake.des.DesApplication; + +public class ApplicationTest { + @Test + public void testRunner() { + DesApplication application = new DesApplication(); + } +} diff --git a/components/datalake-handler/des/src/test/java/org/onap/datalake/des/DesSwaggerConfigTest.java b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/DesSwaggerConfigTest.java new file mode 100644 index 00000000..223fb3ce --- /dev/null +++ b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/DesSwaggerConfigTest.java @@ -0,0 +1,47 @@ +/* +* ============LICENSE_START======================================================= +* ONAP : DATALAKE +* ================================================================================ +* Copyright 2020 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +package org.onap.datalake.des; + +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.onap.datalake.des.DesSwaggerConfig; + +/** + * Test Swagger integration. + * + * @author Kai Lu + * + */ + +public class DesSwaggerConfigTest { + + @Test + public void test() { + try { + DesSwaggerConfig config = new DesSwaggerConfig(); + config.desProduceApi(); + } catch (Exception e) { + fail("failed to read configure Des Swagger."); + } + } + +} \ No newline at end of file diff --git a/components/datalake-handler/des/src/test/java/org/onap/datalake/des/controller/DataExposureControllerTest.java b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/controller/DataExposureControllerTest.java new file mode 100644 index 00000000..b3ff026d --- /dev/null +++ b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/controller/DataExposureControllerTest.java @@ -0,0 +1,75 @@ +/*- + * ============LICENSE_START======================================================= + * ONAP : DATALAKE + * ================================================================================ + * Copyright (C) 2020 China Mobile. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.controller; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +import javax.servlet.http.HttpServletResponse; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.onap.datalake.des.repository.DataExposureRepository; +import org.onap.datalake.des.service.DataExposureService; +import org.springframework.validation.BindingResult; + + +/** + * Test Data Exposure Controller. + * + * @author Kai Lu + */ +@RunWith(MockitoJUnitRunner.class) +public class DataExposureControllerTest { + + @Mock + private HttpServletResponse httpServletResponse; + + @Mock + private DataExposureRepository dataExposureRepository; + + @Mock + private BindingResult mockBindingResult; + + @InjectMocks + private DataExposureService dataExposureService; + + @Test(expected = NullPointerException.class) + public void testServe() throws IOException, NoSuchFieldException, + IllegalAccessException, ClassNotFoundException, SQLException { + DataExposureController dataExposureController = new DataExposureController(); + String serviceId = "test"; + Map requestMap = new HashMap(); + requestMap.put("name", "oteNB5309"); + HashMap result = dataExposureController + .serve(serviceId, requestMap, mockBindingResult, httpServletResponse); + assertEquals(null, result); + when(mockBindingResult.hasErrors()).thenReturn(true); + } +} diff --git a/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DataExposureTest.java b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DataExposureTest.java new file mode 100644 index 00000000..4b9bcb4f --- /dev/null +++ b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DataExposureTest.java @@ -0,0 +1,52 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2020 China Mobile + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.domain; + +import static org.junit.Assert.assertNotNull; + +import org.junit.Test; +import org.onap.datalake.des.dto.DataExposureConfig; +import org.onap.datalake.des.util.TestUtil; + +/** + * Test Data Exposure. + * + * @author Kai Lu + */ +public class DataExposureTest { + + @Test + public void test() { + DataExposure dataExposure = new DataExposure("1", + " select event.commonEventHeader.sourceName as name, " + + " event.perf3gppFields.measDataCollection.measuredEntityDn as entity " + + " from datalake where event.commonEventHeader.sourceName = '${name}' "); + + Db mongoDb = TestUtil.newDb("MongoDB"); + dataExposure.setDb(mongoDb); + dataExposure.setNote("testsql"); + DataExposureConfig config = dataExposure.getDataExposureConfig(); + assertNotNull(config.getId()); + assertNotNull(config.getSqlTemplate()); + + } + +} diff --git a/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTest.java b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTest.java new file mode 100644 index 00000000..18cfa386 --- /dev/null +++ b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTest.java @@ -0,0 +1,57 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2020 China Mobile + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.domain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; + +import org.junit.Test; +import org.onap.datalake.des.util.TestUtil; + +/** + * Test Db. + * + * @author Kai Lu + */ + +public class DbTest { + + @Test + public void testIs() { + + Db couchbase = TestUtil.newDb("Couchbase"); + Db mongoDb1 = TestUtil.newDb("MongoDB"); + Db mongoDb2 = TestUtil.newDb("MongoDB"); + assertNotEquals(couchbase.hashCode(), mongoDb1.hashCode()); + assertNotEquals(couchbase, mongoDb1); + assertNotEquals(mongoDb1, mongoDb2); + assertEquals(mongoDb1, mongoDb1); + assertFalse(mongoDb2.equals(null)); + + DbType dbType = new DbType("MONGO", "MongoDB"); + dbType.setTool(false); + mongoDb1.setDbType(dbType); + + assertNotEquals(mongoDb2, dbType); + assertFalse(mongoDb1.getDbType().isTool()); + } +} diff --git a/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTypeTest.java b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTypeTest.java new file mode 100644 index 00000000..a94f0482 --- /dev/null +++ b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/domain/DbTypeTest.java @@ -0,0 +1,62 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2020 China Mobile + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.domain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Test Data Type. + * + * @author Kai Lu + */ +public class DbTypeTest { + + @Test + public void test() { + DbType dbType = new DbType("ES","Elasticsearch"); + + dbType.setTool(false); + + assertNotNull(dbType.toString()); + assertEquals(dbType, dbType); + assertNotEquals(dbType, null); + assertNotEquals(dbType, "ES"); + + DbType dbType2 = new DbType("MONGO", "MongoDB"); + assertNotEquals(dbType, dbType2); + assertNotNull(dbType.hashCode()); + + assertEquals("MongoDB", dbType2.getName()); + dbType2.setName(null); + dbType2.setDefaultPort(1); + assertTrue(1 == dbType2.getDefaultPort()); + + dbType2.setDbs(null); + assertNull(dbType2.getDbs()); + } + +} \ No newline at end of file diff --git a/components/datalake-handler/des/src/test/java/org/onap/datalake/des/util/TestUtil.java b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/util/TestUtil.java new file mode 100644 index 00000000..8d3cf6c9 --- /dev/null +++ b/components/datalake-handler/des/src/test/java/org/onap/datalake/des/util/TestUtil.java @@ -0,0 +1,50 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DCAE + * ================================================================================ + * Copyright 2020 China Mobile + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.des.util; + +import org.onap.datalake.des.domain.Db; +import org.onap.datalake.des.domain.DbType; + + +/** + * test utils. + * + * @author Kai Lu + */ +public class TestUtil { + + static int i = 0; + + /** + * getDataExposureConfig. + * + * @param name name + * @return data exposure config + * + */ + public static Db newDb(String name) { + Db db = new Db(); + db.setId(i++); + db.setName(name); + db.setDbType(new DbType(name, name)); + return db; + } +} diff --git a/components/datalake-handler/des/src/test/resources/application.properties b/components/datalake-handler/des/src/test/resources/application.properties new file mode 100644 index 00000000..64ecdee9 --- /dev/null +++ b/components/datalake-handler/des/src/test/resources/application.properties @@ -0,0 +1,50 @@ +#####################App general +server.port = 1680 +server.servlet.context-path = /datalake/v1 + +#tolerate inconsistency when system crash, see PullThread.run() +async=true + +#SSL global flag, if enabled, still need to check each individual DB SSL flag +enableSSL=false + +#names for extra fields that DL adds to each record +timestampLabel=datalake_ts_ +rawDataLabel=datalake_text_ + +defaultTopicName=_DL_DEFAULT_ + + +#####################DMaaP +#dmaapZookeeperHostPort=127.0.0.1:2181 +#dmaapKafkaHostPort=127.0.0.1:9092 +dmaapZookeeperHostPort=message-router-zookeeper:2181 +dmaapKafkaHostPort=message-router-kafka:9092 +dmaapKafkaGroup=dlgroup44 +#in second +dmaapKafkaTimeout=60 +dmaapKafkaExclude[0]=__consumer_offsets +dmaapKafkaExclude[1]=__transaction_state +#dmaapKafkaExclude[2]=msgrtr.apinode.metrics.dmaap +#check for new topics , in millisecond +dmaapCheckNewTopicInterval=60000 + +kafkaConsumerCount=3 + +#####################Elasticsearch +elasticsearchType=_doc + +#####################HDFS +hdfsBufferSize=4096 +#how often we flush stall updates, in millisecond +hdfsFlushInterval=30000 +hdfsBatchSize=500 + +#####################Logging +logging.level.org.springframework.web=ERROR +logging.level.com.att.nsa.apiClient.http=ERROR +logging.level.org.onap.datalake=DEBUG + +#####################Verison +datalakeVersion=0.0.1 + -- cgit 1.2.3-korg