diff options
Diffstat (limited to 'components/datalake-handler')
13 files changed, 291 insertions, 52 deletions
diff --git a/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts b/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts index b3ed616f..98eef9ab 100644 --- a/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts +++ b/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts @@ -152,7 +152,7 @@ export class RestApiService { getDbEncryptList(flag): Observable<any> { return this.http - .get(prefix + "dbs/list?tool=" + flag) + .get(prefix + "dbs/list?isDb=" + flag) .pipe(retry(1), map(this.extractData), catchError(this.handleError)); } diff --git a/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts b/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts index 1acd6171..b8dbb0e0 100644 --- a/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts +++ b/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts @@ -30,11 +30,7 @@ import { NgxSpinnerService } from "ngx-spinner"; import { ToastrNotificationService } from "src/app/shared/components/toastr-notification/toastr-notification.service"; import {AlertComponent} from "../../../shared/components/alert/alert.component"; import {NewKafkaModalComponent} from "./new-kafka-modal/new-kafka-modal.component"; -import {ModalContentData} from "../../../shared/modules/modal/modal.data"; -import {ModalDemoComponent} from "../../test/modal-demo/modal-demo.component"; -import {ModalComponent} from "../../../shared/modules/modal/modal.component"; import {EditKafkaModalComponent} from "./edit-kafka-modal/edit-kafka-modal.component"; -import {el} from "@angular/platform-browser/testing/src/browser_util"; @Component({ selector: 'app-kafka-list', @@ -51,6 +47,7 @@ export class KafkaListComponent implements OnInit { Kafka_New: Kafka; Kafka_Newbody: Kafka; cardIconPathList: any = []; + kafkaData: any = []; constructor( private kafkaApiService: RestApiService, @@ -74,6 +71,7 @@ export class KafkaListComponent implements OnInit { if (this.kafkas.length > 0) { let a = "assets/icons/kafka_able.svg"; let b = "assets/icons/kafka_disable.svg"; + this.cardIconPathList.splice(0,this.cardIconPathList.length); for (let i = 0; i < this.kafkas.length; i++) { this.cardIconPath = (this.kafkas[i].enabled == true) ? a : b; this.cardIconPathList.push(this.cardIconPath); @@ -98,6 +96,16 @@ export class KafkaListComponent implements OnInit { return data; } + async getKafkaDetailModal(id: number) { + this.kafkaData = []; + this.kafkaData = await this.getKafkaModal(id); + return this.kafkaData; + } + + getKafkaModal(id: number) { + return this.kafkaApiService.getKafka(id).toPromise(); + } + async initKafkasList(kafkaList: []) { let k: Kafka[] = []; if (kafkaList.length > 0) { @@ -208,33 +216,38 @@ export class KafkaListComponent implements OnInit { } editKafkaModal(id: number) { - console.log("id", id) - const index = this.kafkaList.findIndex(t => t.id === id); - const modalRef = this.modalService.open(EditKafkaModalComponent, { - windowClass: "dl-md-modal kafkas", - centered: true - }); - modalRef.componentInstance.editKafka = this.kafkaList[index]; - modalRef.componentInstance.passEntry.subscribe(receivedEntry => { - this.Kafka_New = receivedEntry; - this.kafkaApiService - .updateKafka(this.Kafka_New) - .subscribe( - res => { - if (res.statusCode == 200) { - this.kafkaList[index] = this.Kafka_New; - this.kafkaList = [...this.kafkaList]; - this.notificationService.success("SUCCESSFULLY_UPDATED"); - } else { - this.notificationService.error("FAILED_UPDATED"); + this.getKafkaDetailModal(id).then(data => { + console.log("id", id); + const index = this.kafkaList.findIndex(t => t.id === id); + const modalRef = this.modalService.open(EditKafkaModalComponent, { + windowClass: "dl-md-modal kafkas", + centered: true + }); + modalRef.componentInstance.editKafka = data; + modalRef.componentInstance.passEntry.subscribe(receivedEntry => { + this.Kafka_New = receivedEntry; + this.kafkaApiService + .updateKafka(this.Kafka_New) + .subscribe( + res => { + this.spinner.hide(); + if (res.statusCode == 200) { + this.kafkaList[index] = this.Kafka_New; + this.kafkaList = [...this.kafkaList]; + this.notificationService.success("SUCCESSFULLY_UPDATED"); + this.initList(); + } else { + this.notificationService.error("FAILED_UPDATED"); + } + modalRef.close(); + }, + err => { + this.notificationService.error(err); + modalRef.close(); } - modalRef.close(); - }, - err => { - this.notificationService.error(err); - modalRef.close(); - } - ); - }) + ); + }); + }); + } } diff --git a/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts b/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts index be0d253f..65de6ada 100644 --- a/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts +++ b/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts @@ -50,7 +50,7 @@ export class ToolsComponent implements OnInit { private adminService: AdminService, private notificationService: ToastrNotificationService, private modalService: NgbModal, - private dbApiService: RestApiService, + private dbApiService: RestApiService ) { this.adminService.setTitle("SIDEBAR.DASHBOARDLIST"); this.initData().then(data => { }); diff --git a/components/datalake-handler/feeder/src/assembly/Dockerfile b/components/datalake-handler/feeder/Dockerfile index 7cd6380c..20444f97 100644 --- a/components/datalake-handler/feeder/src/assembly/Dockerfile +++ b/components/datalake-handler/feeder/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:8-jre +FROM openjdk:8-jre-slim MAINTAINER Guobiao Mo <guobiaomo@chinamobile.com> @@ -12,7 +12,7 @@ USER datalake WORKDIR /home/datalake #add the fat jar -COPY target/feeder-1.0.0-SNAPSHOT.jar /home/datalake/ +COPY target/${JAR_FILE} /home/datalake/ COPY src/assembly/run.sh /home/datalake/ CMD ["sh", "run.sh"] diff --git a/components/datalake-handler/feeder/pom.xml b/components/datalake-handler/feeder/pom.xml index 560e1e71..67f62a0f 100644 --- a/components/datalake-handler/feeder/pom.xml +++ b/components/datalake-handler/feeder/pom.xml @@ -1,6 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> @@ -15,108 +14,146 @@ <packaging>jar</packaging> <name>DataLake Feeder</name> + <properties> + <swagger.version>2.9.2</swagger.version> + <dockerfile-maven.version>1.4.13</dockerfile-maven.version> + <docker.image.path>onap/org.onap.dcaegen2.services.datalakefeeder</docker.image.path> + </properties> <dependencies> <dependency> + <groupId>org.jdom</groupId> + <artifactId>jdom2</artifactId> + <version>2.0.6</version> + </dependency> + + <dependency> + <groupId>com.facebook.presto</groupId> + <artifactId>presto-jdbc</artifactId> + <version>0.229</version> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.mariadb.jdbc</groupId> <artifactId>mariadb-java-client</artifactId> + <version>2.4.1</version> </dependency> <dependency> <groupId>org.json</groupId> <artifactId>json</artifactId> + <version>20190722</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> + <version>4.5.10</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> + <version>2.3.1</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-actuator</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-couchbase</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> + <version>${springboot.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-configuration-processor</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> + <version>${elasticsearchjava.version}</version> </dependency> <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> + <version>2.6</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-yaml</artifactId> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-xml</artifactId> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> + <version>2.8.2</version> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> + <version>1.18.10</version> + <scope>provided</scope> </dependency> <dependency> <groupId>io.druid</groupId> <artifactId>tranquility-core_2.11</artifactId> + <version>0.8.3</version> </dependency> <dependency> <groupId>org.apache.velocity</groupId> <artifactId>velocity-engine-core</artifactId> + <version>2.1</version> </dependency> @@ -136,26 +173,27 @@ <dependency> <groupId>org.hibernate</groupId> <artifactId>hibernate-validator</artifactId> - <version>6.0.10.Final</version> + <version>6.1.0.Final</version> </dependency> <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-swagger2</artifactId> - <version>2.9.2</version> + <version>${swagger.version}</version> <scope>compile</scope> </dependency> <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-swagger-ui</artifactId> - <version>2.9.2</version> + <version>${swagger.version}</version> <scope>compile</scope> </dependency> <dependency> <groupId>org.mongodb</groupId> <artifactId>mongo-java-driver</artifactId> + <version>${mongojava.version}</version> </dependency> <dependency> <groupId>com.couchbase.mock</groupId> @@ -165,4 +203,24 @@ </dependency> </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.spotify</groupId> + <artifactId>dockerfile-maven-plugin</artifactId> + <version>${dockerfile-maven.version}</version> + <configuration> + <!-- repository>docker.io/moguobiao/datalake-feeder-maven</repository--> + <repository>${onap.nexus.dockerregistry.daily}/${docker.image.path}</repository> + <tag>${project.version}</tag> + <dockerfile>Dockerfile</dockerfile> + <!-- useMavenSettingsForAuth>true</useMavenSettingsForAuth--> + <buildArgs> + <JAR_FILE>${project.build.finalName}.jar</JAR_FILE> + </buildArgs> + </configuration> + </plugin> + </plugins> + </build> </project> diff --git a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql b/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql index 8a914270..3f495e2d 100644 --- a/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql +++ b/components/datalake-handler/feeder/src/assembly/scripts/init_db.sql @@ -2,7 +2,7 @@ * ============LICENSE_START=======================================================
* ONAP : DATALAKE
* ================================================================================
-* Copyright 2019 China Mobile
+* Copyright 2019-2020 China Mobile
*=================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -98,7 +98,7 @@ CREATE TABLE `kafka` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `topic` (
- `id` int(11) NOT NULL,
+ `id` int(11) NOT NULL AUTO_INCREMENT,
`aggregate_array_path` varchar(255) DEFAULT NULL,
`correlate_cleared_message` bit(1) NOT NULL DEFAULT b'0',
`data_format` varchar(255) DEFAULT NULL,
diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java index 41995e04..9865e0af 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java @@ -121,14 +121,14 @@ public class KafkaController { Kafka oldKafka = kafkaService.getKafkaById(id); if (oldKafka == null) { - sendError(response, 400, "kafka not found "+id); + sendError(response, 400, "kafka not found, ID: "+id); } else { kafkaRepository.delete(oldKafka); response.setStatus(204); } } - @GetMapping("") + /*@GetMapping("") @ResponseBody @ApiOperation(value="List all Kafka id") public List<Integer> list() { @@ -139,12 +139,30 @@ public class KafkaController { retString.add(k.getId()); } return retString; - } + }*/ + @GetMapping("") + @ResponseBody + @ApiOperation(value="List all Kafkas") public List<KafkaConfig> queryAllKafka(){ return kafkaService.getAllKafka(); } + @GetMapping("/{id}") + @ResponseBody + @ApiOperation(value="Get detail of kafka by id") + public KafkaConfig getKafkaDetail(@PathVariable int id, HttpServletResponse response) throws IOException { + log.info("Get detail of kafka, ID: " + id); + Kafka oldKafka = kafkaService.getKafkaById(id); + if (oldKafka == null) { + sendError(response, 400, "kafka not found, ID: "+id); + return null; + } else { + log.info("ResponseBody......" + oldKafka.getKafkaConfig()); + return oldKafka.getKafkaConfig(); + } + } + private PostReturnBody<KafkaConfig> mkPostReturnBody(int statusCode, Kafka kafka) { PostReturnBody<KafkaConfig> retBody = new PostReturnBody<>(); retBody.setStatusCode(statusCode); diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/TopicNameController.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/TopicNameController.java new file mode 100644 index 00000000..570dcd1c --- /dev/null +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/TopicNameController.java @@ -0,0 +1,54 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : DataLake + * ================================================================================ + * Copyright 2019-2020 China Mobile + *================================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.feeder.controller; + +import io.swagger.annotations.ApiOperation; +import org.onap.datalake.feeder.domain.TopicName; +import org.onap.datalake.feeder.repository.TopicNameRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; + +@RestController +@RequestMapping(value = "/topicNames", produces = { MediaType.APPLICATION_JSON_VALUE }) +public class TopicNameController { + + @Autowired + private TopicNameRepository topicNameRepository; + + @GetMapping("") + @ResponseBody + @ApiOperation(value="List all topicNames") + public List<String> list() { + Iterable<TopicName> ret = topicNameRepository.findAll(); + List<String> retString = new ArrayList<>(); + for(TopicName item : ret) { + retString.add(item.getId()); + } + return retString; + } +} diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java index 0de004d4..fcbe613e 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/domain/Topic.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * ONAP : DataLake * ================================================================================ -* Copyright 2019 China Mobile +* Copyright 2019-2020 China Mobile *================================================================================= * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.Map; +import java.util.HashMap; import javax.persistence.Column; import javax.persistence.Entity; @@ -33,6 +35,8 @@ import javax.persistence.JoinTable; import javax.persistence.ManyToMany; import javax.persistence.ManyToOne; import javax.persistence.Table; +import javax.persistence.GenerationType; +import javax.persistence.GeneratedValue; import org.apache.commons.lang3.StringUtils; import org.json.JSONObject; @@ -57,6 +61,7 @@ import lombok.Setter; public class Topic { @Id @Column(name = "`id`") + @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; @ManyToOne(fetch = FetchType.EAGER) @@ -201,16 +206,27 @@ public class Topic { Set<Db> topicDb = getDbs(); List<Integer> dbList = new ArrayList<>(); List<Integer> enabledDbList = new ArrayList<>(); + List<String> enabledDbList2 = new ArrayList<>(); if (topicDb != null) { for (Db item : topicDb) { dbList.add(item.getId()); if(item.isEnabled()) { enabledDbList.add(item.getId()); + enabledDbList2.add(item.getDbType().getId()); } } } tConfig.setSinkdbs(dbList); tConfig.setEnabledSinkdbs(enabledDbList); + Map<String,Integer> map = new HashMap<>(); + for (String string : enabledDbList2) { + if(map.containsKey(string)) { + map.put(string, map.get(string).intValue()+1); + }else { + map.put(string, new Integer(1)); + } + } + tConfig.setCountsDb(map); Set<Kafka> topicKafka = getKafkas(); List<Integer> kafkaList = new ArrayList<>(); @@ -220,6 +236,7 @@ public class Topic { } } tConfig.setKafkas(kafkaList); + tConfig.setCountsKafka(kafkaList.size()); return tConfig; } diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java index 1bdad2ec..c865ec91 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/dto/TopicConfig.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * ONAP : DataLake * ================================================================================ - * Copyright 2019 QCT + * Copyright 2019-2020 QCT *================================================================================= * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.Setter; import java.util.List; +import java.util.Map; /** * JSON request body for Topic manipulation. @@ -37,7 +38,7 @@ import java.util.List; public class TopicConfig { - private int id; + private Integer id; private String name; private String login; private String password; @@ -52,7 +53,9 @@ public class TopicConfig { private String aggregateArrayPath; private String flattenArrayPath; private List<Integer> kafkas; - + private Map<String,Integer> countsDb; + private int countsKafka; + @Override public String toString() { return String.format("TopicConfig %s(enabled=%s, enabledSinkdbs=%s)", name, enabled, enabledSinkdbs); diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java index a02cd6a2..3bdbcdba 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java @@ -23,6 +23,7 @@ package org.onap.datalake.feeder.service; import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -60,6 +61,9 @@ public class TopicConfigPollingService implements Runnable { @Autowired private KafkaRepository kafkaRepository; + @Autowired + private TopicNameService topicNameService; + //effectiveTopic Map, 1st key is kafkaId, 2nd is topic name, the value is a list of EffectiveTopic. private Map<Integer, Map<String, List<EffectiveTopic>>> effectiveTopicMap = new HashMap<>(); //private Map<String, TopicConfig> effectiveTopicConfigMap; @@ -114,7 +118,7 @@ public class TopicConfigPollingService implements Runnable { log.info("TopicConfigPollingService started."); while (active) { - try { //sleep first since we already pool in init() + try { //sleep first since we already called poll() in init() Thread.sleep(config.getCheckTopicInterval()); if(!active) { break; @@ -138,6 +142,7 @@ public class TopicConfigPollingService implements Runnable { log.info("activeTopics list is updated, new={}", newTopics); activeTopicMap.put(kafkaId, newTopics); + //update version currentActiveTopicsVersionMap.put(kafkaId, currentActiveTopicsVersionMap.getOrDefault(kafkaId, 1)+1); } else { log.debug("activeTopics list is not updated."); @@ -156,14 +161,20 @@ public class TopicConfigPollingService implements Runnable { } private Map<Integer, Set<String>> poll() throws IOException { + Set<String> allTopicNames = new HashSet<>(); + Map<Integer, Set<String>> ret = new HashMap<>(); Iterable<Kafka> kafkas = kafkaRepository.findAll(); for (Kafka kafka : kafkas) { if (kafka.isEnabled()) { Set<String> topics = poll(kafka); ret.put(kafka.getId(), topics); + allTopicNames.addAll(topics); } } + + topicNameService.update(allTopicNames); + return ret; } diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java new file mode 100644 index 00000000..021d2c94 --- /dev/null +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java @@ -0,0 +1,59 @@ +/* +* ============LICENSE_START======================================================= +* ONAP : DATALAKE +* ================================================================================ +* Copyright 2019 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +package org.onap.datalake.feeder.service; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.commons.collections.CollectionUtils; +import org.onap.datalake.feeder.domain.TopicName; +import org.onap.datalake.feeder.repository.TopicNameRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * Service for TopicName + * + * @author Guobiao Mo + * + */ +@Service +public class TopicNameService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private TopicNameRepository topicNameRepository; + + public void update(Collection<String> allTopicNames) { + + List<TopicName> all = allTopicNames.stream().map(s-> new TopicName(s)).collect(Collectors.toList()); + List<TopicName> allInDb = (List<TopicName>) topicNameRepository.findAll(); + + Collection<TopicName> additions = CollectionUtils.subtract(all, allInDb); + + if(!additions.isEmpty()) + topicNameRepository.saveAll(additions); + + } +} diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java index 2f0761ab..c26d9802 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * ONAP : DATALAKE * ================================================================================ -* Copyright 2019 China Mobile +* Copyright 2019-2020 China Mobile *================================================================================= * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,7 @@ import org.onap.datalake.feeder.domain.Db; import org.onap.datalake.feeder.domain.EffectiveTopic; import org.onap.datalake.feeder.domain.Kafka; import org.onap.datalake.feeder.domain.Topic; +import org.onap.datalake.feeder.domain.TopicName; import org.onap.datalake.feeder.repository.DbRepository; import org.onap.datalake.feeder.repository.KafkaRepository; import org.onap.datalake.feeder.repository.TopicNameRepository; @@ -101,6 +102,8 @@ public class TopicService { return ret; } + // for unique topic string, one can create multiple 'topic' in admin UI. + // for example, one 'topic' setting correlates events, and sends data to ES, another 'topic' sends data to HDFS without such setting //TODO use query public List<Topic> findTopics(Kafka kafka, String topicStr) { List<Topic> ret = new ArrayList<>(); @@ -147,7 +150,10 @@ public class TopicService { private void fillTopic(TopicConfig tConfig, Topic topic) { Set<Db> relateDb = new HashSet<>(); topic.setId(tConfig.getId()); - topic.setTopicName(topicNameRepository.findById(tConfig.getName()).get()); + Optional<TopicName> t = topicNameRepository.findById(tConfig.getName()); + if (!t.isPresent()) + throw new IllegalArgumentException("Can not find topicName in TopicName, topic name " + tConfig.getName()); + topic.setTopicName(t.get()); topic.setLogin(tConfig.getLogin()); topic.setPass(tConfig.getPassword()); topic.setEnabled(tConfig.isEnabled()); |