diff options
9 files changed, 205 insertions, 44 deletions
diff --git a/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts b/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts index 0c520895..dc1a0231 100644 --- a/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts +++ b/components/datalake-handler/admin/src/src/app/core/services/rest-api.service.ts @@ -160,7 +160,7 @@ export class RestApiService { */ getDbEncryptList(flag): Observable<any> { return this.http - .get(prefix + "dbs/list?tool=" + flag) + .get(prefix + "dbs/list?isDb=" + flag) .pipe(retry(1), map(this.extractData), catchError(this.handleError)); } diff --git a/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts b/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts index 1acd6171..b8dbb0e0 100644 --- a/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts +++ b/components/datalake-handler/admin/src/src/app/views/kafka/kafka-list/kafka-list.component.ts @@ -30,11 +30,7 @@ import { NgxSpinnerService } from "ngx-spinner"; import { ToastrNotificationService } from "src/app/shared/components/toastr-notification/toastr-notification.service"; import {AlertComponent} from "../../../shared/components/alert/alert.component"; import {NewKafkaModalComponent} from "./new-kafka-modal/new-kafka-modal.component"; -import {ModalContentData} from "../../../shared/modules/modal/modal.data"; -import {ModalDemoComponent} from "../../test/modal-demo/modal-demo.component"; -import {ModalComponent} from "../../../shared/modules/modal/modal.component"; import {EditKafkaModalComponent} from "./edit-kafka-modal/edit-kafka-modal.component"; -import {el} from "@angular/platform-browser/testing/src/browser_util"; @Component({ selector: 'app-kafka-list', @@ -51,6 +47,7 @@ export class KafkaListComponent implements OnInit { Kafka_New: Kafka; Kafka_Newbody: Kafka; cardIconPathList: any = []; + kafkaData: any = []; constructor( private kafkaApiService: RestApiService, @@ -74,6 +71,7 @@ export class KafkaListComponent implements OnInit { if (this.kafkas.length > 0) { let a = "assets/icons/kafka_able.svg"; let b = "assets/icons/kafka_disable.svg"; + this.cardIconPathList.splice(0,this.cardIconPathList.length); for (let i = 0; i < this.kafkas.length; i++) { this.cardIconPath = (this.kafkas[i].enabled == true) ? a : b; this.cardIconPathList.push(this.cardIconPath); @@ -98,6 +96,16 @@ export class KafkaListComponent implements OnInit { return data; } + async getKafkaDetailModal(id: number) { + this.kafkaData = []; + this.kafkaData = await this.getKafkaModal(id); + return this.kafkaData; + } + + getKafkaModal(id: number) { + return this.kafkaApiService.getKafka(id).toPromise(); + } + async initKafkasList(kafkaList: []) { let k: Kafka[] = []; if (kafkaList.length > 0) { @@ -208,33 +216,38 @@ export class KafkaListComponent implements OnInit { } editKafkaModal(id: number) { - console.log("id", id) - const index = this.kafkaList.findIndex(t => t.id === id); - const modalRef = this.modalService.open(EditKafkaModalComponent, { - windowClass: "dl-md-modal kafkas", - centered: true - }); - modalRef.componentInstance.editKafka = this.kafkaList[index]; - modalRef.componentInstance.passEntry.subscribe(receivedEntry => { - this.Kafka_New = receivedEntry; - this.kafkaApiService - .updateKafka(this.Kafka_New) - .subscribe( - res => { - if (res.statusCode == 200) { - this.kafkaList[index] = this.Kafka_New; - this.kafkaList = [...this.kafkaList]; - this.notificationService.success("SUCCESSFULLY_UPDATED"); - } else { - this.notificationService.error("FAILED_UPDATED"); + this.getKafkaDetailModal(id).then(data => { + console.log("id", id); + const index = this.kafkaList.findIndex(t => t.id === id); + const modalRef = this.modalService.open(EditKafkaModalComponent, { + windowClass: "dl-md-modal kafkas", + centered: true + }); + modalRef.componentInstance.editKafka = data; + modalRef.componentInstance.passEntry.subscribe(receivedEntry => { + this.Kafka_New = receivedEntry; + this.kafkaApiService + .updateKafka(this.Kafka_New) + .subscribe( + res => { + this.spinner.hide(); + if (res.statusCode == 200) { + this.kafkaList[index] = this.Kafka_New; + this.kafkaList = [...this.kafkaList]; + this.notificationService.success("SUCCESSFULLY_UPDATED"); + this.initList(); + } else { + this.notificationService.error("FAILED_UPDATED"); + } + modalRef.close(); + }, + err => { + this.notificationService.error(err); + modalRef.close(); } - modalRef.close(); - }, - err => { - this.notificationService.error(err); - modalRef.close(); - } - ); - }) + ); + }); + }); + } } diff --git a/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts b/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts index be0d253f..65de6ada 100644 --- a/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts +++ b/components/datalake-handler/admin/src/src/app/views/tools/tools.component.ts @@ -50,7 +50,7 @@ export class ToolsComponent implements OnInit { private adminService: AdminService, private notificationService: ToastrNotificationService, private modalService: NgbModal, - private dbApiService: RestApiService, + private dbApiService: RestApiService ) { this.adminService.setTitle("SIDEBAR.DASHBOARDLIST"); this.initData().then(data => { }); diff --git a/components/datalake-handler/feeder/src/assembly/Dockerfile b/components/datalake-handler/feeder/Dockerfile index 7cd6380c..20444f97 100644 --- a/components/datalake-handler/feeder/src/assembly/Dockerfile +++ b/components/datalake-handler/feeder/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:8-jre +FROM openjdk:8-jre-slim MAINTAINER Guobiao Mo <guobiaomo@chinamobile.com> @@ -12,7 +12,7 @@ USER datalake WORKDIR /home/datalake #add the fat jar -COPY target/feeder-1.0.0-SNAPSHOT.jar /home/datalake/ +COPY target/${JAR_FILE} /home/datalake/ COPY src/assembly/run.sh /home/datalake/ CMD ["sh", "run.sh"] diff --git a/components/datalake-handler/feeder/pom.xml b/components/datalake-handler/feeder/pom.xml index 560e1e71..67f62a0f 100644 --- a/components/datalake-handler/feeder/pom.xml +++ b/components/datalake-handler/feeder/pom.xml @@ -1,6 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> @@ -15,108 +14,146 @@ <packaging>jar</packaging> <name>DataLake Feeder</name> + <properties> + <swagger.version>2.9.2</swagger.version> + <dockerfile-maven.version>1.4.13</dockerfile-maven.version> + <docker.image.path>onap/org.onap.dcaegen2.services.datalakefeeder</docker.image.path> + </properties> <dependencies> <dependency> + <groupId>org.jdom</groupId> + <artifactId>jdom2</artifactId> + <version>2.0.6</version> + </dependency> + + <dependency> + <groupId>com.facebook.presto</groupId> + <artifactId>presto-jdbc</artifactId> + <version>0.229</version> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.mariadb.jdbc</groupId> <artifactId>mariadb-java-client</artifactId> + <version>2.4.1</version> </dependency> <dependency> <groupId>org.json</groupId> <artifactId>json</artifactId> + <version>20190722</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> + <version>4.5.10</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> + <version>2.3.1</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-actuator</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-couchbase</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> + <version>${springboot.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-configuration-processor</artifactId> + <version>${springboot.version}</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> + <version>${elasticsearchjava.version}</version> </dependency> <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> + <version>2.6</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-yaml</artifactId> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-xml</artifactId> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> </dependency> <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> + <version>2.8.2</version> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> + <version>1.18.10</version> + <scope>provided</scope> </dependency> <dependency> <groupId>io.druid</groupId> <artifactId>tranquility-core_2.11</artifactId> + <version>0.8.3</version> </dependency> <dependency> <groupId>org.apache.velocity</groupId> <artifactId>velocity-engine-core</artifactId> + <version>2.1</version> </dependency> @@ -136,26 +173,27 @@ <dependency> <groupId>org.hibernate</groupId> <artifactId>hibernate-validator</artifactId> - <version>6.0.10.Final</version> + <version>6.1.0.Final</version> </dependency> <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-swagger2</artifactId> - <version>2.9.2</version> + <version>${swagger.version}</version> <scope>compile</scope> </dependency> <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-swagger-ui</artifactId> - <version>2.9.2</version> + <version>${swagger.version}</version> <scope>compile</scope> </dependency> <dependency> <groupId>org.mongodb</groupId> <artifactId>mongo-java-driver</artifactId> + <version>${mongojava.version}</version> </dependency> <dependency> <groupId>com.couchbase.mock</groupId> @@ -165,4 +203,24 @@ </dependency> </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.spotify</groupId> + <artifactId>dockerfile-maven-plugin</artifactId> + <version>${dockerfile-maven.version}</version> + <configuration> + <!-- repository>docker.io/moguobiao/datalake-feeder-maven</repository--> + <repository>${onap.nexus.dockerregistry.daily}/${docker.image.path}</repository> + <tag>${project.version}</tag> + <dockerfile>Dockerfile</dockerfile> + <!-- useMavenSettingsForAuth>true</useMavenSettingsForAuth--> + <buildArgs> + <JAR_FILE>${project.build.finalName}.jar</JAR_FILE> + </buildArgs> + </configuration> + </plugin> + </plugins> + </build> </project> diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java index 41995e04..9865e0af 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/KafkaController.java @@ -121,14 +121,14 @@ public class KafkaController { Kafka oldKafka = kafkaService.getKafkaById(id); if (oldKafka == null) { - sendError(response, 400, "kafka not found "+id); + sendError(response, 400, "kafka not found, ID: "+id); } else { kafkaRepository.delete(oldKafka); response.setStatus(204); } } - @GetMapping("") + /*@GetMapping("") @ResponseBody @ApiOperation(value="List all Kafka id") public List<Integer> list() { @@ -139,12 +139,30 @@ public class KafkaController { retString.add(k.getId()); } return retString; - } + }*/ + @GetMapping("") + @ResponseBody + @ApiOperation(value="List all Kafkas") public List<KafkaConfig> queryAllKafka(){ return kafkaService.getAllKafka(); } + @GetMapping("/{id}") + @ResponseBody + @ApiOperation(value="Get detail of kafka by id") + public KafkaConfig getKafkaDetail(@PathVariable int id, HttpServletResponse response) throws IOException { + log.info("Get detail of kafka, ID: " + id); + Kafka oldKafka = kafkaService.getKafkaById(id); + if (oldKafka == null) { + sendError(response, 400, "kafka not found, ID: "+id); + return null; + } else { + log.info("ResponseBody......" + oldKafka.getKafkaConfig()); + return oldKafka.getKafkaConfig(); + } + } + private PostReturnBody<KafkaConfig> mkPostReturnBody(int statusCode, Kafka kafka) { PostReturnBody<KafkaConfig> retBody = new PostReturnBody<>(); retBody.setStatusCode(statusCode); diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java index a02cd6a2..3bdbcdba 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicConfigPollingService.java @@ -23,6 +23,7 @@ package org.onap.datalake.feeder.service; import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -60,6 +61,9 @@ public class TopicConfigPollingService implements Runnable { @Autowired private KafkaRepository kafkaRepository; + @Autowired + private TopicNameService topicNameService; + //effectiveTopic Map, 1st key is kafkaId, 2nd is topic name, the value is a list of EffectiveTopic. private Map<Integer, Map<String, List<EffectiveTopic>>> effectiveTopicMap = new HashMap<>(); //private Map<String, TopicConfig> effectiveTopicConfigMap; @@ -114,7 +118,7 @@ public class TopicConfigPollingService implements Runnable { log.info("TopicConfigPollingService started."); while (active) { - try { //sleep first since we already pool in init() + try { //sleep first since we already called poll() in init() Thread.sleep(config.getCheckTopicInterval()); if(!active) { break; @@ -138,6 +142,7 @@ public class TopicConfigPollingService implements Runnable { log.info("activeTopics list is updated, new={}", newTopics); activeTopicMap.put(kafkaId, newTopics); + //update version currentActiveTopicsVersionMap.put(kafkaId, currentActiveTopicsVersionMap.getOrDefault(kafkaId, 1)+1); } else { log.debug("activeTopics list is not updated."); @@ -156,14 +161,20 @@ public class TopicConfigPollingService implements Runnable { } private Map<Integer, Set<String>> poll() throws IOException { + Set<String> allTopicNames = new HashSet<>(); + Map<Integer, Set<String>> ret = new HashMap<>(); Iterable<Kafka> kafkas = kafkaRepository.findAll(); for (Kafka kafka : kafkas) { if (kafka.isEnabled()) { Set<String> topics = poll(kafka); ret.put(kafka.getId(), topics); + allTopicNames.addAll(topics); } } + + topicNameService.update(allTopicNames); + return ret; } diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java new file mode 100644 index 00000000..021d2c94 --- /dev/null +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicNameService.java @@ -0,0 +1,59 @@ +/* +* ============LICENSE_START======================================================= +* ONAP : DATALAKE +* ================================================================================ +* Copyright 2019 China Mobile +*================================================================================= +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* ============LICENSE_END========================================================= +*/ + +package org.onap.datalake.feeder.service; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.commons.collections.CollectionUtils; +import org.onap.datalake.feeder.domain.TopicName; +import org.onap.datalake.feeder.repository.TopicNameRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * Service for TopicName + * + * @author Guobiao Mo + * + */ +@Service +public class TopicNameService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private TopicNameRepository topicNameRepository; + + public void update(Collection<String> allTopicNames) { + + List<TopicName> all = allTopicNames.stream().map(s-> new TopicName(s)).collect(Collectors.toList()); + List<TopicName> allInDb = (List<TopicName>) topicNameRepository.findAll(); + + Collection<TopicName> additions = CollectionUtils.subtract(all, allInDb); + + if(!additions.isEmpty()) + topicNameRepository.saveAll(additions); + + } +} diff --git a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java index 2f0761ab..b6466a85 100644 --- a/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java +++ b/components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/TopicService.java @@ -101,6 +101,8 @@ public class TopicService { return ret; } + // for unique topic string, one can create multiple 'topic' in admin UI. + // for example, one 'topic' setting correlates events, and sends data to ES, another 'topic' sends data to HDFS without such setting //TODO use query public List<Topic> findTopics(Kafka kafka, String topicStr) { List<Topic> ret = new ArrayList<>(); |