diff options
3 files changed, 330 insertions, 179 deletions
diff --git a/components/datalake-handler/feeder/pom.xml b/components/datalake-handler/feeder/pom.xml index 9c1bb785..2af2af86 100644 --- a/components/datalake-handler/feeder/pom.xml +++ b/components/datalake-handler/feeder/pom.xml @@ -1,186 +1,197 @@ <?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.onap.dcaegen2.services.components</groupId> - <artifactId>datalake-handler</artifactId> - <version>1.0.0-SNAPSHOT</version> - </parent> - - <groupId>org.onap.dcaegen2.services.components.datalake-handler</groupId> - <artifactId>feeder</artifactId> - <packaging>jar</packaging> - <name>DataLake Feeder</name> - - - <dependencies> - - <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - </dependency> - - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-web</artifactId> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-actuator</artifactId> - </dependency> - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.onap.dcaegen2.services.components</groupId> + <artifactId>datalake-handler</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + + <groupId>org.onap.dcaegen2.services.components.datalake-handler</groupId> + <artifactId>feeder</artifactId> + <packaging>jar</packaging> + <name>DataLake Feeder</name> + + + <dependencies> + + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + </dependency> + + <dependency> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-actuator</artifactId> + </dependency> + <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-data-couchbase</artifactId> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-test</artifactId> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-configuration-processor</artifactId> - </dependency> - - <dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>elasticsearch-rest-high-level-client</artifactId> - </dependency> - - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-yaml</artifactId> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-xml</artifactId> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </dependency> - - <dependency> - <groupId>org.projectlombok</groupId> - <artifactId>lombok</artifactId> - </dependency> - - <dependency> - <groupId>io.druid</groupId> - <artifactId>tranquility-core_2.11</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.velocity</groupId> - <artifactId>velocity-engine-core</artifactId> - </dependency> - - - <dependency> - <groupId>org.hibernate</groupId> - <artifactId>hibernate-core</artifactId> - <version>5.3.7.Final</version> - </dependency> - - <!-- jsr303 validation --> - <dependency> - <groupId>javax.validation</groupId> - <artifactId>validation-api</artifactId> - <version>2.0.1.Final</version> - </dependency> - - <dependency> - <groupId>org.hibernate</groupId> - <artifactId>hibernate-validator</artifactId> - <version>6.0.10.Final</version> - </dependency> - - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-swagger2</artifactId> - <version>2.9.2</version> - <scope>compile</scope> - </dependency> - - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-swagger-ui</artifactId> - <version>2.9.2</version> - <scope>compile</scope> - </dependency> - - <dependency> - <groupId>org.mongodb</groupId> - <artifactId>mongo-java-driver</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-maven-plugin</artifactId> - <version>${springboot.version}</version> - <executions> - <execution> - <goals> - <goal>repackage</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> - <artifactId>maven-failsafe-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>integration-test</goal> - <goal>verify</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-couchbase</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-configuration-processor</artifactId> + </dependency> + + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-high-level-client</artifactId> + </dependency> + + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-yaml</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-xml</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + </dependency> + + <dependency> + <groupId>io.druid</groupId> + <artifactId>tranquility-core_2.11</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity-engine-core</artifactId> + </dependency> + + + <dependency> + <groupId>org.hibernate</groupId> + <artifactId>hibernate-core</artifactId> + <version>5.3.7.Final</version> + </dependency> + + <!-- jsr303 validation --> + <dependency> + <groupId>javax.validation</groupId> + <artifactId>validation-api</artifactId> + <version>2.0.1.Final</version> + </dependency> + + <dependency> + <groupId>org.hibernate</groupId> + <artifactId>hibernate-validator</artifactId> + <version>6.0.10.Final</version> + </dependency> + + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-swagger2</artifactId> + <version>2.9.2</version> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-swagger-ui</artifactId> + <version>2.9.2</version> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.mongodb</groupId> + <artifactId>mongo-java-driver</artifactId> + </dependency> + <dependency> + <groupId>com.couchbase.mock</groupId> + <artifactId>CouchbaseMock</artifactId> + <version>1.5.22</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.couchbase.client</groupId> + <artifactId>core-io</artifactId> + <version>1.7.4</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <version>${springboot.version}</version> + <executions> + <execution> + <goals> + <goal>repackage</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> </project> diff --git a/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java index b94211e9..afaede76 100644 --- a/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java +++ b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/ApplicationTest.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * org.onap.holmes.common.aai + * ONAP : DATALAKE * ================================================================================ * Copyright (C) 2018-2019 Huawei. All rights reserved. * ================================================================================ diff --git a/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/service/CouchbaseServiceTest.java b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/service/CouchbaseServiceTest.java new file mode 100755 index 00000000..9e40a2b1 --- /dev/null +++ b/components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/service/CouchbaseServiceTest.java @@ -0,0 +1,140 @@ +/*- + * ============LICENSE_START======================================================= + * ONAP : DATALAKE + * ================================================================================ + * Copyright (C) 2018-2019 Huawei. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.datalake.feeder.service; + +import com.couchbase.client.java.Cluster; +import com.couchbase.client.java.CouchbaseCluster; +import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; +import com.couchbase.mock.Bucket; +import com.couchbase.mock.BucketConfiguration; +import com.couchbase.mock.CouchbaseMock; +import com.couchbase.mock.client.MockClient; +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.onap.datalake.feeder.domain.Topic; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class CouchbaseServiceTest { + protected final BucketConfiguration bucketConfiguration = new BucketConfiguration(); + protected MockClient mockClient; + protected CouchbaseMock couchbaseMock; + protected Cluster cluster; + protected com.couchbase.client.java.Bucket bucket; + protected int carrierPort; + protected int httpPort; + + protected void getPortInfo(String bucket) throws Exception { + httpPort = couchbaseMock.getHttpPort(); + carrierPort = couchbaseMock.getCarrierPort(bucket); + } + + protected void createMock(@NotNull String name, @NotNull String password) throws Exception { + bucketConfiguration.numNodes = 1; + bucketConfiguration.numReplicas = 1; + bucketConfiguration.numVBuckets = 1024; + bucketConfiguration.name = name; + bucketConfiguration.type = Bucket.BucketType.COUCHBASE; + bucketConfiguration.password = password; + ArrayList<BucketConfiguration> configList = new ArrayList<BucketConfiguration>(); + configList.add(bucketConfiguration); + couchbaseMock = new CouchbaseMock(0, configList); + couchbaseMock.start(); + couchbaseMock.waitForStartup(); + } + + protected void createClient() { + cluster = CouchbaseCluster.create(DefaultCouchbaseEnvironment.builder() + .bootstrapCarrierDirectPort(carrierPort) + .bootstrapHttpDirectPort(httpPort) + .build(), "couchbase://127.0.0.1"); + bucket = cluster.openBucket("default"); + } + + @Before + public void setUp() throws Exception { + createMock("default", ""); + getPortInfo("default"); + createClient(); + } + + @After + public void tearDown() { + if (cluster != null) { + cluster.disconnect(); + } + if (couchbaseMock != null) { + couchbaseMock.stop(); + } + if (mockClient != null) { + mockClient.shutdown(); + } + } + + @Test + public void testSaveJsonsWithTopicId() { + + String text = "{ data: { data2 : { value : 'hello'}}}"; + + JSONObject json = new JSONObject(text); + + Topic topic = new Topic("test getMessageId"); + topic.setMessageIdPath("/data/data2/value"); + List<JSONObject> jsons = new ArrayList<>(); + json.put("_ts", 1234); + jsons.add(json); + CouchbaseService couchbaseService = new CouchbaseService(); + couchbaseService.bucket = bucket; + couchbaseService.saveJsons(topic, jsons); + + } + + @Test + public void testSaveJsonsWithOutTopicId() { + + String text = "{ data: { data2 : { value : 'hello'}}}"; + + JSONObject json = new JSONObject(text); + + Topic topic = new Topic("test getMessageId"); + List<JSONObject> jsons = new ArrayList<>(); + json.put("_ts", 1234); + jsons.add(json); + CouchbaseService couchbaseService = new CouchbaseService(); + couchbaseService.bucket = bucket; + couchbaseService.saveJsons(topic, jsons); + } + + @Test + public void testCleanupBucket() { + CouchbaseService couchbaseService = new CouchbaseService(); + couchbaseService.bucket = bucket; + couchbaseService.cleanUp(); + } + +}
\ No newline at end of file |