diff options
author | 2018-04-26 09:17:09 +0200 | |
---|---|---|
committer | 2018-07-26 09:18:00 +0200 | |
commit | e98fdcc3087d06b76066ae2d2c7d0bde41d7776b (patch) | |
tree | 3f6b79be2422022233b7e2f6c51064a63cba5fe1 /hv-collector-core/src/test | |
parent | dcbb6333fede6c0cf43ac8690119911b01864d8d (diff) |
HV VES Collector seed code
Contains squashed commits up to 11fe6b63 (2018-05-30). The whole
contains a basic project structure. We are trying to put rest of the
commits one by one so we do not loose the history.
Bellow there are messages of the single commits in this squashed bulk:
Basic project setup
Create base maven project with Gitlab CI configuration.
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Merging guildeline
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Add remote branch delete command
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Sample runtime in Kotlin - PoC
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Setup project internal architecture
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Message routing
Determine target topic and partition by VES Common Header.
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Parse GPB message header
fkrzywka <filip.krzywka@nokia.com>
Set listen port based on command line args
Use Apache Commons CLI to parse cmd line args.
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Drop invalid GPB messages
Instead of propagating error and closing stream just drop the message
and proceed. Final handling logic may include closing the connection or
sending some message depending on the specification.
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Add Apache license file
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Convert to maven multi-module project
fkrzywka <filip.krzywka@nokia.com>
Component tests with current GPB schema
* Using v5 draft protobuf definition
* Code reorganized to so component boundaries are more visible
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Thin logging facade over slf4j
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Introduce code analysis tools
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Implemented reading configuration from consul
Ves Common Header validation added (required parameters existance check)
Micro benchmark for direct vs on-heap NIO buffers
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Decode wire protocol and fix (most?) memory leaks
Proposed wire protocol is just a suggestion and will (should) change
in the future.
Netty's ByteBuf is a reference-counted wrapper over a memory chunk. It
is crucial to free unused buffers by means of release() method.
The general rule regarding memory management was suggested. Let's put
all memory-cleanup logic in main VesHvCollector class so other classes
could focus on their job.
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Minor cleanup
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Add license info in files
Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Change-Id: Ic484aa107eba48ad48f8ab222799e1795dffa865
Issue-ID: DCAEGEN2-601
Signed-off-by: Piotr Jaszczyk <piotr.jaszczyk@nokia.com>
Diffstat (limited to 'hv-collector-core/src/test')
7 files changed, 528 insertions, 0 deletions
diff --git a/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/MessageValidatorTest.kt b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/MessageValidatorTest.kt new file mode 100644 index 00000000..8f6b76fb --- /dev/null +++ b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/MessageValidatorTest.kt @@ -0,0 +1,108 @@ +/* + * ============LICENSE_START======================================================= + * dcaegen2-collectors-veshv + * ================================================================================ + * Copyright (C) 2018 NOKIA Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.dcae.collectors.veshv.impl + +import com.google.protobuf.ByteString +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import io.netty.buffer.Unpooled.wrappedBuffer +import org.jetbrains.spek.api.Spek +import org.jetbrains.spek.api.dsl.given +import org.jetbrains.spek.api.dsl.it +import org.jetbrains.spek.api.dsl.on +import org.onap.dcae.collectors.veshv.domain.VesMessage +import org.onap.ves.VesEventV5.VesEvent.CommonEventHeader +import org.onap.ves.VesEventV5.VesEvent +import org.assertj.core.api.Assertions.assertThat +import org.onap.ves.VesEventV5.VesEvent.CommonEventHeader.* + +internal object MessageValidatorTest : Spek({ + + fun vesMessageBytes(commonHeader: CommonEventHeader): ByteBuf { + val msg = VesEvent.newBuilder() + .setCommonEventHeader(commonHeader) + .setHvRanMeasFields(ByteString.copyFromUtf8("high volume data")) + .build() + return wrappedBuffer(msg.toByteArray()) + } + + given("Message validator") { + val cut = MessageValidator() + + on("ves hv message including header with fully initialized fields") { + val commonHeader = newBuilder() + .setVersion("1.9") + .setEventName("Sample event name") + .setDomain(Domain.HVRANMEAS) + .setEventId("Sample event Id") + .setSourceName("Sample Source") + .setReportingEntityName(ByteString.copyFromUtf8("Sample byte String")) + .setPriority(Priority.MEDIUM) + .setStartEpochMicrosec(120034455) + .setLastEpochMicrosec(120034459) + .setSequence(2) + .build() + + it("should accept message with fully initialized message header") { + val vesMessage = VesMessage(commonHeader, vesMessageBytes(commonHeader)) + assertThat(cut.isValid(vesMessage)).describedAs("message validation result").isTrue() + } + + it("should reject message with domain other than HVRANMEAS") { + Domain.values() + .filter { it != Domain.HVRANMEAS && it != Domain.UNRECOGNIZED } + .forEach { domain -> + val header = newBuilder(commonHeader).setDomain(domain).build() + val vesMessage = VesMessage(header, vesMessageBytes(header)) + assertThat(cut.isValid(vesMessage)) + .describedAs("message with $domain domain") + .isFalse() + } + } + } + + on("ves hv message bytes") { + val vesMessage = VesMessage(getDefaultInstance(), Unpooled.EMPTY_BUFFER) + it("should not accept message with default header") { + assertThat(cut.isValid(vesMessage)).describedAs("message validation result").isFalse() + } + } + + + on("ves hv message including header with not initialized fields") { + val commonHeader = newBuilder() + .setVersion("1.9") + .setEventName("Sample event name") + .setEventId("Sample event Id") + .setSourceName("Sample Source") + .build() + val msg = VesEvent.newBuilder() + .setCommonEventHeader(commonHeader) + .setHvRanMeasFields(ByteString.copyFromUtf8("high volume data !!!")) + .build() + val rawMessageBytes = wrappedBuffer(msg.toByteArray()) + + it("should not accept not fully initialized message header ") { + val vesMessage = VesMessage(commonHeader, rawMessageBytes) + assertThat(cut.isValid(vesMessage)).describedAs("message validation result").isFalse() + } + } + } +})
\ No newline at end of file diff --git a/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/RouterTest.kt b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/RouterTest.kt new file mode 100644 index 00000000..ac91aaeb --- /dev/null +++ b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/RouterTest.kt @@ -0,0 +1,92 @@ +package org.onap.dcae.collectors.veshv.impl + +import io.netty.buffer.Unpooled +import org.assertj.core.api.Assertions.assertThat +import org.jetbrains.spek.api.Spek +import org.jetbrains.spek.api.dsl.given +import org.jetbrains.spek.api.dsl.it +import org.jetbrains.spek.api.dsl.on +import org.onap.dcae.collectors.veshv.domain.VesMessage +import org.onap.dcae.collectors.veshv.domain.routing +import org.onap.ves.VesEventV5.VesEvent.CommonEventHeader +import org.onap.ves.VesEventV5.VesEvent.CommonEventHeader.Domain + +/** + * @author Piotr Jaszczyk <piotr.jaszczyk@nokia.com> + * @since May 2018 + */ +object RouterTest : Spek({ + given("sample configuration") { + val config = routing { + + defineRoute { + fromDomain(Domain.HVRANMEAS) + toTopic("ves_rtpm") + withFixedPartitioning(2) + } + + defineRoute { + fromDomain(Domain.SYSLOG) + toTopic("ves_trace") + withFixedPartitioning() + } + }.build() + val cut = Router(config) + + on("message with existing route (rtpm)") { + val message = VesMessage(vesCommonHeaderWithDomain(Domain.HVRANMEAS), Unpooled.EMPTY_BUFFER) + val result = cut.findDestination(message) + + it("should have route available") { + assertThat(result).isNotNull() + } + + it("should be routed to proper partition") { + assertThat(result?.partition).isEqualTo(2) + } + + it("should be routed to proper topic") { + assertThat(result?.topic).isEqualTo("ves_rtpm") + } + + it("should be routed with a given message") { + assertThat(result?.message).isSameAs(message) + } + } + + on("message with existing route (trace)") { + val message = VesMessage(vesCommonHeaderWithDomain(Domain.SYSLOG), Unpooled.EMPTY_BUFFER) + val result = cut.findDestination(message) + + it("should have route available") { + assertThat(result).isNotNull() + } + + it("should be routed to proper partition") { + assertThat(result?.partition).isEqualTo(1) + } + + it("should be routed to proper topic") { + assertThat(result?.topic).isEqualTo("ves_trace") + } + + it("should be routed with a given message") { + assertThat(result?.message).isSameAs(message) + } + } + + on("message with unknown route") { + val message = VesMessage(vesCommonHeaderWithDomain(Domain.HEARTBEAT), Unpooled.EMPTY_BUFFER) + val result = cut.findDestination(message) + + it("should not have route available") { + assertThat(result).isNull() + } + } + } +}) + +private fun vesCommonHeaderWithDomain(domain: Domain) = + CommonEventHeader.getDefaultInstance().toBuilder() + .setDomain(domain) + .build()
\ No newline at end of file diff --git a/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/VesDecoderTest.kt b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/VesDecoderTest.kt new file mode 100644 index 00000000..4d9c9239 --- /dev/null +++ b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/VesDecoderTest.kt @@ -0,0 +1,69 @@ +/* + * ============LICENSE_START======================================================= + * dcaegen2-collectors-veshv + * ================================================================================ + * Copyright (C) 2018 NOKIA Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.dcae.collectors.veshv.impl + +import com.google.protobuf.ByteString +import io.netty.buffer.Unpooled.wrappedBuffer +import org.assertj.core.api.Assertions.assertThat +import org.jetbrains.spek.api.Spek +import org.jetbrains.spek.api.dsl.given +import org.jetbrains.spek.api.dsl.it +import org.jetbrains.spek.api.dsl.on +import org.onap.dcae.collectors.veshv.domain.VesMessage +import org.onap.ves.VesEventV5.VesEvent +import org.onap.ves.VesEventV5.VesEvent.CommonEventHeader +import reactor.test.StepVerifier +import java.nio.charset.Charset + + +internal object VesDecoderTest : Spek({ + + given("ves message decoder") { + val cut = VesDecoder() + + on("ves hv message bytes") { + val commonHeader = CommonEventHeader.getDefaultInstance() + val msg = VesEvent.newBuilder() + .setCommonEventHeader(commonHeader) + .setHvRanMeasFields(ByteString.copyFromUtf8("highvolume measurements")) + .build() + val rawMessageBytes = wrappedBuffer(msg.toByteArray()) + + + it("should decode only header and pass it on along with raw message") { + val expectedMessage = VesMessage( + commonHeader, + rawMessageBytes + ) + + assertThat(cut.decode(rawMessageBytes)).isEqualTo(expectedMessage) + + } + } + + on("invalid ves hv message bytes") { + val rawMessageBytes = wrappedBuffer("ala ma kota".toByteArray(Charset.defaultCharset())) + + it("should return empty result") { + assertThat(cut.decode(rawMessageBytes)).isNull() + } + } + } +}) diff --git a/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/WireDecoderTest.kt b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/WireDecoderTest.kt new file mode 100644 index 00000000..3563bf6d --- /dev/null +++ b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/WireDecoderTest.kt @@ -0,0 +1,104 @@ +package org.onap.dcae.collectors.veshv.impl + +import io.netty.buffer.Unpooled +import io.netty.buffer.UnpooledByteBufAllocator +import org.assertj.core.api.Assertions.assertThat +import org.jetbrains.spek.api.Spek +import org.jetbrains.spek.api.dsl.describe +import org.jetbrains.spek.api.dsl.given +import org.jetbrains.spek.api.dsl.it +import org.onap.dcae.collectors.veshv.domain.WireFrame + +/** + * @author Piotr Jaszczyk <piotr.jaszczyk></piotr.jaszczyk>@nokia.com> + * @since May 2018 + */ +internal object WireDecoderTest : Spek({ + describe("decoding wire protocol") { + val cut = WireDecoder() + + fun decode(frame: WireFrame) = + cut.decode( + frame.encode(UnpooledByteBufAllocator.DEFAULT)) + + given("empty input") { + val input = Unpooled.EMPTY_BUFFER + + it("should yield empty result") { + assertThat(cut.decode(input)).isNull() + } + } + + given("input without 0xFF first byte") { + val input = WireFrame( + payload = Unpooled.EMPTY_BUFFER, + mark = 0x10, + majorVersion = 1, + minorVersion = 2, + payloadSize = 0) + + it("should yield empty result") { + assertThat(decode(input)).isNull() + } + } + + given("input with unsupported major version") { + val input = WireFrame( + payload = Unpooled.EMPTY_BUFFER, + mark = 0xFF, + majorVersion = 100, + minorVersion = 2, + payloadSize = 0) + + it("should yield empty result") { + assertThat(decode(input)).isNull() + } + } + + given("input with too small payload size") { + val input = WireFrame( + payload = Unpooled.wrappedBuffer(byteArrayOf(1, 2 ,3)), + mark = 0xFF, + majorVersion = 1, + minorVersion = 0, + payloadSize = 1) + + it("should yield empty result") { + assertThat(decode(input)).isNull() + } + } + + given("input with too big payload size") { + val input = WireFrame( + payload = Unpooled.wrappedBuffer(byteArrayOf(1, 2 ,3)), + mark = 0xFF, + majorVersion = 1, + minorVersion = 0, + payloadSize = 8) + + it("should yield empty result") { + assertThat(decode(input)).isNull() + } + } + + given("valid input") { + val payload = byteArrayOf(6, 9, 8, 6) + val input = WireFrame( + payload = Unpooled.wrappedBuffer(payload), + mark = 0xFF, + majorVersion = 1, + minorVersion = 0, + payloadSize = payload.size) + + + it("should yield Google Protocol Buffers payload") { + val result = decode(input)!! + + val actualPayload = ByteArray(result.readableBytes()) + result.readBytes(actualPayload) + + assertThat(actualPayload).containsExactly(*payload) + } + } + } +}) diff --git a/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/adapters/ConsulConfigurationProviderTest.kt b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/adapters/ConsulConfigurationProviderTest.kt new file mode 100644 index 00000000..bf65c859 --- /dev/null +++ b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/adapters/ConsulConfigurationProviderTest.kt @@ -0,0 +1,60 @@ +package org.onap.dcae.collectors.veshv.impl.adapters + +import com.nhaarman.mockito_kotlin.mock +import com.nhaarman.mockito_kotlin.whenever +import org.jetbrains.spek.api.Spek +import org.jetbrains.spek.api.dsl.given +import org.jetbrains.spek.api.dsl.it +import org.onap.ves.VesEventV5.VesEvent.CommonEventHeader.Domain +import reactor.core.publisher.Mono +import java.util.* +import kotlin.test.assertEquals + +/** + * @author Jakub Dudycz <jakub.dudycz@nokia.com> + * @since May 2018 + */ +internal object ConsulConfigurationProviderTest : Spek({ + + given("valid resource url") { + val testUrl = "http://valid-url/" + val httpAdapterMock: HttpAdapter = mock() + val consulConfigProvider = ConsulConfigurationProvider(testUrl, httpAdapterMock) + + whenever(httpAdapterMock.getResponse(testUrl)).thenReturn(Mono.just(constructConsulResponse())) + + + it("should create valid collector configuration") { + val response = consulConfigProvider().blockFirst() + assertEquals("val1", response.kafkaBootstrapServers) + val route = response.routing.routes[0] + assertEquals(Domain.MEASUREMENTS_FOR_VF_SCALING, route.domain) + assertEquals("val3", route.targetTopic) + } + } +}) + +fun constructConsulResponse(): String { + + val config = """{ + "kafkaBootstrapServers": "val1", + "routing": { + "fromDomain": 2, + "toTopic": "val3" + } + }""" + + val encodedValue = String(Base64.getEncoder().encode(config.toByteArray())) + + return """[ + { + "CreateIndex": 100, + "ModifyIndex": 200, + "LockIndex": 200, + "Key": "zip", + "Flags": 0, + "Value": "$encodedValue", + "Session": "adf4238a-882b-9ddc-4a9d-5b6758e4159e" + } + ]""" +} diff --git a/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/adapters/HttpAdapterTest.kt b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/adapters/HttpAdapterTest.kt new file mode 100644 index 00000000..10e8b6bf --- /dev/null +++ b/hv-collector-core/src/test/kotlin/org/onap/dcae/collectors/veshv/impl/adapters/HttpAdapterTest.kt @@ -0,0 +1,60 @@ +package org.onap.dcae.collectors.veshv.impl.adapters + +import com.nhaarman.mockito_kotlin.mock +import com.nhaarman.mockito_kotlin.whenever +import io.netty.buffer.Unpooled +import io.netty.handler.codec.http.HttpContent +import org.jetbrains.spek.api.Spek +import org.jetbrains.spek.api.dsl.given +import org.jetbrains.spek.api.dsl.it +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import reactor.ipc.netty.http.client.HttpClient +import reactor.ipc.netty.http.client.HttpClientResponse +import java.nio.charset.Charset +import kotlin.test.assertEquals + +/** + * @author Jakub Dudycz <jakub.dudycz@nokia.com> + * @since May 2018 + */ +internal object HttpAdapterTest : Spek({ + + given("valid resource url") { + + val httpClientMock: HttpClient = mock() + val httpAdapter = HttpAdapter(httpClientMock) + val testUrl = "http://valid-url/" + val responseContent = """{"key1": "value1", "key2": "value2"}""" + val httpResponse = createHttpResponseMock(responseContent) + whenever(httpClientMock.get(testUrl)).thenReturn(Mono.just(httpResponse)) + + it("should return response string") { + assertEquals(responseContent, httpAdapter.getResponse(testUrl).block()) + } + } + + given("invalid resource url") { + + val httpClientMock: HttpClient = mock() + val httpAdapter = HttpAdapter(httpClientMock) + val testUrl = "http://invalid-url/" + whenever(httpClientMock.get(testUrl)).thenReturn(Mono.error(Exception("Test exception"))) + + + it("should return null response") { + assertEquals(null, httpAdapter.getResponse(testUrl).block()) + } + } +}) + +fun createHttpResponseMock(content: String): HttpClientResponse { + val responseMock: HttpClientResponse = mock() + val contentMock: HttpContent = mock() + val contentByteBuff = Unpooled.copiedBuffer(content, Charset.defaultCharset()) + + whenever(responseMock.receiveContent()).thenReturn(Flux.just(contentMock)) + whenever(contentMock.content()).thenReturn(contentByteBuff) + + return responseMock +} diff --git a/hv-collector-core/src/test/resources/logback.xml b/hv-collector-core/src/test/resources/logback.xml new file mode 100644 index 00000000..809f62d4 --- /dev/null +++ b/hv-collector-core/src/test/resources/logback.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + <property name="LOG_FILE" + value="${LOG_FILE:-${LOG_PATH:-${LOG_TEMP:-${java.io.tmpdir:-/tmp}}/}ves-hv.log}"/> + <property name="FILE_LOG_PATTERN" value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX,UTC} %-5level [%-40.40logger{10}] - %msg%n"/> + + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern> + %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX,UTC} %highlight(%-5level) [%-40.40logger{10}] - %msg%n + </pattern> + </encoder> + </appender> + + <appender name="ROLLING-FILE" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <encoder> + <pattern>${FILE_LOG_PATTERN}</pattern> + </encoder> + <file>${LOG_FILE}</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${LOG_FILE}.%d{yyyy-MM-dd}.log</fileNamePattern> + <maxFileSize>50MB</maxFileSize> + <maxHistory>30</maxHistory> + <totalSizeCap>10GB</totalSizeCap> + </rollingPolicy> + </appender> + + <logger name="org.onap.dcae.collectors.veshv" level="DEBUG"/> + + <root level="INFO"> + <appender-ref ref="CONSOLE"/> + <appender-ref ref="ROLLING-FILE"/> + </root> +</configuration>
\ No newline at end of file |