From 93fa06429f8d8e3d5bce25ae4b338156f428888a Mon Sep 17 00:00:00 2001 From: mpriyank Date: Fri, 23 Jun 2023 15:01:38 +0100 Subject: KafkaListener config for CloudEvents - Introduced kafkalistener factory configs to correctly use the configs and convert the incoming events to CloudEvents - Also legacy events to use a separate listener factory - Increased the wait time from 100ms to 300ms to listen to the message in the test Issue-ID: CPS-1764 Change-Id: I8b9fe0ba82ef87d52b7731941ccd3af8ae980109 Signed-off-by: mpriyank --- .../ncmp/api/impl/config/kafka/KafkaConfig.java | 156 +++++++++++++++++++++ .../api/impl/config/kafka/KafkaTemplateConfig.java | 127 ----------------- .../ncmp/api/impl/events/avc/AvcEventConsumer.java | 3 +- ...ataOperationEventConsumerIntegrationSpec.groovy | 4 +- .../api/impl/config/kafka/KafkaConfigSpec.groovy | 62 ++++++++ .../config/kafka/KafkaTemplateConfigSpec.groovy | 62 -------- 6 files changed, 222 insertions(+), 192 deletions(-) create mode 100644 cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java delete mode 100644 cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfig.java create mode 100644 cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy delete mode 100644 cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfigSpec.groovy (limited to 'cps-ncmp-service') diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java new file mode 100644 index 000000000..514967574 --- /dev/null +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java @@ -0,0 +1,156 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.api.impl.config.kafka; + +import io.cloudevents.CloudEvent; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.support.serializer.JsonDeserializer; +import org.springframework.kafka.support.serializer.JsonSerializer; + +/** + * kafka Configuration for legacy and cloud events. + * + * @param valid legacy event to be published over the wire. + */ +@Configuration +@EnableKafka +@RequiredArgsConstructor +public class KafkaConfig { + + private final KafkaProperties kafkaProperties; + + /** + * This sets the strategy for creating legacy Kafka producer instance from kafka properties defined into + * application.yml and replaces value-serializer by JsonSerializer. + * + * @return legacy event producer instance. + */ + @Bean + public ProducerFactory legacyEventProducerFactory() { + final Map producerConfigProperties = kafkaProperties.buildProducerProperties(); + producerConfigProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); + return new DefaultKafkaProducerFactory<>(producerConfigProperties); + } + + /** + * The ConsumerFactory implementation is to produce new legacy instance for provided kafka properties defined + * into application.yml and replaces deserializer-value by JsonDeserializer. + * + * @return an instance of legacy consumer factory. + */ + @Bean + public ConsumerFactory legacyEventConsumerFactory() { + final Map consumerConfigProperties = kafkaProperties.buildConsumerProperties(); + consumerConfigProperties.put("spring.deserializer.value.delegate.class", JsonDeserializer.class); + return new DefaultKafkaConsumerFactory<>(consumerConfigProperties); + } + + /** + * A legacy Kafka event template for executing high-level operations. The legacy producer factory ensure this. + * + * @return an instance of legacy Kafka template. + */ + @Bean + @Primary + public KafkaTemplate legacyEventKafkaTemplate() { + final KafkaTemplate kafkaTemplate = new KafkaTemplate<>(legacyEventProducerFactory()); + kafkaTemplate.setConsumerFactory(legacyEventConsumerFactory()); + return kafkaTemplate; + } + + /** + * A legacy concurrent kafka listener container factory. + * + * @return instance of Concurrent kafka listener factory + */ + @Bean + public ConcurrentKafkaListenerContainerFactory legacyEventConcurrentKafkaListenerContainerFactory() { + final ConcurrentKafkaListenerContainerFactory containerFactory = + new ConcurrentKafkaListenerContainerFactory<>(); + containerFactory.setConsumerFactory(legacyEventConsumerFactory()); + return containerFactory; + } + + /** + * This sets the strategy for creating cloud Kafka producer instance from kafka properties defined into + * application.yml with CloudEventSerializer. + * + * @return cloud event producer instance. + */ + @Bean + public ProducerFactory cloudEventProducerFactory() { + final Map producerConfigProperties = kafkaProperties.buildProducerProperties(); + return new DefaultKafkaProducerFactory<>(producerConfigProperties); + } + + /** + * The ConsumerFactory implementation to produce new legacy instance for provided kafka properties defined + * into application.yml having CloudEventDeserializer as deserializer-value. + * + * @return an instance of cloud consumer factory. + */ + @Bean + public ConsumerFactory cloudEventConsumerFactory() { + final Map consumerConfigProperties = kafkaProperties.buildConsumerProperties(); + return new DefaultKafkaConsumerFactory<>(consumerConfigProperties); + } + + + /** + * A cloud Kafka event template for executing high-level operations. The cloud producer factory ensure this. + * + * @return an instance of cloud Kafka template. + */ + @Bean + public KafkaTemplate cloudEventKafkaTemplate() { + final KafkaTemplate kafkaTemplate = new KafkaTemplate<>(cloudEventProducerFactory()); + kafkaTemplate.setConsumerFactory(cloudEventConsumerFactory()); + return kafkaTemplate; + } + + /** + * A Concurrent CloudEvent kafka listener container factory. + * + * @return instance of Concurrent kafka listener factory + */ + @Bean + public ConcurrentKafkaListenerContainerFactory + cloudEventConcurrentKafkaListenerContainerFactory() { + final ConcurrentKafkaListenerContainerFactory containerFactory = + new ConcurrentKafkaListenerContainerFactory<>(); + containerFactory.setConsumerFactory(cloudEventConsumerFactory()); + return containerFactory; + } + +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfig.java deleted file mode 100644 index b76f86ebe..000000000 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfig.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ncmp.api.impl.config.kafka; - -import io.cloudevents.CloudEvent; -import java.util.Map; -import lombok.RequiredArgsConstructor; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.kafka.annotation.EnableKafka; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.core.ProducerFactory; -import org.springframework.kafka.support.serializer.JsonDeserializer; -import org.springframework.kafka.support.serializer.JsonSerializer; - -/** - * kafka Configuration for legacy and cloud events. - * - * @param valid legacy event to be published over the wire. - */ -@Configuration -@EnableKafka -@RequiredArgsConstructor -public class KafkaTemplateConfig { - - private final KafkaProperties kafkaProperties; - - /** - * This sets the strategy for creating legacy Kafka producer instance from kafka properties defined into - * application.yml and replaces value-serializer by JsonSerializer. - * - * @return legacy event producer instance. - */ - @Bean - public ProducerFactory legacyEventProducerFactory() { - final Map producerConfigProperties = kafkaProperties.buildProducerProperties(); - producerConfigProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); - return new DefaultKafkaProducerFactory<>(producerConfigProperties); - } - - /** - * The ConsumerFactory implementation is to produce new legacy instance for provided kafka properties defined - * into application.yml and replaces deserializer-value by JsonDeserializer. - * - * @return an instance of legacy consumer factory. - */ - @Bean - public ConsumerFactory legacyEventConsumerFactory() { - final Map consumerConfigProperties = kafkaProperties.buildConsumerProperties(); - consumerConfigProperties.put("spring.deserializer.value.delegate.class", JsonDeserializer.class); - return new DefaultKafkaConsumerFactory<>(consumerConfigProperties); - } - - /** - * This sets the strategy for creating cloud Kafka producer instance from kafka properties defined into - * application.yml with CloudEventSerializer. - * - * @return cloud event producer instance. - */ - @Bean - public ProducerFactory cloudEventProducerFactory() { - final Map producerConfigProperties = kafkaProperties.buildProducerProperties(); - return new DefaultKafkaProducerFactory<>(producerConfigProperties); - } - - /** - * The ConsumerFactory implementation to produce new legacy instance for provided kafka properties defined - * into application.yml having CloudEventDeserializer as deserializer-value. - * - * @return an instance of cloud consumer factory. - */ - @Bean - public ConsumerFactory cloudEventConsumerFactory() { - final Map consumerConfigProperties = kafkaProperties.buildConsumerProperties(); - return new DefaultKafkaConsumerFactory<>(consumerConfigProperties); - } - - /** - * A legacy Kafka event template for executing high-level operations. The legacy producer factory ensure this. - * - * @return an instance of legacy Kafka template. - */ - @Bean - @Primary - public KafkaTemplate legacyEventKafkaTemplate() { - final KafkaTemplate kafkaTemplate = new KafkaTemplate<>(legacyEventProducerFactory()); - kafkaTemplate.setConsumerFactory(legacyEventConsumerFactory()); - return kafkaTemplate; - } - - /** - * A cloud Kafka event template for executing high-level operations. The cloud producer factory ensure this. - * - * @return an instance of cloud Kafka template. - */ - @Bean - public KafkaTemplate cloudEventKafkaTemplate() { - final KafkaTemplate kafkaTemplate = new KafkaTemplate<>(cloudEventProducerFactory()); - kafkaTemplate.setConsumerFactory(cloudEventConsumerFactory()); - return kafkaTemplate; - } - -} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/events/avc/AvcEventConsumer.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/events/avc/AvcEventConsumer.java index b5ca176d1..88ebd35c8 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/events/avc/AvcEventConsumer.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/events/avc/AvcEventConsumer.java @@ -52,7 +52,8 @@ public class AvcEventConsumer { * * @param avcEventConsumerRecord Incoming raw consumer record */ - @KafkaListener(topics = "${app.dmi.cm-events.topic}") + @KafkaListener(topics = "${app.dmi.cm-events.topic}", + containerFactory = "cloudEventConcurrentKafkaListenerContainerFactory") public void consumeAndForward(final ConsumerRecord avcEventConsumerRecord) { log.debug("Consuming AVC event {} ...", avcEventConsumerRecord.value()); final String newEventId = UUID.randomUUID().toString(); diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/NcmpAsyncDataOperationEventConsumerIntegrationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/NcmpAsyncDataOperationEventConsumerIntegrationSpec.groovy index c0bdf3d1d..f577f55ba 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/NcmpAsyncDataOperationEventConsumerIntegrationSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/async/NcmpAsyncDataOperationEventConsumerIntegrationSpec.groovy @@ -69,7 +69,7 @@ class NcmpAsyncDataOperationEventConsumerIntegrationSpec extends MessagingBaseSp KafkaProducer producer = new KafkaProducer<>(eventProducerConfigProperties(CloudEventSerializer)) producer.send(record) and: 'wait a little for async processing of message' - TimeUnit.MILLISECONDS.sleep(100) + TimeUnit.MILLISECONDS.sleep(300) then: 'the event has only been forwarded for the correct type' expectedNUmberOfCallsToPublishForwardedEvent * mockEventsPublisher.publishCloudEvent(*_) where: 'the following event types are used' @@ -85,7 +85,7 @@ class NcmpAsyncDataOperationEventConsumerIntegrationSpec extends MessagingBaseSp KafkaProducer producer = new KafkaProducer<>(eventProducerConfigProperties(StringSerializer)) producer.send(record) and: 'wait a little for async processing of message' - TimeUnit.MILLISECONDS.sleep(100) + TimeUnit.MILLISECONDS.sleep(300) then: 'the event is not processed by this consumer' 0 * mockEventsPublisher.publishCloudEvent(*_) } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy new file mode 100644 index 000000000..d5b091552 --- /dev/null +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy @@ -0,0 +1,62 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.api.impl.config.kafka; + +import io.cloudevents.CloudEvent +import io.cloudevents.kafka.CloudEventDeserializer +import io.cloudevents.kafka.CloudEventSerializer +import org.spockframework.spring.EnableSharedInjection +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.kafka.KafkaProperties +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.kafka.core.KafkaTemplate +import org.springframework.kafka.support.serializer.JsonDeserializer +import org.springframework.kafka.support.serializer.JsonSerializer +import spock.lang.Shared +import spock.lang.Specification + +@SpringBootTest(classes = [KafkaProperties, KafkaConfig]) +@EnableSharedInjection +@EnableConfigurationProperties +class KafkaConfigSpec extends Specification { + + @Shared + @Autowired + KafkaTemplate legacyEventKafkaTemplate + + @Shared + @Autowired + KafkaTemplate cloudEventKafkaTemplate + + def 'Verify kafka template serializer and deserializer configuration for #eventType.'() { + expect: 'kafka template is instantiated' + assert kafkaTemplateInstance.properties['beanName'] == beanName + and: 'verify event key and value serializer' + assert kafkaTemplateInstance.properties['producerFactory'].configs['value.serializer'].asType(String.class).contains(valueSerializer.getCanonicalName()) + and: 'verify event key and value deserializer' + assert kafkaTemplateInstance.properties['consumerFactory'].configs['spring.deserializer.value.delegate.class'].asType(String.class).contains(delegateDeserializer.getCanonicalName()) + where: 'the following event type is used' + eventType | kafkaTemplateInstance || beanName | valueSerializer | delegateDeserializer + 'legacy event' | legacyEventKafkaTemplate || 'legacyEventKafkaTemplate' | JsonSerializer | JsonDeserializer + 'cloud event' | cloudEventKafkaTemplate || 'cloudEventKafkaTemplate' | CloudEventSerializer | CloudEventDeserializer + } +} diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfigSpec.groovy deleted file mode 100644 index ed5f16125..000000000 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaTemplateConfigSpec.groovy +++ /dev/null @@ -1,62 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.ncmp.api.impl.config.kafka; - -import io.cloudevents.CloudEvent -import io.cloudevents.kafka.CloudEventDeserializer -import io.cloudevents.kafka.CloudEventSerializer -import org.spockframework.spring.EnableSharedInjection -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.autoconfigure.kafka.KafkaProperties -import org.springframework.boot.context.properties.EnableConfigurationProperties -import org.springframework.boot.test.context.SpringBootTest -import org.springframework.kafka.core.KafkaTemplate -import org.springframework.kafka.support.serializer.JsonDeserializer -import org.springframework.kafka.support.serializer.JsonSerializer -import spock.lang.Shared -import spock.lang.Specification - -@SpringBootTest(classes = [KafkaProperties, KafkaTemplateConfig]) -@EnableSharedInjection -@EnableConfigurationProperties -class KafkaTemplateConfigSpec extends Specification { - - @Shared - @Autowired - KafkaTemplate legacyEventKafkaTemplate - - @Shared - @Autowired - KafkaTemplate cloudEventKafkaTemplate - - def 'Verify kafka template serializer and deserializer configuration for #eventType.'() { - expect: 'kafka template is instantiated' - assert kafkaTemplateInstance.properties['beanName'] == beanName - and: 'verify event key and value serializer' - assert kafkaTemplateInstance.properties['producerFactory'].configs['value.serializer'].asType(String.class).contains(valueSerializer.getCanonicalName()) - and: 'verify event key and value deserializer' - assert kafkaTemplateInstance.properties['consumerFactory'].configs['spring.deserializer.value.delegate.class'].asType(String.class).contains(delegateDeserializer.getCanonicalName()) - where: 'the following event type is used' - eventType | kafkaTemplateInstance || beanName | valueSerializer | delegateDeserializer - 'legacy event' | legacyEventKafkaTemplate || 'legacyEventKafkaTemplate' | JsonSerializer | JsonDeserializer - 'cloud event' | cloudEventKafkaTemplate || 'cloudEventKafkaTemplate' | CloudEventSerializer | CloudEventDeserializer - } -} -- cgit 1.2.3-korg