From c489a2eb22484e798c39a978bc8b61821b92322f Mon Sep 17 00:00:00 2001 From: an4828 Date: Mon, 22 Jan 2018 17:17:34 -0500 Subject: TCA: Replace any openecomp reference by onap Change-Id: I7c6d812ab5c1d7b30c63653d1974b0b1abc099be Signed-off-by: an4828 Issue-ID: DCAEGEN2-224 Signed-off-by: an4828 --- dcae-analytics-cdap-plugins/pom.xml | 632 ++++++++++----------- .../batch/sink/dmaap/DMaaPMROutputFormat.java | 94 +++ .../sink/dmaap/DMaaPMROutputFormatProvider.java | 116 ++++ .../batch/sink/dmaap/DMaaPMRRecordWriter.java | 58 ++ .../cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java | 90 +++ .../cdap/plugins/common/PluginSchema.java | 37 ++ .../config/dmaap/BaseDMaaPMRPluginConfig.java | 159 ++++++ .../config/dmaap/DMaaPMRSinkPluginConfig.java | 101 ++++ .../config/dmaap/DMaaPMRSourcePluginConfig.java | 134 +++++ .../config/filter/JsonPathFilterPluginConfig.java | 125 ++++ .../domain/config/tca/SimpleTCAPluginConfig.java | 154 +++++ .../schema/dmaap/DMaaPSourceOutputSchema.java | 59 ++ .../plugins/sparkcompute/tca/SimpleTCAPlugin.java | 175 ++++++ .../plugins/streaming/dmaap/DMaaPMRReceiver.java | 118 ++++ .../plugins/streaming/dmaap/DMaaPMRSource.java | 70 +++ .../streaming/dmaap/MockDMaaPMRReceiver.java | 132 +++++ .../plugins/streaming/dmaap/MockDMaaPMRSource.java | 73 +++ .../plugins/transform/filter/JsonPathFilter.java | 134 +++++ .../cdap/plugins/utils/CDAPPluginUtils.java | 295 ++++++++++ .../cdap/plugins/utils/DMaaPSinkConfigMapper.java | 112 ++++ .../plugins/utils/DMaaPSourceConfigMapper.java | 118 ++++ .../BaseDMaaPMRPluginConfigValidator.java | 72 +++ .../DMaaPMRSinkPluginConfigValidator.java | 58 ++ .../DMaaPMRSourcePluginConfigValidator.java | 58 ++ .../JsonPathFilterPluginConfigValidator.java | 83 +++ .../validator/SimpleTCAPluginConfigValidator.java | 91 +++ .../batch/sink/dmaap/DMaaPMROutputFormat.java | 94 --- .../sink/dmaap/DMaaPMROutputFormatProvider.java | 116 ---- .../batch/sink/dmaap/DMaaPMRRecordWriter.java | 58 -- .../cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java | 90 --- .../cdap/plugins/common/PluginSchema.java | 37 -- .../config/dmaap/BaseDMaaPMRPluginConfig.java | 159 ------ .../config/dmaap/DMaaPMRSinkPluginConfig.java | 101 ---- .../config/dmaap/DMaaPMRSourcePluginConfig.java | 134 ----- .../config/filter/JsonPathFilterPluginConfig.java | 125 ---- .../domain/config/tca/SimpleTCAPluginConfig.java | 154 ----- .../schema/dmaap/DMaaPSourceOutputSchema.java | 59 -- .../plugins/sparkcompute/tca/SimpleTCAPlugin.java | 175 ------ .../plugins/streaming/dmaap/DMaaPMRReceiver.java | 118 ---- .../plugins/streaming/dmaap/DMaaPMRSource.java | 70 --- .../streaming/dmaap/MockDMaaPMRReceiver.java | 132 ----- .../plugins/streaming/dmaap/MockDMaaPMRSource.java | 73 --- .../plugins/transform/filter/JsonPathFilter.java | 134 ----- .../cdap/plugins/utils/CDAPPluginUtils.java | 295 ---------- .../cdap/plugins/utils/DMaaPSinkConfigMapper.java | 112 ---- .../plugins/utils/DMaaPSourceConfigMapper.java | 118 ---- .../BaseDMaaPMRPluginConfigValidator.java | 72 --- .../DMaaPMRSinkPluginConfigValidator.java | 58 -- .../DMaaPMRSourcePluginConfigValidator.java | 58 -- .../JsonPathFilterPluginConfigValidator.java | 83 --- .../validator/SimpleTCAPluginConfigValidator.java | 91 --- .../plugins/BaseAnalyticsCDAPPluginsUnitTest.java | 238 ++++++++ .../dmaap/DMaaPMROutputFormatProviderTest.java | 77 +++ .../batch/sink/dmaap/DMaaPMROutputFormatTest.java | 75 +++ .../batch/sink/dmaap/DMaaPMRRecordWriterTest.java | 62 ++ .../plugins/batch/sink/dmaap/DMaaPMRSinkTest.java | 95 ++++ .../config/dmaap/DMaaPMRSinkPluginConfigTest.java | 80 +++ .../dmaap/DMaaPMRSourcePluginConfigTest.java | 84 +++ .../config/dmaap/TestDMaaPMRSinkPluginConfig.java | 76 +++ .../dmaap/TestDMaaPMRSourcePluginConfig.java | 84 +++ .../filter/TestJsonPathFilterPluginConfig.java | 50 ++ .../config/tca/TestSimpleTCAPluginConfig.java | 56 ++ .../schema/dmaap/DMaaPSourceOutputSchemaTest.java | 63 ++ .../cdap/plugins/it/SimpleTCAPluginCDAPIT.java | 229 ++++++++ .../sparkcompute/tca/SimpleTCAPluginTest.java | 119 ++++ .../streaming/dmaap/DMaaPMRReceiverTest.java | 75 +++ .../plugins/streaming/dmaap/DMaaPMRSourceTest.java | 91 +++ .../streaming/dmaap/MockDMaaPMRReceiverTest.java | 81 +++ .../streaming/dmaap/MockDMaaPMRSourceTest.java | 74 +++ .../streaming/dmaap/TestDMaaPMRReceiver.java | 58 ++ .../transform/filter/JsonPathFilterTest.java | 84 +++ .../cdap/plugins/utils/CDAPPluginUtilsTest.java | 171 ++++++ .../plugins/utils/DMaaPSinkConfigMapperTest.java | 57 ++ .../plugins/utils/DMaaPSourceConfigMapperTest.java | 64 +++ .../DMaaPMRSinkPluginConfigValidatorTest.java | 86 +++ .../DMaaPMRSourcePluginConfigValidatorTest.java | 85 +++ .../JsonPathFilterPluginConfigValidatorTest.java | 107 ++++ .../SimpleTCAPluginConfigValidatorTest.java | 157 +++++ .../plugins/BaseAnalyticsCDAPPluginsUnitTest.java | 238 -------- .../dmaap/DMaaPMROutputFormatProviderTest.java | 77 --- .../batch/sink/dmaap/DMaaPMROutputFormatTest.java | 75 --- .../batch/sink/dmaap/DMaaPMRRecordWriterTest.java | 62 -- .../plugins/batch/sink/dmaap/DMaaPMRSinkTest.java | 95 ---- .../config/dmaap/DMaaPMRSinkPluginConfigTest.java | 80 --- .../dmaap/DMaaPMRSourcePluginConfigTest.java | 84 --- .../config/dmaap/TestDMaaPMRSinkPluginConfig.java | 76 --- .../dmaap/TestDMaaPMRSourcePluginConfig.java | 84 --- .../filter/TestJsonPathFilterPluginConfig.java | 50 -- .../config/tca/TestSimpleTCAPluginConfig.java | 56 -- .../schema/dmaap/DMaaPSourceOutputSchemaTest.java | 63 -- .../cdap/plugins/it/SimpleTCAPluginCDAPIT.java | 229 -------- .../sparkcompute/tca/SimpleTCAPluginTest.java | 119 ---- .../streaming/dmaap/DMaaPMRReceiverTest.java | 75 --- .../plugins/streaming/dmaap/DMaaPMRSourceTest.java | 91 --- .../streaming/dmaap/MockDMaaPMRReceiverTest.java | 81 --- .../streaming/dmaap/MockDMaaPMRSourceTest.java | 74 --- .../streaming/dmaap/TestDMaaPMRReceiver.java | 58 -- .../transform/filter/JsonPathFilterTest.java | 84 --- .../cdap/plugins/utils/CDAPPluginUtilsTest.java | 171 ------ .../plugins/utils/DMaaPSinkConfigMapperTest.java | 57 -- .../plugins/utils/DMaaPSourceConfigMapperTest.java | 64 --- .../DMaaPMRSinkPluginConfigValidatorTest.java | 86 --- .../DMaaPMRSourcePluginConfigValidatorTest.java | 85 --- .../JsonPathFilterPluginConfigValidatorTest.java | 107 ---- .../SimpleTCAPluginConfigValidatorTest.java | 157 ----- .../src/test/resources/logback-test.xml | 110 ++-- 106 files changed, 5665 insertions(+), 5665 deletions(-) create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java create mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java delete mode 100644 dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java create mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java delete mode 100644 dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java (limited to 'dcae-analytics-cdap-plugins') diff --git a/dcae-analytics-cdap-plugins/pom.xml b/dcae-analytics-cdap-plugins/pom.xml index b936d8c..76b119b 100644 --- a/dcae-analytics-cdap-plugins/pom.xml +++ b/dcae-analytics-cdap-plugins/pom.xml @@ -1,316 +1,316 @@ - - - - - - 4.0.0 - - - dcae-analytics - org.onap.dcaegen2.analytics.tca - 2.1.0-SNAPSHOT - - - dcae-analytics-cdap-plugins - jar - - - DCAE Analytics CDAP Plugins - DCAE Analytics CDAP Plugins to build CDAP Pipelines - - - ${project.parent.basedir} - ${project.basedir} - widgets - docs - - system:cdap-etl-batch[4.0.0,4.2.0-SNAPSHOT),system:cdap-etl-realtime[4.0.0,4.2.0-SNAPSHOT),system:cdap-data-pipeline[4.0.0,4.2.0-SNAPSHOT),system:cdap-data-streams[4.0.0,4.2.0-SNAPSHOT) - - - - - - - - org.onap.dcaegen2.analytics.tca - dcae-analytics-cdap-common - 2.1.0-SNAPSHOT - - - - org.onap.dcaegen2.analytics.tca - dcae-analytics-tca - 2.1.0-SNAPSHOT - - - - - - co.cask.cdap - cdap-api - - - - co.cask.cdap - cdap-etl-api - - - - co.cask.cdap - cdap-etl-api-spark - - - - co.cask.cdap - cdap-etl-realtime - - - - co.cask.cdap - cdap-data-pipeline - - - - co.cask.cdap - cdap-data-streams - - - - org.apache.spark - spark-streaming_2.10 - - - - - co.cask.http - netty-http - - - co.cask.common - common-http - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - - - org.apache.hadoop - hadoop-common - - - - - org.apache.spark - spark-core_2.10 - - - - - org.slf4j - slf4j-api - - - - ch.qos.logback - logback-core - - - - ch.qos.logback - logback-classic - - - - - com.google.code.findbugs - jsr305 - ${findbugs.jsr305.version} - provided - - - - com.google.code.findbugs - annotations - ${findbugs.annotations.version} - provided - - - - - org.onap.dcaegen2.analytics.tca - dcae-analytics-test - 2.1.0-SNAPSHOT - test - - - - co.cask.cdap - hydrator-test - - - - co.cask.cdap - cdap-unit-test - - - org.apache.httpcomponents - httpcore - - - - - - - - - - - - org.apache.felix - maven-bundle-plugin - - - - <_exportcontents> - org.openecomp.dcae.apod.analytics.dmaap.domain.response.*, - org.openecomp.dcae.apod.analytics.dmaap.domain.config.*; - org.openecomp.dcae.apod.analytics.dmaap.service.subscriber.*; - org.openecomp.dcae.apod.analytics.dmaap.service.*; - org.openecomp.dcae.apod.analytics.cdap.common.settings.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.common.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.sparkcompute.tca.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap.*; - org.openecomp.dcae.apod.analytics.cdap.plugins.transform.filter.*; - com.google.common.base.*; - com.google.inject.*; - org.apache.http.*; - org.aopalliance.*; - javax.inject.*; - org.apache.commons.* - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - - create-artifact-config - prepare-package - - - - - - - run - - - - - - - - - - + + + + + + 4.0.0 + + + dcae-analytics + org.onap.dcaegen2.analytics.tca + 2.1.0-SNAPSHOT + + + dcae-analytics-cdap-plugins + jar + + + DCAE Analytics CDAP Plugins + DCAE Analytics CDAP Plugins to build CDAP Pipelines + + + ${project.parent.basedir} + ${project.basedir} + widgets + docs + + system:cdap-etl-batch[4.0.0,4.2.0-SNAPSHOT),system:cdap-etl-realtime[4.0.0,4.2.0-SNAPSHOT),system:cdap-data-pipeline[4.0.0,4.2.0-SNAPSHOT),system:cdap-data-streams[4.0.0,4.2.0-SNAPSHOT) + + + + + + + + org.onap.dcaegen2.analytics.tca + dcae-analytics-cdap-common + 2.1.0-SNAPSHOT + + + + org.onap.dcaegen2.analytics.tca + dcae-analytics-tca + 2.1.0-SNAPSHOT + + + + + + co.cask.cdap + cdap-api + + + + co.cask.cdap + cdap-etl-api + + + + co.cask.cdap + cdap-etl-api-spark + + + + co.cask.cdap + cdap-etl-realtime + + + + co.cask.cdap + cdap-data-pipeline + + + + co.cask.cdap + cdap-data-streams + + + + org.apache.spark + spark-streaming_2.10 + + + + + co.cask.http + netty-http + + + co.cask.common + common-http + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-common + + + + + org.apache.spark + spark-core_2.10 + + + + + org.slf4j + slf4j-api + + + + ch.qos.logback + logback-core + + + + ch.qos.logback + logback-classic + + + + + com.google.code.findbugs + jsr305 + ${findbugs.jsr305.version} + provided + + + + com.google.code.findbugs + annotations + ${findbugs.annotations.version} + provided + + + + + org.onap.dcaegen2.analytics.tca + dcae-analytics-test + 2.1.0-SNAPSHOT + test + + + + co.cask.cdap + hydrator-test + + + + co.cask.cdap + cdap-unit-test + + + org.apache.httpcomponents + httpcore + + + + + + + + + + + + org.apache.felix + maven-bundle-plugin + + + + <_exportcontents> + org.onap.dcae.apod.analytics.dmaap.domain.response.*, + org.onap.dcae.apod.analytics.dmaap.domain.config.*; + org.onap.dcae.apod.analytics.dmaap.service.subscriber.*; + org.onap.dcae.apod.analytics.dmaap.service.*; + org.onap.dcae.apod.analytics.cdap.common.settings.*; + org.onap.dcae.apod.analytics.cdap.plugins.common.*; + org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap.*; + org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.*; + org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter.*; + org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.*; + org.onap.dcae.apod.analytics.cdap.plugins.sparkcompute.tca.*; + org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap.*; + org.onap.dcae.apod.analytics.cdap.plugins.transform.filter.*; + com.google.common.base.*; + com.google.inject.*; + org.apache.http.*; + org.aopalliance.*; + javax.inject.*; + org.apache.commons.* + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + + create-artifact-config + prepare-package + + + + + + + run + + + + + + + + + + diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java new file mode 100644 index 0000000..d627fb0 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java @@ -0,0 +1,94 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.DMaaPSinkConfigMapper; +import org.onap.dcae.apod.analytics.dmaap.DMaaPMRFactory; +import org.onap.dcae.apod.analytics.dmaap.domain.config.DMaaPMRPublisherConfig; +import org.onap.dcae.apod.analytics.dmaap.service.publisher.DMaaPMRPublisher; + +import java.io.IOException; + +/** + * DMaaP MR Output format used by DMaaP MR Sink Plugin to create a MR Publisher and pass to custom {@link + * DMaaPMRRecordWriter} + *

+ * @author Rajiv Singla . Creation Date: 1/27/2017. + */ +public class DMaaPMROutputFormat extends OutputFormat { + + @Override + public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, + InterruptedException { + final Configuration configuration = context.getConfiguration(); + final DMaaPMRPublisherConfig publisherConfig = DMaaPSinkConfigMapper.map(configuration); + final DMaaPMRPublisher publisher = DMaaPMRFactory.create().createPublisher(publisherConfig); + return new DMaaPMRRecordWriter(publisher); + } + + @Override + public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { + // do nothing + } + + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException { + return new NoOpOutputCommitter(); + } + + /** + * A dummy implementation for {@link OutputCommitter} that does nothing. + */ + protected static class NoOpOutputCommitter extends OutputCommitter { + + @Override + public void setupJob(JobContext jobContext) throws IOException { + // no op + } + + @Override + public void setupTask(TaskAttemptContext taskContext) throws IOException { + // no op + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { + return false; + } + + @Override + public void commitTask(TaskAttemptContext taskContext) throws IOException { + // no op + } + + @Override + public void abortTask(TaskAttemptContext taskContext) throws IOException { + // no op + } + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java new file mode 100644 index 0000000..bc6780b --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java @@ -0,0 +1,116 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import co.cask.cdap.api.data.batch.OutputFormatProvider; +import org.onap.dcae.apod.analytics.cdap.common.CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields; +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; +import org.onap.dcae.apod.analytics.common.AnalyticsConstants; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * DMaaP MR Output Format Provider used to create Batch Sink Plugin + *

+ * @author Rajiv Singla . Creation Date: 1/27/2017. + */ +public class DMaaPMROutputFormatProvider implements OutputFormatProvider { + + private final Map sinkConfig; + + + public DMaaPMROutputFormatProvider(DMaaPMRSinkPluginConfig sinkPluginConfig) { + + // initialize Sink Config - with DMaaP MR Publisher config values + sinkConfig = new LinkedHashMap<>(); + + // Required fields for sink config + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.HOST_NAME, sinkPluginConfig.getHostName()); + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.TOPIC_NAME, sinkPluginConfig.getTopicName()); + + final Integer configPortNumber = sinkPluginConfig.getPortNumber(); + if (configPortNumber != null) { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER, configPortNumber.toString()); + } else { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER, + AnalyticsConstants.DEFAULT_PORT_NUMBER.toString()); + } + + final String configProtocol = sinkPluginConfig.getProtocol(); + if (ValidationUtils.isPresent(configProtocol)) { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PROTOCOL, configProtocol); + } else { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PROTOCOL, AnalyticsConstants.DEFAULT_PROTOCOL); + } + + + final String configUserName = sinkPluginConfig.getUserName(); + if (ValidationUtils.isPresent(configUserName)) { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_NAME, configUserName); + } else { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_NAME, AnalyticsConstants.DEFAULT_USER_NAME); + } + + final String configUserPass = sinkPluginConfig.getUserPassword(); + if (ValidationUtils.isPresent(configUserPass)) { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_PASS, configUserPass); + } else { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_PASS, AnalyticsConstants.DEFAULT_USER_PASSWORD); + } + + final String configContentType = sinkPluginConfig.getContentType(); + if (ValidationUtils.isPresent(configContentType)) { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE, configContentType); + } else { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE, AnalyticsConstants.DEFAULT_CONTENT_TYPE); + } + + + final Integer configMaxBatchSize = sinkPluginConfig.getMaxBatchSize(); + if (configMaxBatchSize != null) { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE, configMaxBatchSize.toString()); + } else { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE, + String.valueOf(AnalyticsConstants.DEFAULT_PUBLISHER_MAX_BATCH_SIZE)); + } + + final Integer configMaxRecoveryQueueSize = sinkPluginConfig.getMaxRecoveryQueueSize(); + if (configMaxRecoveryQueueSize != null) { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE, configMaxRecoveryQueueSize.toString()); + } else { + sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE, + String.valueOf(AnalyticsConstants.DEFAULT_PUBLISHER_MAX_RECOVERY_QUEUE_SIZE)); + } + + } + + @Override + public String getOutputFormatClassName() { + return DMaaPMROutputFormat.class.getName(); + } + + @Override + public Map getOutputFormatConfiguration() { + return sinkConfig; + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java new file mode 100644 index 0000000..a123edd --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java @@ -0,0 +1,58 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.onap.dcae.apod.analytics.dmaap.service.publisher.DMaaPMRPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; + +/** + * A simple implementation of {@link RecordWriter} which writes messages to DMaaP MR topic + *

+ * @author Rajiv Singla . Creation Date: 1/27/2017. + */ +public class DMaaPMRRecordWriter extends RecordWriter { + + private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRRecordWriter.class); + + private final DMaaPMRPublisher dMaaPMRPublisher; + + public DMaaPMRRecordWriter(DMaaPMRPublisher dMaaPMRPublisher) { + this.dMaaPMRPublisher = dMaaPMRPublisher; + } + + @Override + public void write(String message, NullWritable value) throws IOException, InterruptedException { + LOG.debug("Writing message to DMaaP MR Topic: {}", message); + dMaaPMRPublisher.publish(Arrays.asList(message)); + } + + @Override + public void close(TaskAttemptContext context) throws IOException, InterruptedException { + dMaaPMRPublisher.flush(); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java new file mode 100644 index 0000000..4003e4a --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java @@ -0,0 +1,90 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Name; +import co.cask.cdap.api.annotation.Plugin; +import co.cask.cdap.api.data.batch.Output; +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.api.dataset.lib.KeyValue; +import co.cask.cdap.etl.api.Emitter; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.batch.BatchSink; +import co.cask.cdap.etl.api.batch.BatchSinkContext; +import org.apache.hadoop.io.NullWritable; +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.validator.DMaaPMRSinkPluginConfigValidator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Rajiv Singla . Creation Date: 1/26/2017. + */ +@Plugin(type = BatchSink.PLUGIN_TYPE) +@Name("DMaaPMRSink") +@Description("A batch sink Plugin that publishes messages to DMaaP MR Topic.") +public class DMaaPMRSink extends BatchSink { + + private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRSink.class); + + private final DMaaPMRSinkPluginConfig pluginConfig; + + public DMaaPMRSink(final DMaaPMRSinkPluginConfig pluginConfig) { + LOG.debug("Creating DMaaP MR Sink Plugin with plugin Config: {}", pluginConfig); + this.pluginConfig = pluginConfig; + } + + @Override + public void configurePipeline(final PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + ValidationUtils.validateSettings(pluginConfig, new DMaaPMRSinkPluginConfigValidator()); + // validates that input schema contains the field provided in Sink Message Column Name property + final Schema inputSchema = pipelineConfigurer.getStageConfigurer().getInputSchema(); + CDAPPluginUtils.validateSchemaContainsFields(inputSchema, pluginConfig.getMessageColumnName()); + } + + + @Override + public void prepareRun(BatchSinkContext context) throws Exception { + context.addOutput(Output.of(pluginConfig.getReferenceName(), new DMaaPMROutputFormatProvider(pluginConfig))); + } + + @Override + public void transform(StructuredRecord structuredRecord, + Emitter> emitter) throws Exception { + // get incoming message from structured record + final String incomingMessage = structuredRecord.get(pluginConfig.getMessageColumnName()); + + // if incoming messages does not have message column name log warning as it should not happen + if (incomingMessage == null) { + LOG.warn("Column Name: {}, contains no message.Skipped for DMaaP MR Publishing....", + pluginConfig.getMessageColumnName()); + } else { + + // emit the messages as key + emitter.emit(new KeyValue(incomingMessage, null)); + } + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java new file mode 100644 index 0000000..076478c --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java @@ -0,0 +1,37 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.common; + +/** + * Contract interface for all DCAE Analytics Plugin Schemas + * + * @author Rajiv Singla . Creation Date: 1/25/2017. + */ +public interface PluginSchema { + + /** + * Provides column name that will be used in Schema Definition + * + * @return Column name that will be used in Schema Definition + */ + String getSchemaColumnName(); + +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java new file mode 100644 index 0000000..276c9a1 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java @@ -0,0 +1,159 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Macro; +import com.google.common.base.Objects; +import org.onap.dcae.apod.analytics.cdap.common.settings.CDAPBasePluginConfig; + +import javax.annotation.Nullable; + +/** + * Base class for all DMaaP MR Configs + *

+ * @author Rajiv Singla . Creation Date: 1/17/2017. + */ +public abstract class BaseDMaaPMRPluginConfig extends CDAPBasePluginConfig { + + @Description("DMaaP Message Router HostName") + @Macro + protected String hostName; + + @Description("DMaaP Message Router Host Port number. Defaults to Port 80") + @Nullable + @Macro + protected Integer portNumber; + + @Description("DMaaP Message Router Topic Name") + @Macro + protected String topicName; + + @Description("DMaaP Message Router HTTP Protocol e.g. HTTP or HTTPS. Defaults to HTTPS") + @Nullable + @Macro + protected String protocol; + + @Description("DMaaP Message Router User Name used for AAF Authentication. Defaults to no authentication") + @Nullable + @Macro + protected String userName; + + @Description("DMaaP Message Router User Password used for AAF Authentication. Defaults to no authentication") + @Nullable + @Macro + protected String userPassword; + + @Description("DMaaP Message Router Content Type. Defaults to 'application/json'") + @Nullable + @Macro + protected String contentType; + + + public BaseDMaaPMRPluginConfig(final String referenceName, final String hostName, final String topicName) { + this.referenceName = referenceName; + this.hostName = hostName; + this.topicName = topicName; + } + + /** + * Host Name for DMaaP MR Publisher or Subscriber + * + * @return host name + */ + public String getHostName() { + return hostName; + } + + /** + * Port Number for DMaaP MR Publisher or Subscriber + * + * @return port number + */ + @Nullable + public Integer getPortNumber() { + return portNumber; + } + + /** + * DMaaP MR Topic Name for Subscriber or Publisher + * + * @return topic name + */ + public String getTopicName() { + return topicName; + } + + + /** + * DMaaP MR HTTP or HTTPS protocol + * + * @return http or https protocol + */ + @Nullable + public String getProtocol() { + return protocol; + } + + /** + * User name used for DMaaP MR AAF Authentication + * + * @return User name for DMaaP MR AAF Authentication + */ + @Nullable + public String getUserName() { + return userName; + } + + /** + * User password used for DMaaP MR AAF Authentication + * + * @return User password used for DMaaP MR AAF Authentication + */ + @Nullable + public String getUserPassword() { + return userPassword; + } + + /** + * Content type used for DMaaP MR Topic e.g. 'application/json' + * + * @return content type for DMaaP MR Topic + */ + @Nullable + public String getContentType() { + return contentType; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("referenceName", referenceName) + .add("hostName", hostName) + .add("portNumber", portNumber) + .add("topicName", topicName) + .add("protocol", protocol) + .add("userName", userName) + .add("userPassword", "xxxx") + .add("contentType", contentType) + .toString(); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java new file mode 100644 index 0000000..5b03644 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java @@ -0,0 +1,101 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Macro; +import com.google.common.base.Objects; + +import javax.annotation.Nullable; + +/** + * DMaaP MR Publisher Config + *

+ * @author Rajiv Singla . Creation Date: 1/17/2017. + */ +public class DMaaPMRSinkPluginConfig extends BaseDMaaPMRPluginConfig { + + private static final long serialVersionUID = 1L; + + @Description("Column name of input schema which contains the message that needs to be written to DMaaP MR Topic") + @Macro + protected String messageColumnName; + + @Description("DMaaP MR Publisher Max Batch Size. Defaults to no Batch") + @Nullable + @Macro + protected Integer maxBatchSize; + + @Description("DMaaP MR Publisher Recovery Queue Size. Default to 1000K messages which can be buffered in memory " + + "in case DMaaP MR Publisher is temporarily unavailable") + @Nullable + @Macro + protected Integer maxRecoveryQueueSize; + + // Required No Arg constructor + public DMaaPMRSinkPluginConfig() { + this(null, null, null, null); + } + + public DMaaPMRSinkPluginConfig(String referenceName, String hostName, String topicName, String messageColumnName) { + super(referenceName, hostName, topicName); + this.messageColumnName = messageColumnName; + } + + /** + * Column name of incoming Schema field that contains the message that needs to published to DMaaP MR Topic + * + * @return Column name of incoming schema which contains message that needs to published to DMaaP MR Topic + */ + public String getMessageColumnName() { + return messageColumnName; + } + + /** + * DMaaP MR Publisher Max Batch Size. + * + * @return DMaaP MR Publisher Max Batch Size + */ + @Nullable + public Integer getMaxBatchSize() { + return maxBatchSize; + } + + /** + * DMaaP MR Publisher Max Recovery Queue Size + * + * @return DMaaP MR Publisher Max Recovery Queue Size + */ + @Nullable + public Integer getMaxRecoveryQueueSize() { + return maxRecoveryQueueSize; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("super", super.toString()) + .add("messageColumnName", messageColumnName) + .add("maxBatchSize", maxBatchSize) + .add("maxRecoveryQueueSize", maxRecoveryQueueSize) + .toString(); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java new file mode 100644 index 0000000..b1aa3dc --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java @@ -0,0 +1,134 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Macro; +import com.google.common.base.Objects; + +import javax.annotation.Nullable; + +/** + * DMaaP MR Subscriber Config + *

+ * @author Rajiv Singla . Creation Date: 1/17/2017. + */ +public class DMaaPMRSourcePluginConfig extends BaseDMaaPMRPluginConfig { + + private static final long serialVersionUID = 1L; + + @Description("DMaaP MR Polling Interval in MS") + @Macro + protected Integer pollingInterval; + + @Description("DMaaP Message Router Subscriber Consumer ID. Defaults to some randomly created userID") + @Nullable + @Macro + protected String consumerId; + + @Description("DMaaP Message Router Subscriber Consumer Group. Defaults to some randomly created user Group") + @Nullable + @Macro + protected String consumerGroup; + + @Description("DMaaP Message Router Subscriber Timeout in MS. Defaults to no timeout") + @Nullable + @Macro + protected Integer timeoutMS; + + @Description("DMaaP Message Router Subscriber Message Limit. Defaults to no message limit") + @Nullable + @Macro + protected Integer messageLimit; + + // Required No Arg constructor + public DMaaPMRSourcePluginConfig() { + this(null, null, null, 0); + } + + public DMaaPMRSourcePluginConfig(String referenceName, String hostName, String topicName, Integer pollingInterval) { + super(referenceName, hostName, topicName); + this.pollingInterval = pollingInterval; + } + + /** + * DMaaP MR Subscriber Polling interval + * + * @return DMaaP MR Subscriber Polling interval + */ + public Integer getPollingInterval() { + return pollingInterval; + } + + /** + * DMaaP MR Subscriber Consumer ID + * + * @return DMaaP MR Subscriber Consumer ID + */ + @Nullable + public String getConsumerId() { + return consumerId; + } + + /** + * DMaaP MR Subscriber Consumer Group + * + * @return DMaaP MR Subscriber Consumer Group + */ + @Nullable + public String getConsumerGroup() { + return consumerGroup; + } + + /** + * DMaaP MR Subscriber Timeout in MS + * + * @return DMaaP MR Subscriber Timeout in MS + */ + @Nullable + public Integer getTimeoutMS() { + return timeoutMS; + } + + /** + * DMaaP MR Subscriber message limit + * + * @return DMaaP MR Subscriber Message limit + */ + @Nullable + public Integer getMessageLimit() { + return messageLimit; + } + + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("super", super.toString()) + .add("pollingInterval", pollingInterval) + .add("consumerId", consumerId) + .add("consumerGroup", consumerGroup) + .add("timeoutMS", timeoutMS) + .add("messageLimit", messageLimit) + .toString(); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java new file mode 100644 index 0000000..bffb403 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java @@ -0,0 +1,125 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Macro; +import co.cask.cdap.api.annotation.Name; +import com.google.common.base.Objects; +import org.onap.dcae.apod.analytics.cdap.common.settings.CDAPBasePluginConfig; + +/** + * Configuration for Json Path Filter Plugin + * + * @author Rajiv Singla . Creation Date: 3/2/2017. + */ +public class JsonPathFilterPluginConfig extends CDAPBasePluginConfig { + + private static final long serialVersionUID = 1L; + + @Name("incomingJsonFieldName") + @Description("Input schema field name that contain JSON used for filtering") + @Macro + protected String incomingJsonFieldName; + + + @Name("outputSchemaFieldName") + @Description("Name of the nullable boolean schema field name that will contain result of the filter matching") + @Macro + protected String outputSchemaFieldName; + + + @Name("jsonFilterMappings") + @Macro + @Description("Filters incoming JSON based on given filter mappings - in terms of JSON path and expected values." + + "Right hand side contains JSON path. Left hand side contains semicolon (';') separated expected values " + + "for that JSON Path. If all provided JSON Path mappings and corresponding values matches - " + + "output schema field will be marked as true") + protected String jsonFilterMappings; + + + @Name("schema") + @Description("Output Schema") + protected String schema; + + + public JsonPathFilterPluginConfig(final String referenceName, final String incomingJsonFieldName, + final String outputSchemaFieldName, final String jsonFilterMappings, + final String schema) { + this.referenceName = referenceName; + this.incomingJsonFieldName = incomingJsonFieldName; + this.outputSchemaFieldName = outputSchemaFieldName; + this.jsonFilterMappings = jsonFilterMappings; + this.schema = schema; + } + + /** + * Provides incoming plugin schema field name which contains json used to apply filter + * + * @return name of incoming schema field containing JSON to be filtered + */ + public String getIncomingJsonFieldName() { + return incomingJsonFieldName; + } + + /** + * Provides plugin output schema filed name that will contain result of filter application + * It must be nullable and boolean type + * + * @return name of outgoing schema filed name that will contain filtering result + */ + public String getOutputSchemaFieldName() { + return outputSchemaFieldName; + } + + /** + * Provides JSON filter mappings. LHS contains JSON path value and RHS contains expected + * values separated by semicolon + * + * + * @return String for JSON filter mappings + */ + public String getJsonFilterMappings() { + return jsonFilterMappings; + } + + /** + * Output Schema + * + * @return output schema string + */ + public String getSchema() { + return schema; + } + + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("referenceName", referenceName) + .add("incomingJsonFieldName", incomingJsonFieldName) + .add("outputSchemaFieldName", outputSchemaFieldName) + .add("jsonFilterMappings", jsonFilterMappings) + .add("schema", schema) + .toString(); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java new file mode 100644 index 0000000..dda3b54 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java @@ -0,0 +1,154 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Macro; +import com.google.common.base.Objects; +import org.onap.dcae.apod.analytics.cdap.common.settings.CDAPBasePluginConfig; + +import javax.annotation.Nullable; + +/** + * Simple TCA Plugin Configuration + *

+ * @author Rajiv Singla . Creation Date: 2/13/2017. + */ +public class SimpleTCAPluginConfig extends CDAPBasePluginConfig { + + private static final long serialVersionUID = 1L; + + @Description("Field name containing VES Message") + @Macro + protected String vesMessageFieldName; + + @Description("Policy JSON that need to be applied to VES Message") + @Macro + protected String policyJson; + + @Description("Name of the output field that will contain the alert") + @Macro + protected String alertFieldName; + + @Description("Name of the output field that will contain message type: INAPPLICABLE, COMPLIANT, NON_COMPLIANT") + @Macro + protected String messageTypeFieldName; + + @Description("Specifies the output schema") + protected String schema; + + @Description("Enables") + @Nullable + @Macro + protected Boolean enableAlertCEFFormat; + + + /** + * Creates an instance of TCA Plugin Configs + * + * @param vesMessageFieldName Ves message field name from incoming plugin schema + * @param policyJson TCA Policy Json String + * @param alertFieldName Alert field name that will be added in TCA plugin output schema + * @param messageTypeFieldName Message type field name that will be added in TCA plugin output schema + * @param schema TCA Plugin output schema + * @param enableAlertCEFFormat enables alert message to be formatted in VES format + */ + public SimpleTCAPluginConfig(final String vesMessageFieldName, final String policyJson, + final String alertFieldName, final String messageTypeFieldName, + final String schema, final Boolean enableAlertCEFFormat) { + this.vesMessageFieldName = vesMessageFieldName; + this.policyJson = policyJson; + this.alertFieldName = alertFieldName; + this.messageTypeFieldName = messageTypeFieldName; + this.schema = schema; + this.enableAlertCEFFormat = enableAlertCEFFormat; + } + + /** + * Name of the field containing VES Message + * + * @return VES Message field name + */ + public String getVesMessageFieldName() { + return vesMessageFieldName; + } + + /** + * Policy Json String + * + * @return Policy Json String + */ + public String getPolicyJson() { + return policyJson; + } + + + /** + * Alert Field name in outgoing schema + * + * @return alert field name in outgoing schema + */ + public String getAlertFieldName() { + return alertFieldName; + } + + /** + * Returns output schema string + * + * @return output schema string + */ + public String getSchema() { + return schema; + } + + /** + * Return TCA message type - INAPPLICABLE, COMPLIANT, NON_COMPLIANT + * + * @return tca message type + */ + public String getMessageTypeFieldName() { + return messageTypeFieldName; + } + + + /** + * Returns if Alert output in Common Event format + * + * @return true if alert output is in common event format + */ + @Nullable + public Boolean getEnableAlertCEFFormat() { + return enableAlertCEFFormat; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("referenceName", referenceName) + .add("vesMessageFieldName", vesMessageFieldName) + .add("policyJson", policyJson) + .add("alertFieldName", alertFieldName) + .add("messageTypeFieldName", messageTypeFieldName) + .add("schema", schema) + .add("enableAlertCEFFormat", true) + .toString(); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java new file mode 100644 index 0000000..183425b --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java @@ -0,0 +1,59 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap; + +import co.cask.cdap.api.data.schema.Schema; +import org.onap.dcae.apod.analytics.cdap.plugins.common.PluginSchema; + +/** + * Output Schema for DMaaP MR Source Plugin + * + * @author Rajiv Singla . Creation Date: 1/25/2017. + */ +public enum DMaaPSourceOutputSchema implements PluginSchema { + + TIMESTAMP("ts"), + RESPONSE_CODE("responseCode"), + RESPONSE_MESSAGE("responseMessage"), + FETCHED_MESSAGE("message"); + + private String schemaColumnName; + + DMaaPSourceOutputSchema(String schemaColumnName) { + this.schemaColumnName = schemaColumnName; + } + + @Override + public String getSchemaColumnName() { + return schemaColumnName; + } + + public static Schema getSchema() { + return Schema.recordOf( + "DMaaPMRSourcePluginResponse", + Schema.Field.of(TIMESTAMP.getSchemaColumnName(), Schema.of(Schema.Type.LONG)), + Schema.Field.of(RESPONSE_CODE.getSchemaColumnName(), Schema.of(Schema.Type.INT)), + Schema.Field.of(RESPONSE_MESSAGE.getSchemaColumnName(), Schema.of(Schema.Type.STRING)), + Schema.Field.of(FETCHED_MESSAGE.getSchemaColumnName(), Schema.of(Schema.Type.STRING)) + ); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java new file mode 100644 index 0000000..d2dd742 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java @@ -0,0 +1,175 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.sparkcompute.tca; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Name; +import co.cask.cdap.api.annotation.Plugin; +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.format.StructuredRecord.Builder; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.StageMetrics; +import co.cask.cdap.etl.api.batch.SparkCompute; +import co.cask.cdap.etl.api.batch.SparkExecutionPluginContext; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.Function; +import org.onap.dcae.apod.analytics.cdap.common.CDAPMetricsConstants; +import org.onap.dcae.apod.analytics.cdap.common.persistance.tca.TCACalculatorMessageType; +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.validator.SimpleTCAPluginConfigValidator; +import org.onap.dcae.apod.analytics.model.domain.policy.tca.MetricsPerEventName; +import org.onap.dcae.apod.analytics.model.domain.policy.tca.TCAPolicy; +import org.onap.dcae.apod.analytics.model.domain.policy.tca.Threshold; +import org.onap.dcae.apod.analytics.tca.processor.TCACEFJsonProcessor; +import org.onap.dcae.apod.analytics.tca.processor.TCACEFProcessorContext; +import org.onap.dcae.apod.analytics.tca.utils.TCAUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Rajiv Singla . Creation Date: 2/13/2017. + */ + +@Plugin(type = SparkCompute.PLUGIN_TYPE) +@Name("SimpleTCAPlugin") +@Description("Used to create TCA (Threshold Crossing Alert) based on given Policy") +@SuppressFBWarnings("SE_INNER_CLASS") +public class SimpleTCAPlugin extends SparkCompute { + + private static final Logger LOG = LoggerFactory.getLogger(SimpleTCAPlugin.class); + private static final long serialVersionUID = 1L; + + private final SimpleTCAPluginConfig pluginConfig; + + /** + * Create an instance of Simple TCA Plugin with give Simple TCA Plugin Config + * + * @param pluginConfig Simple TCA Plugin Config + */ + public SimpleTCAPlugin(SimpleTCAPluginConfig pluginConfig) { + this.pluginConfig = pluginConfig; + LOG.info("Creating instance of Simple TCA Plugin with plugin config: {}", pluginConfig); + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + ValidationUtils.validateSettings(pluginConfig, new SimpleTCAPluginConfigValidator()); + final Schema inputSchema = pipelineConfigurer.getStageConfigurer().getInputSchema(); + CDAPPluginUtils.validateSchemaContainsFields(inputSchema, pluginConfig.getVesMessageFieldName()); + CDAPPluginUtils.setOutputSchema(pipelineConfigurer, pluginConfig.getSchema()); + } + + @Override + public JavaRDD transform(final SparkExecutionPluginContext context, + final JavaRDD input) throws Exception { + final StageMetrics metrics = context.getMetrics(); + + LOG.debug("Invoking Spark Transform for Simple TCA Plugin"); + return input.map(new Function() { + + @Override + public StructuredRecord call(StructuredRecord inputStructuredRecord) throws Exception { + TCACalculatorMessageType calculatorMessageType; + String alertMessage = null; + + // Get input structured record + final String cefMessage = inputStructuredRecord.get(pluginConfig.getVesMessageFieldName()); + + // Get TCA Policy + final TCAPolicy tcaPolicy = CDAPPluginUtils.readValue(pluginConfig.getPolicyJson(), TCAPolicy.class); + + // create initial processor context + final TCACEFProcessorContext initialProcessorContext = + new TCACEFProcessorContext(cefMessage, tcaPolicy); + + final TCACEFJsonProcessor jsonProcessor = new TCACEFJsonProcessor(); + final TCACEFProcessorContext jsonProcessorContext = + jsonProcessor.processMessage(initialProcessorContext); + + if (jsonProcessorContext.getCEFEventListener() != null) { + + LOG.debug("Json to CEF parsing successful. Parsed object {}", + jsonProcessorContext.getCEFEventListener()); + + // compute violations + final TCACEFProcessorContext processorContextWithViolations = + TCAUtils.computeThresholdViolations(jsonProcessorContext); + + // if violation are found then create alert message + if (processorContextWithViolations.canProcessingContinue()) { + + alertMessage = TCAUtils.createTCAAlertString(processorContextWithViolations, + pluginConfig.getReferenceName(), pluginConfig.getEnableAlertCEFFormat()); + calculatorMessageType = TCACalculatorMessageType.NON_COMPLIANT; + + LOG.debug("VES Threshold Violation Detected.An alert message is be generated: {}", + alertMessage); + + final MetricsPerEventName metricsPerEventName = + processorContextWithViolations.getMetricsPerEventName(); + if (metricsPerEventName != null + && metricsPerEventName.getThresholds() != null + && metricsPerEventName.getThresholds().get(0) != null) { + final Threshold violatedThreshold = metricsPerEventName.getThresholds().get(0); + LOG.debug("CEF Message: {}, Violated Threshold: {}", cefMessage, violatedThreshold); + } + + metrics.count(CDAPMetricsConstants.TCA_VES_NON_COMPLIANT_MESSAGES_METRIC, 1); + + } else { + LOG.debug("No Threshold Violation Detected. No alert will be generated."); + calculatorMessageType = TCACalculatorMessageType.COMPLIANT; + metrics.count(CDAPMetricsConstants.TCA_VES_COMPLIANT_MESSAGES_METRIC, 1); + } + + } else { + LOG.info("Unable to parse provided json message to CEF format. Invalid message: {}", cefMessage); + calculatorMessageType = TCACalculatorMessageType.INAPPLICABLE; + } + + LOG.debug("Calculator message type: {} for message: {}", calculatorMessageType, cefMessage); + + final Schema outputSchema = Schema.parseJson(pluginConfig.getSchema()); + + // create new output record builder and copy any input record values to output record builder + final Builder outputRecordBuilder = + CDAPPluginUtils.createOutputStructuredRecordBuilder(outputSchema, inputStructuredRecord); + + // add alert field + final Builder outputRecordBuilderWithAlertField = + CDAPPluginUtils.addFieldValueToStructuredRecordBuilder(outputRecordBuilder, + outputSchema, pluginConfig.getAlertFieldName(), alertMessage); + + // add message field type + final Builder outRecordBuilderWithMessageTypeField = + CDAPPluginUtils.addFieldValueToStructuredRecordBuilder(outputRecordBuilderWithAlertField, + outputSchema, pluginConfig.getMessageTypeFieldName(), calculatorMessageType.toString()); + + return outRecordBuilderWithMessageTypeField.build(); + } + }); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java new file mode 100644 index 0000000..593d5d1 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java @@ -0,0 +1,118 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.metrics.Metrics; +import com.google.common.base.Optional; +import org.apache.spark.storage.StorageLevel; +import org.apache.spark.streaming.receiver.Receiver; +import org.onap.dcae.apod.analytics.cdap.common.utils.DMaaPMRUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.DMaaPSourceConfigMapper; +import org.onap.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; +import org.onap.dcae.apod.analytics.dmaap.DMaaPMRFactory; +import org.onap.dcae.apod.analytics.dmaap.service.subscriber.DMaaPMRSubscriber; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * DMaaP MR Receiver which calls DMaaP MR Topic and stores structured records + *

+ * @author Rajiv Singla . Creation Date: 1/19/2017. + */ +public class DMaaPMRReceiver extends Receiver { + + private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRReceiver.class); + private static final long serialVersionUID = 1L; + + private final DMaaPMRSourcePluginConfig pluginConfig; + private final transient Metrics metrics; + + public DMaaPMRReceiver(final StorageLevel storageLevel, final DMaaPMRSourcePluginConfig pluginConfig, + final Metrics metrics) { + super(storageLevel); + this.pluginConfig = pluginConfig; + this.metrics = metrics; + LOG.debug("Created DMaaP MR Receiver instance with plugin Config: {}", pluginConfig); + } + + @Override + public void onStart() { + + // create DMaaP MR Subscriber + final DMaaPMRSubscriber subscriber = + DMaaPMRFactory.create().createSubscriber(DMaaPSourceConfigMapper.map(pluginConfig)); + + // Start a new thread with indefinite loop until receiver is stopped + new Thread() { + @Override + public void run() { + while (!isStopped()) { + storeStructuredRecords(subscriber); + try { + final Integer pollingInterval = pluginConfig.getPollingInterval(); + LOG.debug("DMaaP MR Receiver sleeping for polling interval: {}", pollingInterval); + TimeUnit.MILLISECONDS.sleep(pollingInterval); + } catch (InterruptedException e) { + final String errorMessage = String.format( + "Interrupted Exception while DMaaP MR Receiver sleeping polling interval: %s", e); + throw new DCAEAnalyticsRuntimeException(errorMessage, LOG, e); + } + } + } + }.start(); + + } + + @Override + public void onStop() { + LOG.debug("Stopping DMaaP MR Receiver with plugin config: {}", pluginConfig); + } + + /** + * Fetches records from DMaaP MR Subscriber and store them as structured records + * + * @param subscriber DMaaP MR Subscriber Instance + */ + public void storeStructuredRecords(final DMaaPMRSubscriber subscriber) { + + LOG.debug("DMaaP MR Receiver start fetching messages from DMaaP MR Topic"); + + // Fetch messages from DMaaP MR Topic + final Optional> subscriberMessagesOptional = + DMaaPMRUtils.getSubscriberMessages(subscriber, metrics); + + // store records + if (subscriberMessagesOptional.isPresent()) { + final List messages = subscriberMessagesOptional.get(); + for (final String message : messages) { + store(CDAPPluginUtils.createDMaaPMRResponseStructuredRecord(message)); + } + LOG.debug("Stored DMaaP Subscriber messages as Structured Records. Message count {}", messages.size()); + } + } + +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java new file mode 100644 index 0000000..b85f653 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java @@ -0,0 +1,70 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Name; +import co.cask.cdap.api.annotation.Plugin; +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.streaming.StreamingContext; +import co.cask.cdap.etl.api.streaming.StreamingSource; +import org.apache.spark.storage.StorageLevel; +import org.apache.spark.streaming.api.java.JavaDStream; +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap.DMaaPSourceOutputSchema; +import org.onap.dcae.apod.analytics.cdap.plugins.validator.DMaaPMRSourcePluginConfigValidator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * DMaaP MR Source Plugin which polls DMaaP MR topic at frequent intervals + *

+ * @author Rajiv Singla . Creation Date: 1/18/2017. + */ +@Plugin(type = StreamingSource.PLUGIN_TYPE) +@Name("DMaaPMRSource") +@Description("Fetches DMaaP MR Messages at regular intervals") +public class DMaaPMRSource extends StreamingSource { + + private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRSource.class); + private static final long serialVersionUID = 1L; + + private final DMaaPMRSourcePluginConfig pluginConfig; + + public DMaaPMRSource(final DMaaPMRSourcePluginConfig pluginConfig) { + LOG.debug("Creating DMaaP MR Source plugin with plugin Config: {}", pluginConfig); + this.pluginConfig = pluginConfig; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + ValidationUtils.validateSettings(pluginConfig, new DMaaPMRSourcePluginConfigValidator()); + pipelineConfigurer.getStageConfigurer().setOutputSchema(DMaaPSourceOutputSchema.getSchema()); + } + + @Override + public JavaDStream getStream(final StreamingContext streamingContext) throws Exception { + return streamingContext.getSparkStreamingContext().receiverStream( + new DMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), pluginConfig, streamingContext.getMetrics())); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java new file mode 100644 index 0000000..f4f781a --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java @@ -0,0 +1,132 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.data.format.StructuredRecord; +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.spark.storage.StorageLevel; +import org.apache.spark.streaming.receiver.Receiver; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.DMaaPSourceConfigMapper; +import org.onap.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; +import org.onap.dcae.apod.analytics.dmaap.DMaaPMRFactory; +import org.onap.dcae.apod.analytics.dmaap.service.subscriber.DMaaPMRSubscriber; +import org.onap.dcae.apod.analytics.model.domain.cef.EventListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.onap.dcae.apod.analytics.model.util.AnalyticsModelJsonUtils.readValue; +import static org.onap.dcae.apod.analytics.model.util.AnalyticsModelJsonUtils.writeValueAsString; + +/** + * DMaaP MR Receiver which calls DMaaP MR Topic and stores structured records + *

+ * @author Rajiv Singla . Creation Date: 1/19/2017. + */ +public class MockDMaaPMRReceiver extends Receiver { + + private static final Logger LOG = LoggerFactory.getLogger(MockDMaaPMRReceiver.class); + private static final long serialVersionUID = 1L; + + private static final String MOCK_MESSAGE_FILE_LOCATION = "ves_mock_messages.json"; + private static final TypeReference> EVENT_LISTENER_TYPE_REFERENCE = + new TypeReference>() { + }; + + private final DMaaPMRSourcePluginConfig pluginConfig; + + public MockDMaaPMRReceiver(final StorageLevel storageLevel, final DMaaPMRSourcePluginConfig pluginConfig) { + super(storageLevel); + this.pluginConfig = pluginConfig; + LOG.debug("Created DMaaP MR Receiver instance with plugin Config: {}", pluginConfig); + } + + @Override + public void onStart() { + + // create DMaaP MR Subscriber + final DMaaPMRSubscriber subscriber = + DMaaPMRFactory.create().createSubscriber(DMaaPSourceConfigMapper.map(pluginConfig)); + storeStructuredRecords(subscriber); + + } + + @Override + public void onStop() { + LOG.debug("Stopping DMaaP MR Receiver with plugin config: {}", pluginConfig); + } + + /** + * Fetches records from DMaaP MR Subscriber and store them as structured records + * + * @param subscriber DMaaP MR Subscriber Instance + */ + public void storeStructuredRecords(final DMaaPMRSubscriber subscriber) { + + LOG.debug("DMaaP MR Receiver start fetching messages from DMaaP MR Topic"); + + try (InputStream resourceAsStream = + Thread.currentThread().getContextClassLoader().getResourceAsStream(MOCK_MESSAGE_FILE_LOCATION)) { + + if (resourceAsStream == null) { + LOG.error("Unable to find file at location: {}", MOCK_MESSAGE_FILE_LOCATION); + throw new DCAEAnalyticsRuntimeException("Unable to find file", LOG, new FileNotFoundException()); + } + + List eventListeners = readValue(resourceAsStream, EVENT_LISTENER_TYPE_REFERENCE); + + final int totalMessageCount = eventListeners.size(); + LOG.debug("Mock message count to be written to cdap stream: ()", totalMessageCount); + + int i = 1; + for (EventListener eventListener : eventListeners) { + if (isStopped()) { + return; + } + final String eventListenerString = writeValueAsString(eventListener); + LOG.debug("=======>> Writing message to cdap stream no: {} of {}", i, totalMessageCount); + store(CDAPPluginUtils.createDMaaPMRResponseStructuredRecord(eventListenerString)); + i++; + try { + TimeUnit.MILLISECONDS.sleep(pluginConfig.getPollingInterval()); + } catch (InterruptedException e) { + LOG.error("Error while sleeping"); + throw new DCAEAnalyticsRuntimeException("Error while sleeping", LOG, e); + } + + } + + LOG.debug("Finished writing mock messages to CDAP Stream"); + + } catch (IOException e) { + LOG.error("Error while parsing json file"); + throw new DCAEAnalyticsRuntimeException("Error while parsing mock json file", LOG, e); + } + } + +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java new file mode 100644 index 0000000..53b63e0 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java @@ -0,0 +1,73 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Name; +import co.cask.cdap.api.annotation.Plugin; +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.streaming.StreamingContext; +import co.cask.cdap.etl.api.streaming.StreamingSource; +import org.apache.spark.storage.StorageLevel; +import org.apache.spark.streaming.api.java.JavaDStream; +import org.onap.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A mock implementation of DMaaP MR Receiver which sends mock ves messages + *

+ * @author Rajiv Singla . Creation Date: 2/15/2017. + */ +@Plugin(type = StreamingSource.PLUGIN_TYPE) +@Name("MockDMaaPMRSource") +@Description("Fetches DMaaP MR Messages at regular intervals") +public class MockDMaaPMRSource extends StreamingSource { + + private static final Logger LOG = LoggerFactory.getLogger(MockDMaaPMRSource.class); + private static final long serialVersionUID = 1L; + + private final DMaaPMRSourcePluginConfig pluginConfig; + + public MockDMaaPMRSource(final DMaaPMRSourcePluginConfig pluginConfig) { + LOG.debug("Creating DMaaP MR Source plugin with plugin Config: {}", pluginConfig); + this.pluginConfig = pluginConfig; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + final Integer pollingInterval = pluginConfig.getPollingInterval(); + if (pollingInterval == null) { + final String errorMessage = "Polling Interval field must be present"; + throw new CDAPSettingsException(errorMessage, LOG, new IllegalArgumentException(errorMessage)); + } else { + LOG.info("Mock Message will be send every ms: {}", pollingInterval); + } + } + + @Override + public JavaDStream getStream(final StreamingContext streamingContext) throws Exception { + return streamingContext.getSparkStreamingContext().receiverStream( + new MockDMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), pluginConfig)); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java new file mode 100644 index 0000000..49c1ca4 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java @@ -0,0 +1,134 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.transform.filter; + +import co.cask.cdap.api.annotation.Description; +import co.cask.cdap.api.annotation.Name; +import co.cask.cdap.api.annotation.Plugin; +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.Emitter; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.Transform; +import co.cask.cdap.etl.api.TransformContext; +import com.google.common.base.Splitter; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter.JsonPathFilterPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.validator.JsonPathFilterPluginConfigValidator; +import org.onap.dcae.apod.analytics.common.service.filter.JsonMessageFilterProcessorContext; +import org.onap.dcae.apod.analytics.common.utils.MessageProcessorUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Set; + +/** + * Json Path filter Plugin filters incoming schema field based of given json path expected values + *

+ * @author Rajiv Singla . Creation Date: 3/2/2017. + */ + +@Plugin(type = Transform.PLUGIN_TYPE) +@Name("JsonPathFilter") +@Description("Filters incoming schema field based of given json path expected values") +public class JsonPathFilter extends Transform { + + private static final Logger LOG = LoggerFactory.getLogger(JsonPathFilter.class); + + private final JsonPathFilterPluginConfig pluginConfig; + private final Map> jsonFilterPathMappings; + + public JsonPathFilter(final JsonPathFilterPluginConfig pluginConfig) { + this.pluginConfig = pluginConfig; + jsonFilterPathMappings = Maps.newHashMap(); + LOG.info("Created instance of Json Path Filter Plugin with plugin config: {}", pluginConfig); + } + + + @Override + public void initialize(final TransformContext context) throws Exception { + super.initialize(context); + populateJsonFilterMapping(); + } + + @Override + public void configurePipeline(final PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + ValidationUtils.validateSettings(pluginConfig, new JsonPathFilterPluginConfigValidator()); + final Schema inputSchema = pipelineConfigurer.getStageConfigurer().getInputSchema(); + CDAPPluginUtils.validateSchemaContainsFields(inputSchema, pluginConfig.getIncomingJsonFieldName()); + populateJsonFilterMapping(); + CDAPPluginUtils.setOutputSchema(pipelineConfigurer, pluginConfig.getSchema()); + } + + @Override + public void transform(final StructuredRecord inputStructuredRecord, final Emitter emitter) + throws Exception { + + // get input json message + final String jsonMessage = inputStructuredRecord.get(pluginConfig.getIncomingJsonFieldName()); + + // process Json Filter Mappings + final JsonMessageFilterProcessorContext jsonMessageFilterProcessorContext = + MessageProcessorUtils.processJsonFilterMappings(jsonMessage, jsonFilterPathMappings); + + // create new output record builder and copy any input Structured record values to output record builder + final Schema outputSchema = Schema.parseJson(pluginConfig.getSchema()); + final StructuredRecord.Builder outputRecordBuilder = + CDAPPluginUtils.createOutputStructuredRecordBuilder(outputSchema, inputStructuredRecord); + + // add json filter matched field + final StructuredRecord.Builder outputRecordBuilderWithMatchedField = + CDAPPluginUtils.addFieldValueToStructuredRecordBuilder(outputRecordBuilder, + outputSchema, pluginConfig.getOutputSchemaFieldName(), + jsonMessageFilterProcessorContext.getMatched()); + + // emit structured record with filtering matched field + final StructuredRecord outputStructuredRecord = outputRecordBuilderWithMatchedField.build(); + + LOG.debug("Incoming Json Message: {}.Json Path Filter Output Matched Field: {}", jsonMessage, + outputStructuredRecord.get(pluginConfig.getOutputSchemaFieldName())); + + emitter.emit(outputStructuredRecord); + + } + + /** + * Populates Json Filter Mapping + */ + private void populateJsonFilterMapping() { + final Map fieldMappings = + CDAPPluginUtils.extractFieldMappings(pluginConfig.getJsonFilterMappings()); + if (fieldMappings.isEmpty()) { + throw new IllegalArgumentException("No Field Mapping found. Invalid Filter mapping configuration"); + } + final Splitter semiColonSplitter = Splitter.on(";"); + for (Map.Entry fieldMappingEntry : fieldMappings.entrySet()) { + jsonFilterPathMappings.put(fieldMappingEntry.getKey(), + Sets.newLinkedHashSet(semiColonSplitter.split(fieldMappingEntry.getValue()))); + } + LOG.info("Input Json Filter Mappings: {}", jsonFilterPathMappings); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java new file mode 100644 index 0000000..5f622cd --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java @@ -0,0 +1,295 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.utils; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.PipelineConfigurer; +import com.google.common.base.Function; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; +import org.onap.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap.DMaaPSourceOutputSchema; +import org.onap.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; +import org.onap.dcae.apod.analytics.model.util.AnalyticsModelJsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * @author Rajiv Singla . Creation Date: 1/26/2017. + */ +public abstract class CDAPPluginUtils extends AnalyticsModelJsonUtils { + + private static final Logger LOG = LoggerFactory.getLogger(CDAPPluginUtils.class); + + public static final Function SCHEMA_TO_TYPE_FUNCTION = new Function() { + @Override + public Schema.Type apply(@Nonnull Schema schema) { + return schema.getType(); + } + }; + + + + private CDAPPluginUtils() { + // private constructor + } + + /** + * Validates if CDAP Schema contains expected fields + * + * @param schema schema that need to be validated + * @param expectedFields fields that are expected to be in the schema + */ + + public static void validateSchemaContainsFields(@Nullable final Schema schema, final String... expectedFields) { + + LOG.debug("Validating schema:{} contains expected fields:{}", schema, Arrays.toString(expectedFields)); + + if (schema == null) { + // If input schema is null then no validation possible + LOG.warn("Input Schema is null. No validation possible"); + } else { + // Check if expected fields are indeed present in the schema + for (String expectedField : expectedFields) { + final Schema.Field schemaField = schema.getField(expectedField); + if (schemaField == null) { + final String errorMessage = String.format( + "Unable to find expected field: %s, in schema: %s", expectedField, schema); + throw new CDAPSettingsException(errorMessage, LOG, new IllegalArgumentException(errorMessage)); + } + } + LOG.debug("Successfully validated schema:{}, contains expected fields:{}", schema, + Arrays.toString(expectedFields)); + } + } + + + /** + * Creates a new Structured Record containing DMaaP MR fetched message + * + * @param message DMaaP MR fetch message + * + * @return Structured record containing DMaaP MR Message + */ + public static StructuredRecord createDMaaPMRResponseStructuredRecord(final String message) { + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(DMaaPSourceOutputSchema.getSchema()); + recordBuilder + .set(DMaaPSourceOutputSchema.TIMESTAMP.getSchemaColumnName(), System.nanoTime()) + .set(DMaaPSourceOutputSchema.RESPONSE_CODE.getSchemaColumnName(), 200) + .set(DMaaPSourceOutputSchema.RESPONSE_MESSAGE.getSchemaColumnName(), "OK") + .set(DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName(), message); + return recordBuilder.build(); + } + + + /** + * Creates output StructuredRecord Builder which has copied values from input StructuredRecord + * + * @param outputSchema output Schema + * @param inputStructuredRecord input Structured Record + * + * @return output Structured Record builder with pre populated values from input structured record + */ + public static StructuredRecord.Builder createOutputStructuredRecordBuilder( + @Nonnull final Schema outputSchema, + @Nonnull final StructuredRecord inputStructuredRecord) { + + // Get input structured Record Schema + final Schema inputSchema = inputStructuredRecord.getSchema(); + // Create new instance of output Structured Record Builder from output Schema + final StructuredRecord.Builder outputStructuredRecordBuilder = StructuredRecord.builder(outputSchema); + + // iterate over input fields and if output schema has field with same name copy the value to out record builder + for (Schema.Field inputField : inputSchema.getFields()) { + final String inputFieldName = inputField.getName(); + if (outputSchema.getField(inputFieldName) != null) { + outputStructuredRecordBuilder.set(inputFieldName, inputStructuredRecord.get(inputFieldName)); + } + } + + return outputStructuredRecordBuilder; + } + + + /** + * Adds Field value to StructuredRecord Builder if schema contains that field Name + * + * @param structuredRecordBuilder structured record builder + * @param structuredRecordSchema schema for structured record builder + * @param fieldName field name + * @param fieldValue field value + * + * @return structured record builder with populated field name and value if schema contains field name + */ + public static StructuredRecord.Builder addFieldValueToStructuredRecordBuilder( + @Nonnull final StructuredRecord.Builder structuredRecordBuilder, + @Nonnull final Schema structuredRecordSchema, + @Nonnull final String fieldName, + final Object fieldValue) { + + // check if schema contains field Name + if (structuredRecordSchema.getField(fieldName) != null) { + structuredRecordBuilder.set(fieldName, fieldValue); + } else { + LOG.info("Unable to populate value for field Name: {} with field value: {}. " + + "Schema Fields: {} does not contain field name: {}", + fieldName, fieldValue, structuredRecordSchema.getFields(), fieldName); + } + + return structuredRecordBuilder; + } + + + /** + * Validates that given schema String has fieldName of expected type. If field does not exist in given schema + * then validation will pass with warning. If field does exist in given schema then this validation will return + * true if field type is same as expected type else false + * + * @param schemaString CDAP Plugin output or input schema string + * @param fieldName field name + * @param expectedFieldType expected schema field type + * + * @return true if field type matches expected field type else false. If field does not exist in + * give schema validation will pass but will generate a warning message + */ + public static boolean validateSchemaFieldType(@Nonnull final String schemaString, + @Nonnull final String fieldName, + @Nonnull final Schema.Type expectedFieldType) { + + try { + // parse given schema String + final Schema outputSchema = Schema.parseJson(schemaString); + final Schema.Field schemaField = outputSchema.getField(fieldName); + + // if given schema does contain field then validated fieldName type + if (schemaField != null) { + + final List schemas = new LinkedList<>(); + + // if it is a union type then grab all union schemas + if (outputSchema.getField(fieldName).getSchema().getType() == Schema.Type.UNION) { + final List unionFieldSchemas = + outputSchema.getField(fieldName).getSchema().getUnionSchemas(); + schemas.addAll(unionFieldSchemas); + } else { + // if not union type the just get the field schema + final Schema fieldSchema = outputSchema.getField(fieldName).getSchema(); + schemas.add(fieldSchema); + } + + // get all schema types + final List fieldTypes = + Lists.transform(schemas, CDAPPluginUtils.SCHEMA_TO_TYPE_FUNCTION); + + // if all schema types does not contain expected field type then return false + if (!fieldTypes.contains(expectedFieldType)) { + LOG.error("Validation failed for fieldName: {} is NOT of expected Type: {} in schema: {}", + fieldName, expectedFieldType, outputSchema); + return false; + } + + // field type validation passed + LOG.debug("Successfully validated fieldName: {} is of expected Type: {}", + fieldName, expectedFieldType); + + return true; + + } else { + + // if field does not exist then the validation will pass but will generate warning message + LOG.warn("Validation of field type not possible. Field name: {} does not exist in schema: {}", + fieldName, outputSchema); + return true; + } + + } catch (IOException e) { + final String errorMessage = + String.format("Unable to parse schema: %s for field type validation. " + + "Field Name: %s, Expected Field Type: %s Exception: %s", + schemaString, fieldName, expectedFieldType, e); + throw new DCAEAnalyticsRuntimeException(errorMessage, LOG, e); + } + + } + + + /** + * Parses provided schema String as Schema object and set it as output Schema format + * + * @param pipelineConfigurer plugin pipeline configurer + * @param schemaString schema String to be set as output schema + */ + public static void setOutputSchema(final PipelineConfigurer pipelineConfigurer, final String schemaString) { + try { + final Schema outputSchema = Schema.parseJson(schemaString); + pipelineConfigurer.getStageConfigurer().setOutputSchema(outputSchema); + } catch (IOException e) { + final String errorMessage = String.format( + "Schema specified is not a valid JSON. Schema String: %s, Exception: %s", schemaString, e); + throw new CDAPSettingsException(errorMessage, LOG, new IllegalArgumentException(errorMessage)); + } + } + + + /** + * Parses incoming plugin config mapping to key value map. If any of the key value map is blank an Illegal Argument + * exception will be thrown + * + * @param mappingFieldString field Mapping String + * + * @return map containing mapping key values + */ + public static Map extractFieldMappings(final String mappingFieldString) { + final Map fieldMappings = Maps.newHashMap(); + if (StringUtils.isNotBlank(mappingFieldString)) { + final Splitter commaSplitter = Splitter.on(","); + for (String fieldMapping : commaSplitter.split(mappingFieldString)) { + final String[] keyValueMappings = fieldMapping.split(":"); + if (keyValueMappings.length != 2 || + StringUtils.isBlank(keyValueMappings[0]) || + StringUtils.isBlank(keyValueMappings[1])) { + final String errorMessage = "Field Mapping key or value is Blank. All field mappings must " + + "be present in mappings: " + mappingFieldString; + throw new DCAEAnalyticsRuntimeException( + errorMessage, LOG, new IllegalArgumentException(errorMessage)); + } + fieldMappings.put(keyValueMappings[0].trim(), keyValueMappings[1].trim()); + } + } + return fieldMappings; + } + + + + +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java new file mode 100644 index 0000000..d5aa2c0 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java @@ -0,0 +1,112 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.utils; + +import com.google.common.base.Function; +import org.apache.hadoop.conf.Configuration; +import org.onap.dcae.apod.analytics.cdap.common.CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields; +import org.onap.dcae.apod.analytics.dmaap.domain.config.DMaaPMRPublisherConfig; + +import javax.annotation.Nonnull; + +import static org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isEmpty; +import static org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isPresent; + +/** + * Function that converts {@link Configuration} to {@link DMaaPMRPublisherConfig} + *

+ * @author Rajiv Singla . Creation Date: 1/26/2017. + */ +public class DMaaPSinkConfigMapper implements Function { + + /** + * Static method to map {@link Configuration} to {@link DMaaPMRPublisherConfig} + * + * @param sinkPluginConfig DMaaP Sink Plugin Config + * + * @return DMaaP MR Publisher Config + */ + public static DMaaPMRPublisherConfig map(final Configuration sinkPluginConfig) { + return new DMaaPSinkConfigMapper().apply(sinkPluginConfig); + } + + /** + * Converts {@link Configuration} to {@link DMaaPMRPublisherConfig} + * + * @param configuration Hadoop Configuration containing DMaaP MR Sink field values + * + * @return DMaaP MR Publisher Config + */ + @Nonnull + @Override + public DMaaPMRPublisherConfig apply(@Nonnull Configuration configuration) { + + // Create a new publisher settings builder + final String hostName = configuration.get(DMaaPMRSinkHadoopConfigFields.HOST_NAME); + final String topicName = configuration.get(DMaaPMRSinkHadoopConfigFields.TOPIC_NAME); + + if (isEmpty(hostName) || isEmpty(topicName)) { + throw new IllegalStateException("DMaaP MR Sink Host Name and Topic Name must be present"); + } + + final DMaaPMRPublisherConfig.Builder publisherConfigBuilder = + new DMaaPMRPublisherConfig.Builder(hostName, topicName); + + // Setup up any optional publisher parameters if they are present + final String portNumber = configuration.get(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER); + if (portNumber != null) { + publisherConfigBuilder.setPortNumber(Integer.parseInt(portNumber)); + } + + final String protocol = configuration.get(DMaaPMRSinkHadoopConfigFields.PROTOCOL); + if (isPresent(protocol)) { + publisherConfigBuilder.setProtocol(protocol); + } + + final String userName = configuration.get(DMaaPMRSinkHadoopConfigFields.USER_NAME); + if (isPresent(userName)) { + publisherConfigBuilder.setUserName(userName); + } + + final String userPassword = configuration.get(DMaaPMRSinkHadoopConfigFields.USER_PASS); + if (isPresent(userPassword)) { + publisherConfigBuilder.setUserPassword(userPassword); + } + + final String contentType = configuration.get(DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE); + if (isPresent(contentType)) { + publisherConfigBuilder.setContentType(contentType); + } + + final String maxBatchSize = configuration.get(DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE); + if (maxBatchSize != null) { + publisherConfigBuilder.setMaxBatchSize(Integer.parseInt(maxBatchSize)); + } + + final String maxRecoveryQueueSize = configuration.get(DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE); + if (maxRecoveryQueueSize != null) { + publisherConfigBuilder.setMaxRecoveryQueueSize(Integer.parseInt(maxRecoveryQueueSize)); + } + + return publisherConfigBuilder.build(); + + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java new file mode 100644 index 0000000..c492b2d --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java @@ -0,0 +1,118 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.utils; + +import com.google.common.base.Function; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.dmaap.domain.config.DMaaPMRSubscriberConfig; + +import javax.annotation.Nonnull; + +import static org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isEmpty; +import static org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isPresent; + +/** + * Function that converts {@link DMaaPMRSourcePluginConfig} to {@link DMaaPMRSubscriberConfig} + *

+ * @author Rajiv Singla . Creation Date: 1/18/2017. + */ +public class DMaaPSourceConfigMapper implements Function { + + /** + * Static factory method to map {@link DMaaPMRSourcePluginConfig} to {@link DMaaPMRSubscriberConfig} + * + * @param pluginConfig DMaaP MR Souce Plugin Config + * + * @return DMaaP MR Subscriber Config + */ + public static DMaaPMRSubscriberConfig map(final DMaaPMRSourcePluginConfig pluginConfig) { + return new DMaaPSourceConfigMapper().apply(pluginConfig); + } + + /** + * Converts {@link DMaaPMRSourcePluginConfig} to {@link DMaaPMRSubscriberConfig} object + * + * @param sourcePluginConfig DMaaP MR Source Plugin Config + * + * @return DMaaP MR Subscriber Config + */ + @Nonnull + @Override + public DMaaPMRSubscriberConfig apply(@Nonnull DMaaPMRSourcePluginConfig sourcePluginConfig) { + + // Create a new subscriber settings builder + final String hostName = sourcePluginConfig.getHostName(); + final String topicName = sourcePluginConfig.getTopicName(); + if (isEmpty(hostName) || isEmpty(topicName)) { + throw new IllegalStateException("DMaaP MR Source Host Name and Topic Name must be present"); + } + final DMaaPMRSubscriberConfig.Builder subscriberConfigBuilder = new DMaaPMRSubscriberConfig.Builder( + hostName, topicName); + + // Setup up any optional subscriber parameters if they are present + final Integer subscriberHostPortNumber = sourcePluginConfig.getPortNumber(); + if (subscriberHostPortNumber != null) { + subscriberConfigBuilder.setPortNumber(subscriberHostPortNumber); + } + + final String subscriberProtocol = sourcePluginConfig.getProtocol(); + if (isPresent(subscriberProtocol)) { + subscriberConfigBuilder.setProtocol(subscriberProtocol); + } + + final String subscriberUserName = sourcePluginConfig.getUserName(); + if (isPresent(subscriberUserName)) { + subscriberConfigBuilder.setUserName(subscriberUserName); + } + + final String subscriberUserPassword = sourcePluginConfig.getUserPassword(); + if (isPresent(subscriberUserPassword)) { + subscriberConfigBuilder.setUserPassword(subscriberUserPassword); + } + + final String subscriberContentType = sourcePluginConfig.getContentType(); + if (isPresent(subscriberContentType)) { + subscriberConfigBuilder.setContentType(subscriberContentType); + } + + final String subscriberConsumerId = sourcePluginConfig.getConsumerId(); + if (isPresent(subscriberConsumerId)) { + subscriberConfigBuilder.setConsumerId(subscriberConsumerId); + } + + final String subscriberConsumerGroup = sourcePluginConfig.getConsumerGroup(); + if (isPresent(subscriberConsumerGroup)) { + subscriberConfigBuilder.setConsumerGroup(subscriberConsumerGroup); + } + + final Integer subscriberTimeoutMS = sourcePluginConfig.getTimeoutMS(); + if (subscriberTimeoutMS != null) { + subscriberConfigBuilder.setTimeoutMS(subscriberTimeoutMS); + } + final Integer subscriberMessageLimit = sourcePluginConfig.getMessageLimit(); + if (subscriberMessageLimit != null) { + subscriberConfigBuilder.setMessageLimit(subscriberMessageLimit); + } + + // return Subscriber config + return subscriberConfigBuilder.build(); + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java new file mode 100644 index 0000000..8ebc8e6 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java @@ -0,0 +1,72 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.BaseDMaaPMRPluginConfig; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +/** + * Validates plugin config values which are common in DMaaP MR Configs - {@link BaseDMaaPMRPluginConfig} + *

+ * @author Rajiv Singla . Creation Date: 1/23/2017. + * + * @param {@link BaseDMaaPMRPluginConfig} Sub classes + */ +public abstract class BaseDMaaPMRPluginConfigValidator implements + CDAPAppSettingsValidator> { + + private static final long serialVersionUID = 1L; + + /** + * Validates the {@link BaseDMaaPMRPluginConfig} parameters + * + * @param baseDMaaPMRPluginConfig DMaaP MR Plugin Config + * + * @return Validation Response containing validation errors if any + */ + @Override + public GenericValidationResponse validateAppSettings(final T baseDMaaPMRPluginConfig) { + + final GenericValidationResponse validationResponse = new GenericValidationResponse<>(); + + if (ValidationUtils.isEmpty(baseDMaaPMRPluginConfig.getHostName())) { + validationResponse.addErrorMessage( + "hostName", + "DMaaPMRPluginConfig - hostname field is undefined: " + baseDMaaPMRPluginConfig); + } + + if (baseDMaaPMRPluginConfig.getPortNumber() == null) { + validationResponse.addErrorMessage( + "port Number", + "DMaaPMRPluginConfig - host port number field is undefined: " + baseDMaaPMRPluginConfig); + } + + if (ValidationUtils.isEmpty(baseDMaaPMRPluginConfig.getTopicName())) { + validationResponse.addErrorMessage( + "topic Name", + "DMaaPMRSourcePluginConfig - topic name field is undefined: " + baseDMaaPMRPluginConfig); + } + + return validationResponse; + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java new file mode 100644 index 0000000..ab9dd30 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java @@ -0,0 +1,58 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +/** + * Validates plugin config values in {@link DMaaPMRSinkPluginConfig} + *

+ * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMRSinkPluginConfigValidator extends BaseDMaaPMRPluginConfigValidator { + + private static final long serialVersionUID = 1L; + + /** + * Validates plugin config values in {@link DMaaPMRSinkPluginConfig} + * + * @param sinkPluginConfig Sink Plugin Config + * + * @return Validation response containing validation errors if any + */ + @Override + public GenericValidationResponse validateAppSettings( + final DMaaPMRSinkPluginConfig sinkPluginConfig) { + + // validate settings in BaseDMaaPMRPluginConfig + final GenericValidationResponse validationResponse = + super.validateAppSettings(sinkPluginConfig); + + if (ValidationUtils.isEmpty(sinkPluginConfig.getMessageColumnName())) { + validationResponse.addErrorMessage("messageColumn Name", + "DMaaPMRSinkPluginConfig - message column name field is undefined: " + sinkPluginConfig); + } + + return validationResponse; + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java new file mode 100644 index 0000000..d57cf67 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java @@ -0,0 +1,58 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +/** + * Validates plugin config values in {@link DMaaPMRSourcePluginConfig} + *

+ * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMRSourcePluginConfigValidator extends BaseDMaaPMRPluginConfigValidator { + + private static final long serialVersionUID = 1L; + + /** + * Validates plugin config values in {@link DMaaPMRSourcePluginConfig} + * + * @param sourcePluginConfig Source Plugin Config + * + * @return Validation response containing validation errors if any + */ + @Override + public GenericValidationResponse validateAppSettings( + final DMaaPMRSourcePluginConfig sourcePluginConfig) { + + // validate settings in BaseDMaaPMRPluginConfig + final GenericValidationResponse validationResponse = + super.validateAppSettings(sourcePluginConfig); + + if (sourcePluginConfig.getPollingInterval() == null) { + validationResponse.addErrorMessage( + "port Number", + "DMaaPMRSourcePluginConfig - polling interval is undefined: " + sourcePluginConfig); + } + + return validationResponse; + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java new file mode 100644 index 0000000..e41fec6 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java @@ -0,0 +1,83 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import co.cask.cdap.api.data.schema.Schema; +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter.JsonPathFilterPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +/** + * Validator to validate {@link JsonPathFilterPluginConfig} + *

+ * @author Rajiv Singla . Creation Date: 3/2/2017. + */ +public class JsonPathFilterPluginConfigValidator implements CDAPAppSettingsValidator> { + + private static final long serialVersionUID = 1L; + + @Override + public GenericValidationResponse validateAppSettings( + final JsonPathFilterPluginConfig jsonPathFilterPluginConfig) { + + final GenericValidationResponse validationResponse = + new GenericValidationResponse<>(); + + final String jsonFilterMappings = jsonPathFilterPluginConfig.getJsonFilterMappings(); + if (ValidationUtils.isEmpty(jsonFilterMappings)) { + + validationResponse.addErrorMessage("JsonFilterMappings", "Json Filter Mappings must be present"); + } + + + final String matchedField = jsonPathFilterPluginConfig.getOutputSchemaFieldName(); + final String outputSchemaJson = jsonPathFilterPluginConfig.getSchema(); + + if (ValidationUtils.isEmpty(outputSchemaJson)) { + + validationResponse.addErrorMessage("output schema", "Output schema is not present"); + + } else { + + // validate matched output field type is boolean + if (matchedField != null && + !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, matchedField, Schema.Type.BOOLEAN)) { + validationResponse.addErrorMessage("OutputSchemaFieldName", + String.format( + "OutputSchemaFieldName: %s must be marked as boolean type", matchedField)); + } + + // validate matched output field type is nullable + if (matchedField != null && + !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, matchedField, Schema.Type.NULL)) { + validationResponse.addErrorMessage("OutputSchemaFieldName", + String.format( + "OutputSchemaFieldName: %s must be marked as nullable type", matchedField)); + } + + } + + return validationResponse; + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java new file mode 100644 index 0000000..425aab6 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/main/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java @@ -0,0 +1,91 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import co.cask.cdap.api.data.schema.Schema; +import org.onap.dcae.apod.analytics.cdap.common.utils.ValidationUtils; +import org.onap.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +/** + * Validator that validate {@link SimpleTCAPluginConfig} + *

+ * @author Rajiv Singla . Creation Date: 2/21/2017. + */ +public class SimpleTCAPluginConfigValidator implements CDAPAppSettingsValidator> { + + private static final long serialVersionUID = 1L; + + @Override + public GenericValidationResponse validateAppSettings( + final SimpleTCAPluginConfig tcaPluginConfig) { + + final GenericValidationResponse validationResponse = new GenericValidationResponse<>(); + + if (ValidationUtils.isEmpty(tcaPluginConfig.getVesMessageFieldName())) { + validationResponse.addErrorMessage("vesMessageFieldName", + "Missing VES Message Field Name from plugin incoming schema"); + } + + if (ValidationUtils.isEmpty(tcaPluginConfig.getPolicyJson())) { + validationResponse.addErrorMessage("policyJson", + "Missing tca Policy Json"); + } + + final String alertFieldValue = tcaPluginConfig.getAlertFieldName(); + final String alertFieldName = "alertFieldName"; + if (ValidationUtils.isEmpty(alertFieldValue)) { + validationResponse.addErrorMessage(alertFieldName, + "Missing alert Field Name that will be placed in plugin outgoing schema"); + } + + if (ValidationUtils.isEmpty(tcaPluginConfig.getMessageTypeFieldName())) { + validationResponse.addErrorMessage("messageTypeField", + "Missing message Type Field Name that will be placed in plugin outgoing schema"); + } + + + final String outputSchemaJson = tcaPluginConfig.getSchema(); + if (ValidationUtils.isEmpty(outputSchemaJson)) { + validationResponse.addErrorMessage("output schema", "Output schema is not present"); + } else { + // validate output schema - alert field name is of type string + if (alertFieldValue != null && + !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, alertFieldValue, Schema.Type.STRING)) { + validationResponse.addErrorMessage(alertFieldName, + String.format( + "Alert Field Name: %s must be String type", alertFieldValue)); + } + // validate output schema - alert field name is nullable + if (alertFieldValue != null && + !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, alertFieldValue, Schema.Type.NULL)) { + validationResponse.addErrorMessage(alertFieldName, + String.format( + "Alert Field Name: %s must be marked as nullable type", alertFieldValue)); + } + } + + return validationResponse; + } +} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java deleted file mode 100644 index fdb7975..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormat.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.OutputFormat; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.DMaaPSinkConfigMapper; -import org.openecomp.dcae.apod.analytics.dmaap.DMaaPMRFactory; -import org.openecomp.dcae.apod.analytics.dmaap.domain.config.DMaaPMRPublisherConfig; -import org.openecomp.dcae.apod.analytics.dmaap.service.publisher.DMaaPMRPublisher; - -import java.io.IOException; - -/** - * DMaaP MR Output format used by DMaaP MR Sink Plugin to create a MR Publisher and pass to custom {@link - * DMaaPMRRecordWriter} - *

- * @author Rajiv Singla . Creation Date: 1/27/2017. - */ -public class DMaaPMROutputFormat extends OutputFormat { - - @Override - public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, - InterruptedException { - final Configuration configuration = context.getConfiguration(); - final DMaaPMRPublisherConfig publisherConfig = DMaaPSinkConfigMapper.map(configuration); - final DMaaPMRPublisher publisher = DMaaPMRFactory.create().createPublisher(publisherConfig); - return new DMaaPMRRecordWriter(publisher); - } - - @Override - public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { - // do nothing - } - - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException { - return new NoOpOutputCommitter(); - } - - /** - * A dummy implementation for {@link OutputCommitter} that does nothing. - */ - protected static class NoOpOutputCommitter extends OutputCommitter { - - @Override - public void setupJob(JobContext jobContext) throws IOException { - // no op - } - - @Override - public void setupTask(TaskAttemptContext taskContext) throws IOException { - // no op - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { - return false; - } - - @Override - public void commitTask(TaskAttemptContext taskContext) throws IOException { - // no op - } - - @Override - public void abortTask(TaskAttemptContext taskContext) throws IOException { - // no op - } - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java deleted file mode 100644 index bec04b3..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProvider.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import co.cask.cdap.api.data.batch.OutputFormatProvider; -import org.openecomp.dcae.apod.analytics.cdap.common.CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; -import org.openecomp.dcae.apod.analytics.common.AnalyticsConstants; - -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * DMaaP MR Output Format Provider used to create Batch Sink Plugin - *

- * @author Rajiv Singla . Creation Date: 1/27/2017. - */ -public class DMaaPMROutputFormatProvider implements OutputFormatProvider { - - private final Map sinkConfig; - - - public DMaaPMROutputFormatProvider(DMaaPMRSinkPluginConfig sinkPluginConfig) { - - // initialize Sink Config - with DMaaP MR Publisher config values - sinkConfig = new LinkedHashMap<>(); - - // Required fields for sink config - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.HOST_NAME, sinkPluginConfig.getHostName()); - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.TOPIC_NAME, sinkPluginConfig.getTopicName()); - - final Integer configPortNumber = sinkPluginConfig.getPortNumber(); - if (configPortNumber != null) { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER, configPortNumber.toString()); - } else { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER, - AnalyticsConstants.DEFAULT_PORT_NUMBER.toString()); - } - - final String configProtocol = sinkPluginConfig.getProtocol(); - if (ValidationUtils.isPresent(configProtocol)) { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PROTOCOL, configProtocol); - } else { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.PROTOCOL, AnalyticsConstants.DEFAULT_PROTOCOL); - } - - - final String configUserName = sinkPluginConfig.getUserName(); - if (ValidationUtils.isPresent(configUserName)) { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_NAME, configUserName); - } else { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_NAME, AnalyticsConstants.DEFAULT_USER_NAME); - } - - final String configUserPass = sinkPluginConfig.getUserPassword(); - if (ValidationUtils.isPresent(configUserPass)) { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_PASS, configUserPass); - } else { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.USER_PASS, AnalyticsConstants.DEFAULT_USER_PASSWORD); - } - - final String configContentType = sinkPluginConfig.getContentType(); - if (ValidationUtils.isPresent(configContentType)) { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE, configContentType); - } else { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE, AnalyticsConstants.DEFAULT_CONTENT_TYPE); - } - - - final Integer configMaxBatchSize = sinkPluginConfig.getMaxBatchSize(); - if (configMaxBatchSize != null) { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE, configMaxBatchSize.toString()); - } else { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE, - String.valueOf(AnalyticsConstants.DEFAULT_PUBLISHER_MAX_BATCH_SIZE)); - } - - final Integer configMaxRecoveryQueueSize = sinkPluginConfig.getMaxRecoveryQueueSize(); - if (configMaxRecoveryQueueSize != null) { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE, configMaxRecoveryQueueSize.toString()); - } else { - sinkConfig.put(DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE, - String.valueOf(AnalyticsConstants.DEFAULT_PUBLISHER_MAX_RECOVERY_QUEUE_SIZE)); - } - - } - - @Override - public String getOutputFormatClassName() { - return DMaaPMROutputFormat.class.getName(); - } - - @Override - public Map getOutputFormatConfiguration() { - return sinkConfig; - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java deleted file mode 100644 index f9c99e2..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriter.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.openecomp.dcae.apod.analytics.dmaap.service.publisher.DMaaPMRPublisher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; - -/** - * A simple implementation of {@link RecordWriter} which writes messages to DMaaP MR topic - *

- * @author Rajiv Singla . Creation Date: 1/27/2017. - */ -public class DMaaPMRRecordWriter extends RecordWriter { - - private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRRecordWriter.class); - - private final DMaaPMRPublisher dMaaPMRPublisher; - - public DMaaPMRRecordWriter(DMaaPMRPublisher dMaaPMRPublisher) { - this.dMaaPMRPublisher = dMaaPMRPublisher; - } - - @Override - public void write(String message, NullWritable value) throws IOException, InterruptedException { - LOG.debug("Writing message to DMaaP MR Topic: {}", message); - dMaaPMRPublisher.publish(Arrays.asList(message)); - } - - @Override - public void close(TaskAttemptContext context) throws IOException, InterruptedException { - dMaaPMRPublisher.flush(); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java deleted file mode 100644 index b92ecba..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSink.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Name; -import co.cask.cdap.api.annotation.Plugin; -import co.cask.cdap.api.data.batch.Output; -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.api.dataset.lib.KeyValue; -import co.cask.cdap.etl.api.Emitter; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.batch.BatchSink; -import co.cask.cdap.etl.api.batch.BatchSinkContext; -import org.apache.hadoop.io.NullWritable; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.validator.DMaaPMRSinkPluginConfigValidator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author Rajiv Singla . Creation Date: 1/26/2017. - */ -@Plugin(type = BatchSink.PLUGIN_TYPE) -@Name("DMaaPMRSink") -@Description("A batch sink Plugin that publishes messages to DMaaP MR Topic.") -public class DMaaPMRSink extends BatchSink { - - private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRSink.class); - - private final DMaaPMRSinkPluginConfig pluginConfig; - - public DMaaPMRSink(final DMaaPMRSinkPluginConfig pluginConfig) { - LOG.debug("Creating DMaaP MR Sink Plugin with plugin Config: {}", pluginConfig); - this.pluginConfig = pluginConfig; - } - - @Override - public void configurePipeline(final PipelineConfigurer pipelineConfigurer) { - super.configurePipeline(pipelineConfigurer); - ValidationUtils.validateSettings(pluginConfig, new DMaaPMRSinkPluginConfigValidator()); - // validates that input schema contains the field provided in Sink Message Column Name property - final Schema inputSchema = pipelineConfigurer.getStageConfigurer().getInputSchema(); - CDAPPluginUtils.validateSchemaContainsFields(inputSchema, pluginConfig.getMessageColumnName()); - } - - - @Override - public void prepareRun(BatchSinkContext context) throws Exception { - context.addOutput(Output.of(pluginConfig.getReferenceName(), new DMaaPMROutputFormatProvider(pluginConfig))); - } - - @Override - public void transform(StructuredRecord structuredRecord, - Emitter> emitter) throws Exception { - // get incoming message from structured record - final String incomingMessage = structuredRecord.get(pluginConfig.getMessageColumnName()); - - // if incoming messages does not have message column name log warning as it should not happen - if (incomingMessage == null) { - LOG.warn("Column Name: {}, contains no message.Skipped for DMaaP MR Publishing....", - pluginConfig.getMessageColumnName()); - } else { - - // emit the messages as key - emitter.emit(new KeyValue(incomingMessage, null)); - } - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java deleted file mode 100644 index e9afde9..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/common/PluginSchema.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.common; - -/** - * Contract interface for all DCAE Analytics Plugin Schemas - * - * @author Rajiv Singla . Creation Date: 1/25/2017. - */ -public interface PluginSchema { - - /** - * Provides column name that will be used in Schema Definition - * - * @return Column name that will be used in Schema Definition - */ - String getSchemaColumnName(); - -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java deleted file mode 100644 index b63375c..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/BaseDMaaPMRPluginConfig.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Macro; -import com.google.common.base.Objects; -import org.openecomp.dcae.apod.analytics.cdap.common.settings.CDAPBasePluginConfig; - -import javax.annotation.Nullable; - -/** - * Base class for all DMaaP MR Configs - *

- * @author Rajiv Singla . Creation Date: 1/17/2017. - */ -public abstract class BaseDMaaPMRPluginConfig extends CDAPBasePluginConfig { - - @Description("DMaaP Message Router HostName") - @Macro - protected String hostName; - - @Description("DMaaP Message Router Host Port number. Defaults to Port 80") - @Nullable - @Macro - protected Integer portNumber; - - @Description("DMaaP Message Router Topic Name") - @Macro - protected String topicName; - - @Description("DMaaP Message Router HTTP Protocol e.g. HTTP or HTTPS. Defaults to HTTPS") - @Nullable - @Macro - protected String protocol; - - @Description("DMaaP Message Router User Name used for AAF Authentication. Defaults to no authentication") - @Nullable - @Macro - protected String userName; - - @Description("DMaaP Message Router User Password used for AAF Authentication. Defaults to no authentication") - @Nullable - @Macro - protected String userPassword; - - @Description("DMaaP Message Router Content Type. Defaults to 'application/json'") - @Nullable - @Macro - protected String contentType; - - - public BaseDMaaPMRPluginConfig(final String referenceName, final String hostName, final String topicName) { - this.referenceName = referenceName; - this.hostName = hostName; - this.topicName = topicName; - } - - /** - * Host Name for DMaaP MR Publisher or Subscriber - * - * @return host name - */ - public String getHostName() { - return hostName; - } - - /** - * Port Number for DMaaP MR Publisher or Subscriber - * - * @return port number - */ - @Nullable - public Integer getPortNumber() { - return portNumber; - } - - /** - * DMaaP MR Topic Name for Subscriber or Publisher - * - * @return topic name - */ - public String getTopicName() { - return topicName; - } - - - /** - * DMaaP MR HTTP or HTTPS protocol - * - * @return http or https protocol - */ - @Nullable - public String getProtocol() { - return protocol; - } - - /** - * User name used for DMaaP MR AAF Authentication - * - * @return User name for DMaaP MR AAF Authentication - */ - @Nullable - public String getUserName() { - return userName; - } - - /** - * User password used for DMaaP MR AAF Authentication - * - * @return User password used for DMaaP MR AAF Authentication - */ - @Nullable - public String getUserPassword() { - return userPassword; - } - - /** - * Content type used for DMaaP MR Topic e.g. 'application/json' - * - * @return content type for DMaaP MR Topic - */ - @Nullable - public String getContentType() { - return contentType; - } - - @Override - public String toString() { - return Objects.toStringHelper(this) - .add("referenceName", referenceName) - .add("hostName", hostName) - .add("portNumber", portNumber) - .add("topicName", topicName) - .add("protocol", protocol) - .add("userName", userName) - .add("userPassword", "xxxx") - .add("contentType", contentType) - .toString(); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java deleted file mode 100644 index 454f384..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfig.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Macro; -import com.google.common.base.Objects; - -import javax.annotation.Nullable; - -/** - * DMaaP MR Publisher Config - *

- * @author Rajiv Singla . Creation Date: 1/17/2017. - */ -public class DMaaPMRSinkPluginConfig extends BaseDMaaPMRPluginConfig { - - private static final long serialVersionUID = 1L; - - @Description("Column name of input schema which contains the message that needs to be written to DMaaP MR Topic") - @Macro - protected String messageColumnName; - - @Description("DMaaP MR Publisher Max Batch Size. Defaults to no Batch") - @Nullable - @Macro - protected Integer maxBatchSize; - - @Description("DMaaP MR Publisher Recovery Queue Size. Default to 1000K messages which can be buffered in memory " + - "in case DMaaP MR Publisher is temporarily unavailable") - @Nullable - @Macro - protected Integer maxRecoveryQueueSize; - - // Required No Arg constructor - public DMaaPMRSinkPluginConfig() { - this(null, null, null, null); - } - - public DMaaPMRSinkPluginConfig(String referenceName, String hostName, String topicName, String messageColumnName) { - super(referenceName, hostName, topicName); - this.messageColumnName = messageColumnName; - } - - /** - * Column name of incoming Schema field that contains the message that needs to published to DMaaP MR Topic - * - * @return Column name of incoming schema which contains message that needs to published to DMaaP MR Topic - */ - public String getMessageColumnName() { - return messageColumnName; - } - - /** - * DMaaP MR Publisher Max Batch Size. - * - * @return DMaaP MR Publisher Max Batch Size - */ - @Nullable - public Integer getMaxBatchSize() { - return maxBatchSize; - } - - /** - * DMaaP MR Publisher Max Recovery Queue Size - * - * @return DMaaP MR Publisher Max Recovery Queue Size - */ - @Nullable - public Integer getMaxRecoveryQueueSize() { - return maxRecoveryQueueSize; - } - - @Override - public String toString() { - return Objects.toStringHelper(this) - .add("super", super.toString()) - .add("messageColumnName", messageColumnName) - .add("maxBatchSize", maxBatchSize) - .add("maxRecoveryQueueSize", maxRecoveryQueueSize) - .toString(); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java deleted file mode 100644 index d3e966b..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfig.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Macro; -import com.google.common.base.Objects; - -import javax.annotation.Nullable; - -/** - * DMaaP MR Subscriber Config - *

- * @author Rajiv Singla . Creation Date: 1/17/2017. - */ -public class DMaaPMRSourcePluginConfig extends BaseDMaaPMRPluginConfig { - - private static final long serialVersionUID = 1L; - - @Description("DMaaP MR Polling Interval in MS") - @Macro - protected Integer pollingInterval; - - @Description("DMaaP Message Router Subscriber Consumer ID. Defaults to some randomly created userID") - @Nullable - @Macro - protected String consumerId; - - @Description("DMaaP Message Router Subscriber Consumer Group. Defaults to some randomly created user Group") - @Nullable - @Macro - protected String consumerGroup; - - @Description("DMaaP Message Router Subscriber Timeout in MS. Defaults to no timeout") - @Nullable - @Macro - protected Integer timeoutMS; - - @Description("DMaaP Message Router Subscriber Message Limit. Defaults to no message limit") - @Nullable - @Macro - protected Integer messageLimit; - - // Required No Arg constructor - public DMaaPMRSourcePluginConfig() { - this(null, null, null, 0); - } - - public DMaaPMRSourcePluginConfig(String referenceName, String hostName, String topicName, Integer pollingInterval) { - super(referenceName, hostName, topicName); - this.pollingInterval = pollingInterval; - } - - /** - * DMaaP MR Subscriber Polling interval - * - * @return DMaaP MR Subscriber Polling interval - */ - public Integer getPollingInterval() { - return pollingInterval; - } - - /** - * DMaaP MR Subscriber Consumer ID - * - * @return DMaaP MR Subscriber Consumer ID - */ - @Nullable - public String getConsumerId() { - return consumerId; - } - - /** - * DMaaP MR Subscriber Consumer Group - * - * @return DMaaP MR Subscriber Consumer Group - */ - @Nullable - public String getConsumerGroup() { - return consumerGroup; - } - - /** - * DMaaP MR Subscriber Timeout in MS - * - * @return DMaaP MR Subscriber Timeout in MS - */ - @Nullable - public Integer getTimeoutMS() { - return timeoutMS; - } - - /** - * DMaaP MR Subscriber message limit - * - * @return DMaaP MR Subscriber Message limit - */ - @Nullable - public Integer getMessageLimit() { - return messageLimit; - } - - - @Override - public String toString() { - return Objects.toStringHelper(this) - .add("super", super.toString()) - .add("pollingInterval", pollingInterval) - .add("consumerId", consumerId) - .add("consumerGroup", consumerGroup) - .add("timeoutMS", timeoutMS) - .add("messageLimit", messageLimit) - .toString(); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java deleted file mode 100644 index d8a224d..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/JsonPathFilterPluginConfig.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Macro; -import co.cask.cdap.api.annotation.Name; -import com.google.common.base.Objects; -import org.openecomp.dcae.apod.analytics.cdap.common.settings.CDAPBasePluginConfig; - -/** - * Configuration for Json Path Filter Plugin - * - * @author Rajiv Singla . Creation Date: 3/2/2017. - */ -public class JsonPathFilterPluginConfig extends CDAPBasePluginConfig { - - private static final long serialVersionUID = 1L; - - @Name("incomingJsonFieldName") - @Description("Input schema field name that contain JSON used for filtering") - @Macro - protected String incomingJsonFieldName; - - - @Name("outputSchemaFieldName") - @Description("Name of the nullable boolean schema field name that will contain result of the filter matching") - @Macro - protected String outputSchemaFieldName; - - - @Name("jsonFilterMappings") - @Macro - @Description("Filters incoming JSON based on given filter mappings - in terms of JSON path and expected values." + - "Right hand side contains JSON path. Left hand side contains semicolon (';') separated expected values " + - "for that JSON Path. If all provided JSON Path mappings and corresponding values matches - " + - "output schema field will be marked as true") - protected String jsonFilterMappings; - - - @Name("schema") - @Description("Output Schema") - protected String schema; - - - public JsonPathFilterPluginConfig(final String referenceName, final String incomingJsonFieldName, - final String outputSchemaFieldName, final String jsonFilterMappings, - final String schema) { - this.referenceName = referenceName; - this.incomingJsonFieldName = incomingJsonFieldName; - this.outputSchemaFieldName = outputSchemaFieldName; - this.jsonFilterMappings = jsonFilterMappings; - this.schema = schema; - } - - /** - * Provides incoming plugin schema field name which contains json used to apply filter - * - * @return name of incoming schema field containing JSON to be filtered - */ - public String getIncomingJsonFieldName() { - return incomingJsonFieldName; - } - - /** - * Provides plugin output schema filed name that will contain result of filter application - * It must be nullable and boolean type - * - * @return name of outgoing schema filed name that will contain filtering result - */ - public String getOutputSchemaFieldName() { - return outputSchemaFieldName; - } - - /** - * Provides JSON filter mappings. LHS contains JSON path value and RHS contains expected - * values separated by semicolon - * - * - * @return String for JSON filter mappings - */ - public String getJsonFilterMappings() { - return jsonFilterMappings; - } - - /** - * Output Schema - * - * @return output schema string - */ - public String getSchema() { - return schema; - } - - - @Override - public String toString() { - return Objects.toStringHelper(this) - .add("referenceName", referenceName) - .add("incomingJsonFieldName", incomingJsonFieldName) - .add("outputSchemaFieldName", outputSchemaFieldName) - .add("jsonFilterMappings", jsonFilterMappings) - .add("schema", schema) - .toString(); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java deleted file mode 100644 index 4cdba6a..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/SimpleTCAPluginConfig.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Macro; -import com.google.common.base.Objects; -import org.openecomp.dcae.apod.analytics.cdap.common.settings.CDAPBasePluginConfig; - -import javax.annotation.Nullable; - -/** - * Simple TCA Plugin Configuration - *

- * @author Rajiv Singla . Creation Date: 2/13/2017. - */ -public class SimpleTCAPluginConfig extends CDAPBasePluginConfig { - - private static final long serialVersionUID = 1L; - - @Description("Field name containing VES Message") - @Macro - protected String vesMessageFieldName; - - @Description("Policy JSON that need to be applied to VES Message") - @Macro - protected String policyJson; - - @Description("Name of the output field that will contain the alert") - @Macro - protected String alertFieldName; - - @Description("Name of the output field that will contain message type: INAPPLICABLE, COMPLIANT, NON_COMPLIANT") - @Macro - protected String messageTypeFieldName; - - @Description("Specifies the output schema") - protected String schema; - - @Description("Enables") - @Nullable - @Macro - protected Boolean enableAlertCEFFormat; - - - /** - * Creates an instance of TCA Plugin Configs - * - * @param vesMessageFieldName Ves message field name from incoming plugin schema - * @param policyJson TCA Policy Json String - * @param alertFieldName Alert field name that will be added in TCA plugin output schema - * @param messageTypeFieldName Message type field name that will be added in TCA plugin output schema - * @param schema TCA Plugin output schema - * @param enableAlertCEFFormat enables alert message to be formatted in VES format - */ - public SimpleTCAPluginConfig(final String vesMessageFieldName, final String policyJson, - final String alertFieldName, final String messageTypeFieldName, - final String schema, final Boolean enableAlertCEFFormat) { - this.vesMessageFieldName = vesMessageFieldName; - this.policyJson = policyJson; - this.alertFieldName = alertFieldName; - this.messageTypeFieldName = messageTypeFieldName; - this.schema = schema; - this.enableAlertCEFFormat = enableAlertCEFFormat; - } - - /** - * Name of the field containing VES Message - * - * @return VES Message field name - */ - public String getVesMessageFieldName() { - return vesMessageFieldName; - } - - /** - * Policy Json String - * - * @return Policy Json String - */ - public String getPolicyJson() { - return policyJson; - } - - - /** - * Alert Field name in outgoing schema - * - * @return alert field name in outgoing schema - */ - public String getAlertFieldName() { - return alertFieldName; - } - - /** - * Returns output schema string - * - * @return output schema string - */ - public String getSchema() { - return schema; - } - - /** - * Return TCA message type - INAPPLICABLE, COMPLIANT, NON_COMPLIANT - * - * @return tca message type - */ - public String getMessageTypeFieldName() { - return messageTypeFieldName; - } - - - /** - * Returns if Alert output in Common Event format - * - * @return true if alert output is in common event format - */ - @Nullable - public Boolean getEnableAlertCEFFormat() { - return enableAlertCEFFormat; - } - - @Override - public String toString() { - return Objects.toStringHelper(this) - .add("referenceName", referenceName) - .add("vesMessageFieldName", vesMessageFieldName) - .add("policyJson", policyJson) - .add("alertFieldName", alertFieldName) - .add("messageTypeFieldName", messageTypeFieldName) - .add("schema", schema) - .add("enableAlertCEFFormat", true) - .toString(); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java deleted file mode 100644 index a3234c0..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchema.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap; - -import co.cask.cdap.api.data.schema.Schema; -import org.openecomp.dcae.apod.analytics.cdap.plugins.common.PluginSchema; - -/** - * Output Schema for DMaaP MR Source Plugin - * - * @author Rajiv Singla . Creation Date: 1/25/2017. - */ -public enum DMaaPSourceOutputSchema implements PluginSchema { - - TIMESTAMP("ts"), - RESPONSE_CODE("responseCode"), - RESPONSE_MESSAGE("responseMessage"), - FETCHED_MESSAGE("message"); - - private String schemaColumnName; - - DMaaPSourceOutputSchema(String schemaColumnName) { - this.schemaColumnName = schemaColumnName; - } - - @Override - public String getSchemaColumnName() { - return schemaColumnName; - } - - public static Schema getSchema() { - return Schema.recordOf( - "DMaaPMRSourcePluginResponse", - Schema.Field.of(TIMESTAMP.getSchemaColumnName(), Schema.of(Schema.Type.LONG)), - Schema.Field.of(RESPONSE_CODE.getSchemaColumnName(), Schema.of(Schema.Type.INT)), - Schema.Field.of(RESPONSE_MESSAGE.getSchemaColumnName(), Schema.of(Schema.Type.STRING)), - Schema.Field.of(FETCHED_MESSAGE.getSchemaColumnName(), Schema.of(Schema.Type.STRING)) - ); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java deleted file mode 100644 index fb5fef5..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPlugin.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.sparkcompute.tca; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Name; -import co.cask.cdap.api.annotation.Plugin; -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.format.StructuredRecord.Builder; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.StageMetrics; -import co.cask.cdap.etl.api.batch.SparkCompute; -import co.cask.cdap.etl.api.batch.SparkExecutionPluginContext; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.function.Function; -import org.openecomp.dcae.apod.analytics.cdap.common.CDAPMetricsConstants; -import org.openecomp.dcae.apod.analytics.cdap.common.persistance.tca.TCACalculatorMessageType; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.validator.SimpleTCAPluginConfigValidator; -import org.openecomp.dcae.apod.analytics.model.domain.policy.tca.MetricsPerEventName; -import org.openecomp.dcae.apod.analytics.model.domain.policy.tca.TCAPolicy; -import org.openecomp.dcae.apod.analytics.model.domain.policy.tca.Threshold; -import org.openecomp.dcae.apod.analytics.tca.processor.TCACEFJsonProcessor; -import org.openecomp.dcae.apod.analytics.tca.processor.TCACEFProcessorContext; -import org.openecomp.dcae.apod.analytics.tca.utils.TCAUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author Rajiv Singla . Creation Date: 2/13/2017. - */ - -@Plugin(type = SparkCompute.PLUGIN_TYPE) -@Name("SimpleTCAPlugin") -@Description("Used to create TCA (Threshold Crossing Alert) based on given Policy") -@SuppressFBWarnings("SE_INNER_CLASS") -public class SimpleTCAPlugin extends SparkCompute { - - private static final Logger LOG = LoggerFactory.getLogger(SimpleTCAPlugin.class); - private static final long serialVersionUID = 1L; - - private final SimpleTCAPluginConfig pluginConfig; - - /** - * Create an instance of Simple TCA Plugin with give Simple TCA Plugin Config - * - * @param pluginConfig Simple TCA Plugin Config - */ - public SimpleTCAPlugin(SimpleTCAPluginConfig pluginConfig) { - this.pluginConfig = pluginConfig; - LOG.info("Creating instance of Simple TCA Plugin with plugin config: {}", pluginConfig); - } - - @Override - public void configurePipeline(PipelineConfigurer pipelineConfigurer) { - super.configurePipeline(pipelineConfigurer); - ValidationUtils.validateSettings(pluginConfig, new SimpleTCAPluginConfigValidator()); - final Schema inputSchema = pipelineConfigurer.getStageConfigurer().getInputSchema(); - CDAPPluginUtils.validateSchemaContainsFields(inputSchema, pluginConfig.getVesMessageFieldName()); - CDAPPluginUtils.setOutputSchema(pipelineConfigurer, pluginConfig.getSchema()); - } - - @Override - public JavaRDD transform(final SparkExecutionPluginContext context, - final JavaRDD input) throws Exception { - final StageMetrics metrics = context.getMetrics(); - - LOG.debug("Invoking Spark Transform for Simple TCA Plugin"); - return input.map(new Function() { - - @Override - public StructuredRecord call(StructuredRecord inputStructuredRecord) throws Exception { - TCACalculatorMessageType calculatorMessageType; - String alertMessage = null; - - // Get input structured record - final String cefMessage = inputStructuredRecord.get(pluginConfig.getVesMessageFieldName()); - - // Get TCA Policy - final TCAPolicy tcaPolicy = CDAPPluginUtils.readValue(pluginConfig.getPolicyJson(), TCAPolicy.class); - - // create initial processor context - final TCACEFProcessorContext initialProcessorContext = - new TCACEFProcessorContext(cefMessage, tcaPolicy); - - final TCACEFJsonProcessor jsonProcessor = new TCACEFJsonProcessor(); - final TCACEFProcessorContext jsonProcessorContext = - jsonProcessor.processMessage(initialProcessorContext); - - if (jsonProcessorContext.getCEFEventListener() != null) { - - LOG.debug("Json to CEF parsing successful. Parsed object {}", - jsonProcessorContext.getCEFEventListener()); - - // compute violations - final TCACEFProcessorContext processorContextWithViolations = - TCAUtils.computeThresholdViolations(jsonProcessorContext); - - // if violation are found then create alert message - if (processorContextWithViolations.canProcessingContinue()) { - - alertMessage = TCAUtils.createTCAAlertString(processorContextWithViolations, - pluginConfig.getReferenceName(), pluginConfig.getEnableAlertCEFFormat()); - calculatorMessageType = TCACalculatorMessageType.NON_COMPLIANT; - - LOG.debug("VES Threshold Violation Detected.An alert message is be generated: {}", - alertMessage); - - final MetricsPerEventName metricsPerEventName = - processorContextWithViolations.getMetricsPerEventName(); - if (metricsPerEventName != null - && metricsPerEventName.getThresholds() != null - && metricsPerEventName.getThresholds().get(0) != null) { - final Threshold violatedThreshold = metricsPerEventName.getThresholds().get(0); - LOG.debug("CEF Message: {}, Violated Threshold: {}", cefMessage, violatedThreshold); - } - - metrics.count(CDAPMetricsConstants.TCA_VES_NON_COMPLIANT_MESSAGES_METRIC, 1); - - } else { - LOG.debug("No Threshold Violation Detected. No alert will be generated."); - calculatorMessageType = TCACalculatorMessageType.COMPLIANT; - metrics.count(CDAPMetricsConstants.TCA_VES_COMPLIANT_MESSAGES_METRIC, 1); - } - - } else { - LOG.info("Unable to parse provided json message to CEF format. Invalid message: {}", cefMessage); - calculatorMessageType = TCACalculatorMessageType.INAPPLICABLE; - } - - LOG.debug("Calculator message type: {} for message: {}", calculatorMessageType, cefMessage); - - final Schema outputSchema = Schema.parseJson(pluginConfig.getSchema()); - - // create new output record builder and copy any input record values to output record builder - final Builder outputRecordBuilder = - CDAPPluginUtils.createOutputStructuredRecordBuilder(outputSchema, inputStructuredRecord); - - // add alert field - final Builder outputRecordBuilderWithAlertField = - CDAPPluginUtils.addFieldValueToStructuredRecordBuilder(outputRecordBuilder, - outputSchema, pluginConfig.getAlertFieldName(), alertMessage); - - // add message field type - final Builder outRecordBuilderWithMessageTypeField = - CDAPPluginUtils.addFieldValueToStructuredRecordBuilder(outputRecordBuilderWithAlertField, - outputSchema, pluginConfig.getMessageTypeFieldName(), calculatorMessageType.toString()); - - return outRecordBuilderWithMessageTypeField.build(); - } - }); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java deleted file mode 100644 index 9822768..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiver.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.metrics.Metrics; -import com.google.common.base.Optional; -import org.apache.spark.storage.StorageLevel; -import org.apache.spark.streaming.receiver.Receiver; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.DMaaPMRUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.DMaaPSourceConfigMapper; -import org.openecomp.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; -import org.openecomp.dcae.apod.analytics.dmaap.DMaaPMRFactory; -import org.openecomp.dcae.apod.analytics.dmaap.service.subscriber.DMaaPMRSubscriber; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * DMaaP MR Receiver which calls DMaaP MR Topic and stores structured records - *

- * @author Rajiv Singla . Creation Date: 1/19/2017. - */ -public class DMaaPMRReceiver extends Receiver { - - private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRReceiver.class); - private static final long serialVersionUID = 1L; - - private final DMaaPMRSourcePluginConfig pluginConfig; - private final transient Metrics metrics; - - public DMaaPMRReceiver(final StorageLevel storageLevel, final DMaaPMRSourcePluginConfig pluginConfig, - final Metrics metrics) { - super(storageLevel); - this.pluginConfig = pluginConfig; - this.metrics = metrics; - LOG.debug("Created DMaaP MR Receiver instance with plugin Config: {}", pluginConfig); - } - - @Override - public void onStart() { - - // create DMaaP MR Subscriber - final DMaaPMRSubscriber subscriber = - DMaaPMRFactory.create().createSubscriber(DMaaPSourceConfigMapper.map(pluginConfig)); - - // Start a new thread with indefinite loop until receiver is stopped - new Thread() { - @Override - public void run() { - while (!isStopped()) { - storeStructuredRecords(subscriber); - try { - final Integer pollingInterval = pluginConfig.getPollingInterval(); - LOG.debug("DMaaP MR Receiver sleeping for polling interval: {}", pollingInterval); - TimeUnit.MILLISECONDS.sleep(pollingInterval); - } catch (InterruptedException e) { - final String errorMessage = String.format( - "Interrupted Exception while DMaaP MR Receiver sleeping polling interval: %s", e); - throw new DCAEAnalyticsRuntimeException(errorMessage, LOG, e); - } - } - } - }.start(); - - } - - @Override - public void onStop() { - LOG.debug("Stopping DMaaP MR Receiver with plugin config: {}", pluginConfig); - } - - /** - * Fetches records from DMaaP MR Subscriber and store them as structured records - * - * @param subscriber DMaaP MR Subscriber Instance - */ - public void storeStructuredRecords(final DMaaPMRSubscriber subscriber) { - - LOG.debug("DMaaP MR Receiver start fetching messages from DMaaP MR Topic"); - - // Fetch messages from DMaaP MR Topic - final Optional> subscriberMessagesOptional = - DMaaPMRUtils.getSubscriberMessages(subscriber, metrics); - - // store records - if (subscriberMessagesOptional.isPresent()) { - final List messages = subscriberMessagesOptional.get(); - for (final String message : messages) { - store(CDAPPluginUtils.createDMaaPMRResponseStructuredRecord(message)); - } - LOG.debug("Stored DMaaP Subscriber messages as Structured Records. Message count {}", messages.size()); - } - } - -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java deleted file mode 100644 index 117c76e..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSource.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Name; -import co.cask.cdap.api.annotation.Plugin; -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.streaming.StreamingContext; -import co.cask.cdap.etl.api.streaming.StreamingSource; -import org.apache.spark.storage.StorageLevel; -import org.apache.spark.streaming.api.java.JavaDStream; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap.DMaaPSourceOutputSchema; -import org.openecomp.dcae.apod.analytics.cdap.plugins.validator.DMaaPMRSourcePluginConfigValidator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * DMaaP MR Source Plugin which polls DMaaP MR topic at frequent intervals - *

- * @author Rajiv Singla . Creation Date: 1/18/2017. - */ -@Plugin(type = StreamingSource.PLUGIN_TYPE) -@Name("DMaaPMRSource") -@Description("Fetches DMaaP MR Messages at regular intervals") -public class DMaaPMRSource extends StreamingSource { - - private static final Logger LOG = LoggerFactory.getLogger(DMaaPMRSource.class); - private static final long serialVersionUID = 1L; - - private final DMaaPMRSourcePluginConfig pluginConfig; - - public DMaaPMRSource(final DMaaPMRSourcePluginConfig pluginConfig) { - LOG.debug("Creating DMaaP MR Source plugin with plugin Config: {}", pluginConfig); - this.pluginConfig = pluginConfig; - } - - @Override - public void configurePipeline(PipelineConfigurer pipelineConfigurer) { - ValidationUtils.validateSettings(pluginConfig, new DMaaPMRSourcePluginConfigValidator()); - pipelineConfigurer.getStageConfigurer().setOutputSchema(DMaaPSourceOutputSchema.getSchema()); - } - - @Override - public JavaDStream getStream(final StreamingContext streamingContext) throws Exception { - return streamingContext.getSparkStreamingContext().receiverStream( - new DMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), pluginConfig, streamingContext.getMetrics())); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java deleted file mode 100644 index dc24ca0..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiver.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.data.format.StructuredRecord; -import com.fasterxml.jackson.core.type.TypeReference; -import org.apache.spark.storage.StorageLevel; -import org.apache.spark.streaming.receiver.Receiver; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.DMaaPSourceConfigMapper; -import org.openecomp.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; -import org.openecomp.dcae.apod.analytics.dmaap.DMaaPMRFactory; -import org.openecomp.dcae.apod.analytics.dmaap.service.subscriber.DMaaPMRSubscriber; -import org.openecomp.dcae.apod.analytics.model.domain.cef.EventListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.openecomp.dcae.apod.analytics.model.util.AnalyticsModelJsonUtils.readValue; -import static org.openecomp.dcae.apod.analytics.model.util.AnalyticsModelJsonUtils.writeValueAsString; - -/** - * DMaaP MR Receiver which calls DMaaP MR Topic and stores structured records - *

- * @author Rajiv Singla . Creation Date: 1/19/2017. - */ -public class MockDMaaPMRReceiver extends Receiver { - - private static final Logger LOG = LoggerFactory.getLogger(MockDMaaPMRReceiver.class); - private static final long serialVersionUID = 1L; - - private static final String MOCK_MESSAGE_FILE_LOCATION = "ves_mock_messages.json"; - private static final TypeReference> EVENT_LISTENER_TYPE_REFERENCE = - new TypeReference>() { - }; - - private final DMaaPMRSourcePluginConfig pluginConfig; - - public MockDMaaPMRReceiver(final StorageLevel storageLevel, final DMaaPMRSourcePluginConfig pluginConfig) { - super(storageLevel); - this.pluginConfig = pluginConfig; - LOG.debug("Created DMaaP MR Receiver instance with plugin Config: {}", pluginConfig); - } - - @Override - public void onStart() { - - // create DMaaP MR Subscriber - final DMaaPMRSubscriber subscriber = - DMaaPMRFactory.create().createSubscriber(DMaaPSourceConfigMapper.map(pluginConfig)); - storeStructuredRecords(subscriber); - - } - - @Override - public void onStop() { - LOG.debug("Stopping DMaaP MR Receiver with plugin config: {}", pluginConfig); - } - - /** - * Fetches records from DMaaP MR Subscriber and store them as structured records - * - * @param subscriber DMaaP MR Subscriber Instance - */ - public void storeStructuredRecords(final DMaaPMRSubscriber subscriber) { - - LOG.debug("DMaaP MR Receiver start fetching messages from DMaaP MR Topic"); - - try (InputStream resourceAsStream = - Thread.currentThread().getContextClassLoader().getResourceAsStream(MOCK_MESSAGE_FILE_LOCATION)) { - - if (resourceAsStream == null) { - LOG.error("Unable to find file at location: {}", MOCK_MESSAGE_FILE_LOCATION); - throw new DCAEAnalyticsRuntimeException("Unable to find file", LOG, new FileNotFoundException()); - } - - List eventListeners = readValue(resourceAsStream, EVENT_LISTENER_TYPE_REFERENCE); - - final int totalMessageCount = eventListeners.size(); - LOG.debug("Mock message count to be written to cdap stream: ()", totalMessageCount); - - int i = 1; - for (EventListener eventListener : eventListeners) { - if (isStopped()) { - return; - } - final String eventListenerString = writeValueAsString(eventListener); - LOG.debug("=======>> Writing message to cdap stream no: {} of {}", i, totalMessageCount); - store(CDAPPluginUtils.createDMaaPMRResponseStructuredRecord(eventListenerString)); - i++; - try { - TimeUnit.MILLISECONDS.sleep(pluginConfig.getPollingInterval()); - } catch (InterruptedException e) { - LOG.error("Error while sleeping"); - throw new DCAEAnalyticsRuntimeException("Error while sleeping", LOG, e); - } - - } - - LOG.debug("Finished writing mock messages to CDAP Stream"); - - } catch (IOException e) { - LOG.error("Error while parsing json file"); - throw new DCAEAnalyticsRuntimeException("Error while parsing mock json file", LOG, e); - } - } - -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java deleted file mode 100644 index e058fab..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSource.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Name; -import co.cask.cdap.api.annotation.Plugin; -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.streaming.StreamingContext; -import co.cask.cdap.etl.api.streaming.StreamingSource; -import org.apache.spark.storage.StorageLevel; -import org.apache.spark.streaming.api.java.JavaDStream; -import org.openecomp.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A mock implementation of DMaaP MR Receiver which sends mock ves messages - *

- * @author Rajiv Singla . Creation Date: 2/15/2017. - */ -@Plugin(type = StreamingSource.PLUGIN_TYPE) -@Name("MockDMaaPMRSource") -@Description("Fetches DMaaP MR Messages at regular intervals") -public class MockDMaaPMRSource extends StreamingSource { - - private static final Logger LOG = LoggerFactory.getLogger(MockDMaaPMRSource.class); - private static final long serialVersionUID = 1L; - - private final DMaaPMRSourcePluginConfig pluginConfig; - - public MockDMaaPMRSource(final DMaaPMRSourcePluginConfig pluginConfig) { - LOG.debug("Creating DMaaP MR Source plugin with plugin Config: {}", pluginConfig); - this.pluginConfig = pluginConfig; - } - - @Override - public void configurePipeline(PipelineConfigurer pipelineConfigurer) { - final Integer pollingInterval = pluginConfig.getPollingInterval(); - if (pollingInterval == null) { - final String errorMessage = "Polling Interval field must be present"; - throw new CDAPSettingsException(errorMessage, LOG, new IllegalArgumentException(errorMessage)); - } else { - LOG.info("Mock Message will be send every ms: {}", pollingInterval); - } - } - - @Override - public JavaDStream getStream(final StreamingContext streamingContext) throws Exception { - return streamingContext.getSparkStreamingContext().receiverStream( - new MockDMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), pluginConfig)); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java deleted file mode 100644 index 135a6c2..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilter.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.transform.filter; - -import co.cask.cdap.api.annotation.Description; -import co.cask.cdap.api.annotation.Name; -import co.cask.cdap.api.annotation.Plugin; -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.Emitter; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.Transform; -import co.cask.cdap.etl.api.TransformContext; -import com.google.common.base.Splitter; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter.JsonPathFilterPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.validator.JsonPathFilterPluginConfigValidator; -import org.openecomp.dcae.apod.analytics.common.service.filter.JsonMessageFilterProcessorContext; -import org.openecomp.dcae.apod.analytics.common.utils.MessageProcessorUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import java.util.Set; - -/** - * Json Path filter Plugin filters incoming schema field based of given json path expected values - *

- * @author Rajiv Singla . Creation Date: 3/2/2017. - */ - -@Plugin(type = Transform.PLUGIN_TYPE) -@Name("JsonPathFilter") -@Description("Filters incoming schema field based of given json path expected values") -public class JsonPathFilter extends Transform { - - private static final Logger LOG = LoggerFactory.getLogger(JsonPathFilter.class); - - private final JsonPathFilterPluginConfig pluginConfig; - private final Map> jsonFilterPathMappings; - - public JsonPathFilter(final JsonPathFilterPluginConfig pluginConfig) { - this.pluginConfig = pluginConfig; - jsonFilterPathMappings = Maps.newHashMap(); - LOG.info("Created instance of Json Path Filter Plugin with plugin config: {}", pluginConfig); - } - - - @Override - public void initialize(final TransformContext context) throws Exception { - super.initialize(context); - populateJsonFilterMapping(); - } - - @Override - public void configurePipeline(final PipelineConfigurer pipelineConfigurer) { - super.configurePipeline(pipelineConfigurer); - ValidationUtils.validateSettings(pluginConfig, new JsonPathFilterPluginConfigValidator()); - final Schema inputSchema = pipelineConfigurer.getStageConfigurer().getInputSchema(); - CDAPPluginUtils.validateSchemaContainsFields(inputSchema, pluginConfig.getIncomingJsonFieldName()); - populateJsonFilterMapping(); - CDAPPluginUtils.setOutputSchema(pipelineConfigurer, pluginConfig.getSchema()); - } - - @Override - public void transform(final StructuredRecord inputStructuredRecord, final Emitter emitter) - throws Exception { - - // get input json message - final String jsonMessage = inputStructuredRecord.get(pluginConfig.getIncomingJsonFieldName()); - - // process Json Filter Mappings - final JsonMessageFilterProcessorContext jsonMessageFilterProcessorContext = - MessageProcessorUtils.processJsonFilterMappings(jsonMessage, jsonFilterPathMappings); - - // create new output record builder and copy any input Structured record values to output record builder - final Schema outputSchema = Schema.parseJson(pluginConfig.getSchema()); - final StructuredRecord.Builder outputRecordBuilder = - CDAPPluginUtils.createOutputStructuredRecordBuilder(outputSchema, inputStructuredRecord); - - // add json filter matched field - final StructuredRecord.Builder outputRecordBuilderWithMatchedField = - CDAPPluginUtils.addFieldValueToStructuredRecordBuilder(outputRecordBuilder, - outputSchema, pluginConfig.getOutputSchemaFieldName(), - jsonMessageFilterProcessorContext.getMatched()); - - // emit structured record with filtering matched field - final StructuredRecord outputStructuredRecord = outputRecordBuilderWithMatchedField.build(); - - LOG.debug("Incoming Json Message: {}.Json Path Filter Output Matched Field: {}", jsonMessage, - outputStructuredRecord.get(pluginConfig.getOutputSchemaFieldName())); - - emitter.emit(outputStructuredRecord); - - } - - /** - * Populates Json Filter Mapping - */ - private void populateJsonFilterMapping() { - final Map fieldMappings = - CDAPPluginUtils.extractFieldMappings(pluginConfig.getJsonFilterMappings()); - if (fieldMappings.isEmpty()) { - throw new IllegalArgumentException("No Field Mapping found. Invalid Filter mapping configuration"); - } - final Splitter semiColonSplitter = Splitter.on(";"); - for (Map.Entry fieldMappingEntry : fieldMappings.entrySet()) { - jsonFilterPathMappings.put(fieldMappingEntry.getKey(), - Sets.newLinkedHashSet(semiColonSplitter.split(fieldMappingEntry.getValue()))); - } - LOG.info("Input Json Filter Mappings: {}", jsonFilterPathMappings); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java deleted file mode 100644 index 3ae1560..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtils.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.utils; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.PipelineConfigurer; -import com.google.common.base.Function; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.commons.lang3.StringUtils; -import org.openecomp.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap.DMaaPSourceOutputSchema; -import org.openecomp.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; -import org.openecomp.dcae.apod.analytics.model.util.AnalyticsModelJsonUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -/** - * @author Rajiv Singla . Creation Date: 1/26/2017. - */ -public abstract class CDAPPluginUtils extends AnalyticsModelJsonUtils { - - private static final Logger LOG = LoggerFactory.getLogger(CDAPPluginUtils.class); - - public static final Function SCHEMA_TO_TYPE_FUNCTION = new Function() { - @Override - public Schema.Type apply(@Nonnull Schema schema) { - return schema.getType(); - } - }; - - - - private CDAPPluginUtils() { - // private constructor - } - - /** - * Validates if CDAP Schema contains expected fields - * - * @param schema schema that need to be validated - * @param expectedFields fields that are expected to be in the schema - */ - - public static void validateSchemaContainsFields(@Nullable final Schema schema, final String... expectedFields) { - - LOG.debug("Validating schema:{} contains expected fields:{}", schema, Arrays.toString(expectedFields)); - - if (schema == null) { - // If input schema is null then no validation possible - LOG.warn("Input Schema is null. No validation possible"); - } else { - // Check if expected fields are indeed present in the schema - for (String expectedField : expectedFields) { - final Schema.Field schemaField = schema.getField(expectedField); - if (schemaField == null) { - final String errorMessage = String.format( - "Unable to find expected field: %s, in schema: %s", expectedField, schema); - throw new CDAPSettingsException(errorMessage, LOG, new IllegalArgumentException(errorMessage)); - } - } - LOG.debug("Successfully validated schema:{}, contains expected fields:{}", schema, - Arrays.toString(expectedFields)); - } - } - - - /** - * Creates a new Structured Record containing DMaaP MR fetched message - * - * @param message DMaaP MR fetch message - * - * @return Structured record containing DMaaP MR Message - */ - public static StructuredRecord createDMaaPMRResponseStructuredRecord(final String message) { - StructuredRecord.Builder recordBuilder = StructuredRecord.builder(DMaaPSourceOutputSchema.getSchema()); - recordBuilder - .set(DMaaPSourceOutputSchema.TIMESTAMP.getSchemaColumnName(), System.nanoTime()) - .set(DMaaPSourceOutputSchema.RESPONSE_CODE.getSchemaColumnName(), 200) - .set(DMaaPSourceOutputSchema.RESPONSE_MESSAGE.getSchemaColumnName(), "OK") - .set(DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName(), message); - return recordBuilder.build(); - } - - - /** - * Creates output StructuredRecord Builder which has copied values from input StructuredRecord - * - * @param outputSchema output Schema - * @param inputStructuredRecord input Structured Record - * - * @return output Structured Record builder with pre populated values from input structured record - */ - public static StructuredRecord.Builder createOutputStructuredRecordBuilder( - @Nonnull final Schema outputSchema, - @Nonnull final StructuredRecord inputStructuredRecord) { - - // Get input structured Record Schema - final Schema inputSchema = inputStructuredRecord.getSchema(); - // Create new instance of output Structured Record Builder from output Schema - final StructuredRecord.Builder outputStructuredRecordBuilder = StructuredRecord.builder(outputSchema); - - // iterate over input fields and if output schema has field with same name copy the value to out record builder - for (Schema.Field inputField : inputSchema.getFields()) { - final String inputFieldName = inputField.getName(); - if (outputSchema.getField(inputFieldName) != null) { - outputStructuredRecordBuilder.set(inputFieldName, inputStructuredRecord.get(inputFieldName)); - } - } - - return outputStructuredRecordBuilder; - } - - - /** - * Adds Field value to StructuredRecord Builder if schema contains that field Name - * - * @param structuredRecordBuilder structured record builder - * @param structuredRecordSchema schema for structured record builder - * @param fieldName field name - * @param fieldValue field value - * - * @return structured record builder with populated field name and value if schema contains field name - */ - public static StructuredRecord.Builder addFieldValueToStructuredRecordBuilder( - @Nonnull final StructuredRecord.Builder structuredRecordBuilder, - @Nonnull final Schema structuredRecordSchema, - @Nonnull final String fieldName, - final Object fieldValue) { - - // check if schema contains field Name - if (structuredRecordSchema.getField(fieldName) != null) { - structuredRecordBuilder.set(fieldName, fieldValue); - } else { - LOG.info("Unable to populate value for field Name: {} with field value: {}. " + - "Schema Fields: {} does not contain field name: {}", - fieldName, fieldValue, structuredRecordSchema.getFields(), fieldName); - } - - return structuredRecordBuilder; - } - - - /** - * Validates that given schema String has fieldName of expected type. If field does not exist in given schema - * then validation will pass with warning. If field does exist in given schema then this validation will return - * true if field type is same as expected type else false - * - * @param schemaString CDAP Plugin output or input schema string - * @param fieldName field name - * @param expectedFieldType expected schema field type - * - * @return true if field type matches expected field type else false. If field does not exist in - * give schema validation will pass but will generate a warning message - */ - public static boolean validateSchemaFieldType(@Nonnull final String schemaString, - @Nonnull final String fieldName, - @Nonnull final Schema.Type expectedFieldType) { - - try { - // parse given schema String - final Schema outputSchema = Schema.parseJson(schemaString); - final Schema.Field schemaField = outputSchema.getField(fieldName); - - // if given schema does contain field then validated fieldName type - if (schemaField != null) { - - final List schemas = new LinkedList<>(); - - // if it is a union type then grab all union schemas - if (outputSchema.getField(fieldName).getSchema().getType() == Schema.Type.UNION) { - final List unionFieldSchemas = - outputSchema.getField(fieldName).getSchema().getUnionSchemas(); - schemas.addAll(unionFieldSchemas); - } else { - // if not union type the just get the field schema - final Schema fieldSchema = outputSchema.getField(fieldName).getSchema(); - schemas.add(fieldSchema); - } - - // get all schema types - final List fieldTypes = - Lists.transform(schemas, CDAPPluginUtils.SCHEMA_TO_TYPE_FUNCTION); - - // if all schema types does not contain expected field type then return false - if (!fieldTypes.contains(expectedFieldType)) { - LOG.error("Validation failed for fieldName: {} is NOT of expected Type: {} in schema: {}", - fieldName, expectedFieldType, outputSchema); - return false; - } - - // field type validation passed - LOG.debug("Successfully validated fieldName: {} is of expected Type: {}", - fieldName, expectedFieldType); - - return true; - - } else { - - // if field does not exist then the validation will pass but will generate warning message - LOG.warn("Validation of field type not possible. Field name: {} does not exist in schema: {}", - fieldName, outputSchema); - return true; - } - - } catch (IOException e) { - final String errorMessage = - String.format("Unable to parse schema: %s for field type validation. " + - "Field Name: %s, Expected Field Type: %s Exception: %s", - schemaString, fieldName, expectedFieldType, e); - throw new DCAEAnalyticsRuntimeException(errorMessage, LOG, e); - } - - } - - - /** - * Parses provided schema String as Schema object and set it as output Schema format - * - * @param pipelineConfigurer plugin pipeline configurer - * @param schemaString schema String to be set as output schema - */ - public static void setOutputSchema(final PipelineConfigurer pipelineConfigurer, final String schemaString) { - try { - final Schema outputSchema = Schema.parseJson(schemaString); - pipelineConfigurer.getStageConfigurer().setOutputSchema(outputSchema); - } catch (IOException e) { - final String errorMessage = String.format( - "Schema specified is not a valid JSON. Schema String: %s, Exception: %s", schemaString, e); - throw new CDAPSettingsException(errorMessage, LOG, new IllegalArgumentException(errorMessage)); - } - } - - - /** - * Parses incoming plugin config mapping to key value map. If any of the key value map is blank an Illegal Argument - * exception will be thrown - * - * @param mappingFieldString field Mapping String - * - * @return map containing mapping key values - */ - public static Map extractFieldMappings(final String mappingFieldString) { - final Map fieldMappings = Maps.newHashMap(); - if (StringUtils.isNotBlank(mappingFieldString)) { - final Splitter commaSplitter = Splitter.on(","); - for (String fieldMapping : commaSplitter.split(mappingFieldString)) { - final String[] keyValueMappings = fieldMapping.split(":"); - if (keyValueMappings.length != 2 || - StringUtils.isBlank(keyValueMappings[0]) || - StringUtils.isBlank(keyValueMappings[1])) { - final String errorMessage = "Field Mapping key or value is Blank. All field mappings must " + - "be present in mappings: " + mappingFieldString; - throw new DCAEAnalyticsRuntimeException( - errorMessage, LOG, new IllegalArgumentException(errorMessage)); - } - fieldMappings.put(keyValueMappings[0].trim(), keyValueMappings[1].trim()); - } - } - return fieldMappings; - } - - - - -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java deleted file mode 100644 index 01dad7e..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapper.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.utils; - -import com.google.common.base.Function; -import org.apache.hadoop.conf.Configuration; -import org.openecomp.dcae.apod.analytics.cdap.common.CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields; -import org.openecomp.dcae.apod.analytics.dmaap.domain.config.DMaaPMRPublisherConfig; - -import javax.annotation.Nonnull; - -import static org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isEmpty; -import static org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isPresent; - -/** - * Function that converts {@link Configuration} to {@link DMaaPMRPublisherConfig} - *

- * @author Rajiv Singla . Creation Date: 1/26/2017. - */ -public class DMaaPSinkConfigMapper implements Function { - - /** - * Static method to map {@link Configuration} to {@link DMaaPMRPublisherConfig} - * - * @param sinkPluginConfig DMaaP Sink Plugin Config - * - * @return DMaaP MR Publisher Config - */ - public static DMaaPMRPublisherConfig map(final Configuration sinkPluginConfig) { - return new DMaaPSinkConfigMapper().apply(sinkPluginConfig); - } - - /** - * Converts {@link Configuration} to {@link DMaaPMRPublisherConfig} - * - * @param configuration Hadoop Configuration containing DMaaP MR Sink field values - * - * @return DMaaP MR Publisher Config - */ - @Nonnull - @Override - public DMaaPMRPublisherConfig apply(@Nonnull Configuration configuration) { - - // Create a new publisher settings builder - final String hostName = configuration.get(DMaaPMRSinkHadoopConfigFields.HOST_NAME); - final String topicName = configuration.get(DMaaPMRSinkHadoopConfigFields.TOPIC_NAME); - - if (isEmpty(hostName) || isEmpty(topicName)) { - throw new IllegalStateException("DMaaP MR Sink Host Name and Topic Name must be present"); - } - - final DMaaPMRPublisherConfig.Builder publisherConfigBuilder = - new DMaaPMRPublisherConfig.Builder(hostName, topicName); - - // Setup up any optional publisher parameters if they are present - final String portNumber = configuration.get(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER); - if (portNumber != null) { - publisherConfigBuilder.setPortNumber(Integer.parseInt(portNumber)); - } - - final String protocol = configuration.get(DMaaPMRSinkHadoopConfigFields.PROTOCOL); - if (isPresent(protocol)) { - publisherConfigBuilder.setProtocol(protocol); - } - - final String userName = configuration.get(DMaaPMRSinkHadoopConfigFields.USER_NAME); - if (isPresent(userName)) { - publisherConfigBuilder.setUserName(userName); - } - - final String userPassword = configuration.get(DMaaPMRSinkHadoopConfigFields.USER_PASS); - if (isPresent(userPassword)) { - publisherConfigBuilder.setUserPassword(userPassword); - } - - final String contentType = configuration.get(DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE); - if (isPresent(contentType)) { - publisherConfigBuilder.setContentType(contentType); - } - - final String maxBatchSize = configuration.get(DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE); - if (maxBatchSize != null) { - publisherConfigBuilder.setMaxBatchSize(Integer.parseInt(maxBatchSize)); - } - - final String maxRecoveryQueueSize = configuration.get(DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE); - if (maxRecoveryQueueSize != null) { - publisherConfigBuilder.setMaxRecoveryQueueSize(Integer.parseInt(maxRecoveryQueueSize)); - } - - return publisherConfigBuilder.build(); - - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java deleted file mode 100644 index 2ae09be..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapper.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.utils; - -import com.google.common.base.Function; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.dmaap.domain.config.DMaaPMRSubscriberConfig; - -import javax.annotation.Nonnull; - -import static org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isEmpty; -import static org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils.isPresent; - -/** - * Function that converts {@link DMaaPMRSourcePluginConfig} to {@link DMaaPMRSubscriberConfig} - *

- * @author Rajiv Singla . Creation Date: 1/18/2017. - */ -public class DMaaPSourceConfigMapper implements Function { - - /** - * Static factory method to map {@link DMaaPMRSourcePluginConfig} to {@link DMaaPMRSubscriberConfig} - * - * @param pluginConfig DMaaP MR Souce Plugin Config - * - * @return DMaaP MR Subscriber Config - */ - public static DMaaPMRSubscriberConfig map(final DMaaPMRSourcePluginConfig pluginConfig) { - return new DMaaPSourceConfigMapper().apply(pluginConfig); - } - - /** - * Converts {@link DMaaPMRSourcePluginConfig} to {@link DMaaPMRSubscriberConfig} object - * - * @param sourcePluginConfig DMaaP MR Source Plugin Config - * - * @return DMaaP MR Subscriber Config - */ - @Nonnull - @Override - public DMaaPMRSubscriberConfig apply(@Nonnull DMaaPMRSourcePluginConfig sourcePluginConfig) { - - // Create a new subscriber settings builder - final String hostName = sourcePluginConfig.getHostName(); - final String topicName = sourcePluginConfig.getTopicName(); - if (isEmpty(hostName) || isEmpty(topicName)) { - throw new IllegalStateException("DMaaP MR Source Host Name and Topic Name must be present"); - } - final DMaaPMRSubscriberConfig.Builder subscriberConfigBuilder = new DMaaPMRSubscriberConfig.Builder( - hostName, topicName); - - // Setup up any optional subscriber parameters if they are present - final Integer subscriberHostPortNumber = sourcePluginConfig.getPortNumber(); - if (subscriberHostPortNumber != null) { - subscriberConfigBuilder.setPortNumber(subscriberHostPortNumber); - } - - final String subscriberProtocol = sourcePluginConfig.getProtocol(); - if (isPresent(subscriberProtocol)) { - subscriberConfigBuilder.setProtocol(subscriberProtocol); - } - - final String subscriberUserName = sourcePluginConfig.getUserName(); - if (isPresent(subscriberUserName)) { - subscriberConfigBuilder.setUserName(subscriberUserName); - } - - final String subscriberUserPassword = sourcePluginConfig.getUserPassword(); - if (isPresent(subscriberUserPassword)) { - subscriberConfigBuilder.setUserPassword(subscriberUserPassword); - } - - final String subscriberContentType = sourcePluginConfig.getContentType(); - if (isPresent(subscriberContentType)) { - subscriberConfigBuilder.setContentType(subscriberContentType); - } - - final String subscriberConsumerId = sourcePluginConfig.getConsumerId(); - if (isPresent(subscriberConsumerId)) { - subscriberConfigBuilder.setConsumerId(subscriberConsumerId); - } - - final String subscriberConsumerGroup = sourcePluginConfig.getConsumerGroup(); - if (isPresent(subscriberConsumerGroup)) { - subscriberConfigBuilder.setConsumerGroup(subscriberConsumerGroup); - } - - final Integer subscriberTimeoutMS = sourcePluginConfig.getTimeoutMS(); - if (subscriberTimeoutMS != null) { - subscriberConfigBuilder.setTimeoutMS(subscriberTimeoutMS); - } - final Integer subscriberMessageLimit = sourcePluginConfig.getMessageLimit(); - if (subscriberMessageLimit != null) { - subscriberConfigBuilder.setMessageLimit(subscriberMessageLimit); - } - - // return Subscriber config - return subscriberConfigBuilder.build(); - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java deleted file mode 100644 index 8cc818f..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/BaseDMaaPMRPluginConfigValidator.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.BaseDMaaPMRPluginConfig; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -/** - * Validates plugin config values which are common in DMaaP MR Configs - {@link BaseDMaaPMRPluginConfig} - *

- * @author Rajiv Singla . Creation Date: 1/23/2017. - * - * @param {@link BaseDMaaPMRPluginConfig} Sub classes - */ -public abstract class BaseDMaaPMRPluginConfigValidator implements - CDAPAppSettingsValidator> { - - private static final long serialVersionUID = 1L; - - /** - * Validates the {@link BaseDMaaPMRPluginConfig} parameters - * - * @param baseDMaaPMRPluginConfig DMaaP MR Plugin Config - * - * @return Validation Response containing validation errors if any - */ - @Override - public GenericValidationResponse validateAppSettings(final T baseDMaaPMRPluginConfig) { - - final GenericValidationResponse validationResponse = new GenericValidationResponse<>(); - - if (ValidationUtils.isEmpty(baseDMaaPMRPluginConfig.getHostName())) { - validationResponse.addErrorMessage( - "hostName", - "DMaaPMRPluginConfig - hostname field is undefined: " + baseDMaaPMRPluginConfig); - } - - if (baseDMaaPMRPluginConfig.getPortNumber() == null) { - validationResponse.addErrorMessage( - "port Number", - "DMaaPMRPluginConfig - host port number field is undefined: " + baseDMaaPMRPluginConfig); - } - - if (ValidationUtils.isEmpty(baseDMaaPMRPluginConfig.getTopicName())) { - validationResponse.addErrorMessage( - "topic Name", - "DMaaPMRSourcePluginConfig - topic name field is undefined: " + baseDMaaPMRPluginConfig); - } - - return validationResponse; - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java deleted file mode 100644 index c9b1df6..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidator.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -/** - * Validates plugin config values in {@link DMaaPMRSinkPluginConfig} - *

- * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMRSinkPluginConfigValidator extends BaseDMaaPMRPluginConfigValidator { - - private static final long serialVersionUID = 1L; - - /** - * Validates plugin config values in {@link DMaaPMRSinkPluginConfig} - * - * @param sinkPluginConfig Sink Plugin Config - * - * @return Validation response containing validation errors if any - */ - @Override - public GenericValidationResponse validateAppSettings( - final DMaaPMRSinkPluginConfig sinkPluginConfig) { - - // validate settings in BaseDMaaPMRPluginConfig - final GenericValidationResponse validationResponse = - super.validateAppSettings(sinkPluginConfig); - - if (ValidationUtils.isEmpty(sinkPluginConfig.getMessageColumnName())) { - validationResponse.addErrorMessage("messageColumn Name", - "DMaaPMRSinkPluginConfig - message column name field is undefined: " + sinkPluginConfig); - } - - return validationResponse; - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java deleted file mode 100644 index 15a7583..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidator.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -/** - * Validates plugin config values in {@link DMaaPMRSourcePluginConfig} - *

- * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMRSourcePluginConfigValidator extends BaseDMaaPMRPluginConfigValidator { - - private static final long serialVersionUID = 1L; - - /** - * Validates plugin config values in {@link DMaaPMRSourcePluginConfig} - * - * @param sourcePluginConfig Source Plugin Config - * - * @return Validation response containing validation errors if any - */ - @Override - public GenericValidationResponse validateAppSettings( - final DMaaPMRSourcePluginConfig sourcePluginConfig) { - - // validate settings in BaseDMaaPMRPluginConfig - final GenericValidationResponse validationResponse = - super.validateAppSettings(sourcePluginConfig); - - if (sourcePluginConfig.getPollingInterval() == null) { - validationResponse.addErrorMessage( - "port Number", - "DMaaPMRSourcePluginConfig - polling interval is undefined: " + sourcePluginConfig); - } - - return validationResponse; - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java deleted file mode 100644 index 428fedb..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidator.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import co.cask.cdap.api.data.schema.Schema; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter.JsonPathFilterPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -/** - * Validator to validate {@link JsonPathFilterPluginConfig} - *

- * @author Rajiv Singla . Creation Date: 3/2/2017. - */ -public class JsonPathFilterPluginConfigValidator implements CDAPAppSettingsValidator> { - - private static final long serialVersionUID = 1L; - - @Override - public GenericValidationResponse validateAppSettings( - final JsonPathFilterPluginConfig jsonPathFilterPluginConfig) { - - final GenericValidationResponse validationResponse = - new GenericValidationResponse<>(); - - final String jsonFilterMappings = jsonPathFilterPluginConfig.getJsonFilterMappings(); - if (ValidationUtils.isEmpty(jsonFilterMappings)) { - - validationResponse.addErrorMessage("JsonFilterMappings", "Json Filter Mappings must be present"); - } - - - final String matchedField = jsonPathFilterPluginConfig.getOutputSchemaFieldName(); - final String outputSchemaJson = jsonPathFilterPluginConfig.getSchema(); - - if (ValidationUtils.isEmpty(outputSchemaJson)) { - - validationResponse.addErrorMessage("output schema", "Output schema is not present"); - - } else { - - // validate matched output field type is boolean - if (matchedField != null && - !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, matchedField, Schema.Type.BOOLEAN)) { - validationResponse.addErrorMessage("OutputSchemaFieldName", - String.format( - "OutputSchemaFieldName: %s must be marked as boolean type", matchedField)); - } - - // validate matched output field type is nullable - if (matchedField != null && - !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, matchedField, Schema.Type.NULL)) { - validationResponse.addErrorMessage("OutputSchemaFieldName", - String.format( - "OutputSchemaFieldName: %s must be marked as nullable type", matchedField)); - } - - } - - return validationResponse; - } -} diff --git a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java b/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java deleted file mode 100644 index 97f3f24..0000000 --- a/dcae-analytics-cdap-plugins/src/main/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidator.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import co.cask.cdap.api.data.schema.Schema; -import org.openecomp.dcae.apod.analytics.cdap.common.utils.ValidationUtils; -import org.openecomp.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.utils.CDAPPluginUtils; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -/** - * Validator that validate {@link SimpleTCAPluginConfig} - *

- * @author Rajiv Singla . Creation Date: 2/21/2017. - */ -public class SimpleTCAPluginConfigValidator implements CDAPAppSettingsValidator> { - - private static final long serialVersionUID = 1L; - - @Override - public GenericValidationResponse validateAppSettings( - final SimpleTCAPluginConfig tcaPluginConfig) { - - final GenericValidationResponse validationResponse = new GenericValidationResponse<>(); - - if (ValidationUtils.isEmpty(tcaPluginConfig.getVesMessageFieldName())) { - validationResponse.addErrorMessage("vesMessageFieldName", - "Missing VES Message Field Name from plugin incoming schema"); - } - - if (ValidationUtils.isEmpty(tcaPluginConfig.getPolicyJson())) { - validationResponse.addErrorMessage("policyJson", - "Missing tca Policy Json"); - } - - final String alertFieldValue = tcaPluginConfig.getAlertFieldName(); - final String alertFieldName = "alertFieldName"; - if (ValidationUtils.isEmpty(alertFieldValue)) { - validationResponse.addErrorMessage(alertFieldName, - "Missing alert Field Name that will be placed in plugin outgoing schema"); - } - - if (ValidationUtils.isEmpty(tcaPluginConfig.getMessageTypeFieldName())) { - validationResponse.addErrorMessage("messageTypeField", - "Missing message Type Field Name that will be placed in plugin outgoing schema"); - } - - - final String outputSchemaJson = tcaPluginConfig.getSchema(); - if (ValidationUtils.isEmpty(outputSchemaJson)) { - validationResponse.addErrorMessage("output schema", "Output schema is not present"); - } else { - // validate output schema - alert field name is of type string - if (alertFieldValue != null && - !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, alertFieldValue, Schema.Type.STRING)) { - validationResponse.addErrorMessage(alertFieldName, - String.format( - "Alert Field Name: %s must be String type", alertFieldValue)); - } - // validate output schema - alert field name is nullable - if (alertFieldValue != null && - !CDAPPluginUtils.validateSchemaFieldType(outputSchemaJson, alertFieldValue, Schema.Type.NULL)) { - validationResponse.addErrorMessage(alertFieldName, - String.format( - "Alert Field Name: %s must be marked as nullable type", alertFieldValue)); - } - } - - return validationResponse; - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java new file mode 100644 index 0000000..e9b4e6f --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java @@ -0,0 +1,238 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins; + +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.StageMetrics; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Suppliers; +import org.apache.hadoop.conf.Configuration; +import org.onap.dcae.apod.analytics.cdap.common.CDAPPluginConstants; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSinkPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter.TestJsonPathFilterPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.TestSimpleTCAPluginConfig; +import org.onap.dcae.apod.analytics.model.util.json.AnalyticsModelObjectMapperSupplier; +import org.onap.dcae.apod.analytics.test.BaseDCAEAnalyticsUnitTest; + +import java.io.IOException; +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author Rajiv Singla . Creation Date: 1/23/2017. + */ +public abstract class BaseAnalyticsCDAPPluginsUnitTest extends BaseDCAEAnalyticsUnitTest { + + protected static final ObjectMapper ANALYTICS_MODEL_OBJECT_MAPPER = + Suppliers.memoize(new AnalyticsModelObjectMapperSupplier()).get(); + + protected static final String TCA_POLICY_JSON_FILE_LOCATION = "data/json/policy/tca_policy.json"; + protected static final String CEF_MESSAGE_JSON_FILE_LOCATION = "data/json/cef/cef_message.json"; + protected static final String CEF_NON_COMPLIANT_MESSAGE_JSON_FILE_LOCATION = + "data/json/cef/cef_message_with_threshold_violation.json"; + + + protected static final String DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME = "testDMaaPMRSource"; + protected static final String DMAAP_MR_SOURCE_PLUGIN_HOST_NAME = "dcae-msrt-mtl1-ftl.homer.com"; + protected static final Integer DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER = 3905; + protected static final String DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME = "com.dcae.dmaap.FTL.DcaeTestVESSub"; + protected static final Integer DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL = 1000; + protected static final String DMAAP_MR_SOURCE_PLUGIN_PROTOCOL = "https"; + protected static final String DMAAP_MR_SOURCE_PLUGIN_USERNAME = "username"; + protected static final String DMAAP_MR_SOURCE_PLUGIN_PASSWORD = "password"; + protected static final String DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE = "application/json"; + protected static final String DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP = "G1"; + protected static final String DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID = "C1"; + protected static final Integer DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT = 100; + protected static final Integer DMAAP_MR_SOURCE_PLUGIN_TIMEOUT = 10000; + + + protected static final String DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME = "testDMaaPMRSINK"; + protected static final String DMAAP_MR_SINK_PLUGIN_HOST_NAME = "dcae-msrt-mtl1-ftl.homer.com"; + protected static final Integer DMAAP_MR_SINK_PLUGIN_PORT_NUMBER = 3905; + protected static final String DMAAP_MR_SINK_PLUGIN_TOPIC_NAME = "com.dcae.dmaap.FTL.DcaeTestVESPub"; + protected static final String DMAAP_MR_SINK_PLUGIN_PROTOCOL = "https"; + protected static final String DMAAP_MR_SINK_PLUGIN_USERNAME = "username"; + protected static final String DMAAP_MR_SINK_PLUGIN_PASSWORD = "password"; + protected static final String DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE = "application/json"; + protected static final String DMAAP_MR_SINK_MESSAGE_COLUMN_NAME = "message"; + protected static final Integer DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE = 10; + protected static final Integer DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE = 100; + + protected static final String VES_MESSAGE_FIELD_NAME = "message"; + protected static final String TCA_PLUGIN_ALERT_FIELD_NAME = "alert"; + protected static final String TCA_PLUGIN_MESSAGE_TYPE_FIELD_NAME = "tcaMessageType"; + + + protected static final String JSON_PATH_FILTER_PLUGIN_REFERENCE_NAME = "JsonPathFilter"; + protected static final String JSON_PATH_FILTER_PLUGIN_INCOMING_JSON_FIELD_NAME = "message"; + protected static final String JSON_PATH_FILTER_PLUGIN_OUTPUT_SCHEMA_FILED_NAME = "filterMatched"; + protected static final String JSON_PATH_FILTER_PLUGIN_JSON_FILTER_MAPPINGS = + "$.event.commonEventHeader.domain:measurementsForVfScaling," + + "$.event.commonEventHeader.eventName:vLoadBalancer;vFirewall"; + protected static final String JSON_PATH_FILTER_PLUGIN_JSON_FILTER_OUTPUT_SCHEMA = + "{\"type\":\"record\"," + + "\"name\":\"etlSchemaBody\",\"fields\":" + + "[" + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"filterMatched\",\"type\":[\"boolean\",\"null\"]}," + + "{\"name\":\"responseCode\",\"type\":\"int\"}," + + "{\"name\":\"responseMessage\",\"type\":\"string\"}," + + "{\"name\":\"message\",\"type\":\"string\"}" + + "]" + + "}"; + + protected static class MockStageMetrics implements StageMetrics, Serializable { + + @Override + public void count(String metricName, int delta) { + LOG.debug("Mocking metric count, MetricName: {}, Delta: {}", metricName, delta); + } + + @Override + public void gauge(String metricName, long value) { + LOG.debug("Mocking metric guage, MetricName: {}, Value: {}", metricName, value); + } + + @Override + public void pipelineCount(String metricName, int delta) { + LOG.debug("Mocking metric pipelineCount, MetricName: {}, Delta: {}", metricName, delta); + } + + @Override + public void pipelineGauge(String metricName, long value) { + LOG.debug("Mocking metric guage, pipelineGauge: {}, Value: {}", metricName, value); + } + } + + protected static TestDMaaPMRSourcePluginConfig getTestDMaaPMRSourcePluginConfig() { + final TestDMaaPMRSourcePluginConfig sourcePluginConfig = new TestDMaaPMRSourcePluginConfig(); + sourcePluginConfig.setReferenceName(DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME); + sourcePluginConfig.setHostName(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME); + sourcePluginConfig.setPortNumber(DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER); + sourcePluginConfig.setTopicName(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME); + sourcePluginConfig.setPollingInterval(DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL); + sourcePluginConfig.setProtocol(DMAAP_MR_SOURCE_PLUGIN_PROTOCOL); + sourcePluginConfig.setUserName(DMAAP_MR_SOURCE_PLUGIN_USERNAME); + sourcePluginConfig.setUserPassword(DMAAP_MR_SOURCE_PLUGIN_PASSWORD); + sourcePluginConfig.setContentType(DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE); + sourcePluginConfig.setConsumerGroup(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP); + sourcePluginConfig.setConsumerId(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID); + sourcePluginConfig.setMessageLimit(DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT); + sourcePluginConfig.setTimeoutMS(DMAAP_MR_SOURCE_PLUGIN_TIMEOUT); + return sourcePluginConfig; + } + + protected static TestDMaaPMRSinkPluginConfig getTestDMaaPMRSinkPluginConfig() { + final TestDMaaPMRSinkPluginConfig sinkPluginConfig = new TestDMaaPMRSinkPluginConfig(); + sinkPluginConfig.setReferenceName(DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME); + sinkPluginConfig.setHostName(DMAAP_MR_SINK_PLUGIN_HOST_NAME); + sinkPluginConfig.setPortNumber(DMAAP_MR_SINK_PLUGIN_PORT_NUMBER); + sinkPluginConfig.setTopicName(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME); + sinkPluginConfig.setProtocol(DMAAP_MR_SINK_PLUGIN_PROTOCOL); + sinkPluginConfig.setUserName(DMAAP_MR_SINK_PLUGIN_USERNAME); + sinkPluginConfig.setUserPassword(DMAAP_MR_SINK_PLUGIN_PASSWORD); + sinkPluginConfig.setContentType(DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE); + sinkPluginConfig.setMessageColumnName(DMAAP_MR_SINK_MESSAGE_COLUMN_NAME); + sinkPluginConfig.setMaxBatchSize(DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE); + sinkPluginConfig.setMaxRecoveryQueueSize(DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE); + return sinkPluginConfig; + } + + + protected static Configuration getTestConfiguration() { + final Configuration configuration = new Configuration(); + final Map sinkConfigurationMap = createSinkConfigurationMap(); + for (Map.Entry property : sinkConfigurationMap.entrySet()) { + configuration.set(property.getKey(), property.getValue()); + } + return configuration; + } + + protected static Map createSinkConfigurationMap() { + + Map sinkConfig = new LinkedHashMap<>(); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.HOST_NAME, DMAAP_MR_SINK_PLUGIN_HOST_NAME); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.TOPIC_NAME, DMAAP_MR_SINK_PLUGIN_TOPIC_NAME); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.PORT_NUMBER, + DMAAP_MR_SINK_PLUGIN_PORT_NUMBER.toString()); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.PROTOCOL, DMAAP_MR_SINK_PLUGIN_PROTOCOL); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.USER_NAME, DMAAP_MR_SINK_PLUGIN_USERNAME); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.USER_PASS, DMAAP_MR_SINK_PLUGIN_PASSWORD); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE, + DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE, + DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE.toString()); + sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE, + DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE.toString()); + return sinkConfig; + } + + protected static Schema getDMaaPMRSinkTestSchema() { + return Schema.recordOf( + "DMaaPMRSinkTestSchema", + Schema.Field.of("message", Schema.of(Schema.Type.STRING)), + Schema.Field.of("field1", Schema.of(Schema.Type.STRING)) + ); + } + + + protected static TestSimpleTCAPluginConfig getTestSimpleTCAPluginConfig() { + final String policyJson; + try { + policyJson = fromStream(TCA_POLICY_JSON_FILE_LOCATION); + } catch (IOException e) { + throw new RuntimeException("Error while parsing policy", e); + } + return new TestSimpleTCAPluginConfig(VES_MESSAGE_FIELD_NAME, policyJson, TCA_PLUGIN_ALERT_FIELD_NAME, + TCA_PLUGIN_MESSAGE_TYPE_FIELD_NAME, getSimpleTCAPluginInputSchema().toString(), false); + } + + protected static Schema getSimpleTCAPluginInputSchema() { + return Schema.recordOf( + "TestSimpleTCAPluginInputSchema", + Schema.Field.of("message", Schema.of(Schema.Type.STRING)), + Schema.Field.of("inputField1", Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of("inputField2", Schema.nullableOf(Schema.of(Schema.Type.STRING))) + ); + } + + protected static Schema getJsonFilterPluginInputSchema() { + return Schema.recordOf( + "TestJsonFilterInputSchema", + Schema.Field.of("ts", Schema.of(Schema.Type.LONG)), + Schema.Field.of("responseCode", Schema.of(Schema.Type.INT)), + Schema.Field.of("responseMessage", Schema.of(Schema.Type.STRING)), + Schema.Field.of("message", Schema.of(Schema.Type.STRING)) + ); + } + + protected static TestJsonPathFilterPluginConfig getJsonPathFilterPluginConfig() { + return new TestJsonPathFilterPluginConfig(JSON_PATH_FILTER_PLUGIN_REFERENCE_NAME, + JSON_PATH_FILTER_PLUGIN_INCOMING_JSON_FIELD_NAME, + JSON_PATH_FILTER_PLUGIN_OUTPUT_SCHEMA_FILED_NAME, + JSON_PATH_FILTER_PLUGIN_JSON_FILTER_MAPPINGS, + JSON_PATH_FILTER_PLUGIN_JSON_FILTER_OUTPUT_SCHEMA); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java new file mode 100644 index 0000000..79c7698 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java @@ -0,0 +1,77 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.common.CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSinkPluginConfig; +import org.onap.dcae.apod.analytics.common.AnalyticsConstants; + +import java.util.Map; + +import static org.junit.Assert.assertTrue; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMROutputFormatProviderTest extends BaseAnalyticsCDAPPluginsUnitTest { + + + @Test + public void testDMaaPMROutputFormatProviderWhenConfigIsMissingNonRequiredValues() throws Exception { + final TestDMaaPMRSinkPluginConfig sinkPluginConfig = new TestDMaaPMRSinkPluginConfig(); + sinkPluginConfig.setHostName(DMAAP_MR_SINK_PLUGIN_HOST_NAME); + sinkPluginConfig.setTopicName(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME); + final DMaaPMROutputFormatProvider dMaaPMROutputFormatProvider = + new DMaaPMROutputFormatProvider(sinkPluginConfig); + final Map outputFormatConfiguration = + dMaaPMROutputFormatProvider.getOutputFormatConfiguration(); + final String hostName = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.HOST_NAME); + assertTrue(hostName.equals(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); + final String topicName = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.TOPIC_NAME); + assertTrue(topicName.equals(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); + final String portNumber = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER); + assertTrue(portNumber.equals(AnalyticsConstants.DEFAULT_PORT_NUMBER.toString())); + final String protocol = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.PROTOCOL); + assertTrue(protocol.equals(AnalyticsConstants.DEFAULT_PROTOCOL)); + } + + @Test + public void testGetOutputFormatClassName() throws Exception { + final DMaaPMROutputFormatProvider dMaaPMROutputFormatProvider = + new DMaaPMROutputFormatProvider(getTestDMaaPMRSinkPluginConfig()); + final String outputFormatClassName = dMaaPMROutputFormatProvider.getOutputFormatClassName(); + assertTrue(outputFormatClassName.equals(DMaaPMROutputFormat.class.getName())); + } + + @Test + public void testGetOutputFormatConfiguration() throws Exception { + final TestDMaaPMRSinkPluginConfig testDMaaPMRSinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); + final DMaaPMROutputFormatProvider dMaaPMROutputFormatProvider = + new DMaaPMROutputFormatProvider(testDMaaPMRSinkPluginConfig); + final Map outputFormatConfiguration = + dMaaPMROutputFormatProvider.getOutputFormatConfiguration(); + assertTrue(outputFormatConfiguration.size() == 9); + + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java new file mode 100644 index 0000000..4b111a3 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java @@ -0,0 +1,75 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMROutputFormatTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private DMaaPMROutputFormat dMaaPMROutputFormat; + + @Before + public void before() { + dMaaPMROutputFormat = new DMaaPMROutputFormat(); + } + + @Test + public void testGetRecordWriter() throws Exception { + final TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class); + when(taskAttemptContext.getConfiguration()).thenReturn(getTestConfiguration()); + final RecordWriter recordWriter = dMaaPMROutputFormat.getRecordWriter(taskAttemptContext); + assertNotNull(recordWriter); + final JobContext jobContext = Mockito.mock(JobContext.class); + dMaaPMROutputFormat.checkOutputSpecs(jobContext); + } + + @Test + public void testGetOutputCommitter() throws Exception { + final TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class); + final OutputCommitter outputCommitter = dMaaPMROutputFormat.getOutputCommitter(taskAttemptContext); + assertTrue(outputCommitter.getClass().equals(DMaaPMROutputFormat.NoOpOutputCommitter.class)); + final JobContext jobContext = Mockito.mock(JobContext.class); + outputCommitter.setupJob(jobContext); + outputCommitter.setupTask(taskAttemptContext); + assertFalse(outputCommitter.needsTaskCommit(taskAttemptContext)); + outputCommitter.commitJob(jobContext); + outputCommitter.commitTask(taskAttemptContext); + outputCommitter.abortTask(taskAttemptContext); + + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java new file mode 100644 index 0000000..3d79057 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java @@ -0,0 +1,62 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.dmaap.service.publisher.DMaaPMRPublisher; + +import java.util.Arrays; + +import static org.mockito.Mockito.times; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMRRecordWriterTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private DMaaPMRPublisher publisher; + private DMaaPMRRecordWriter dMaaPMRRecordWriter; + + @Before + public void before() { + publisher = Mockito.mock(DMaaPMRPublisher.class); + dMaaPMRRecordWriter = new DMaaPMRRecordWriter(publisher); + } + + @Test + public void testWrite() throws Exception { + final String testMessage = "test Message"; + dMaaPMRRecordWriter.write(testMessage, null); + Mockito.verify(publisher, times(1)).publish(Arrays.asList(testMessage)); + } + + @Test + public void testClose() throws Exception { + final TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class); + dMaaPMRRecordWriter.close(taskAttemptContext); + Mockito.verify(publisher, times(1)).flush(); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java new file mode 100644 index 0000000..d649e6e --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java @@ -0,0 +1,95 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.Emitter; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.StageConfigurer; +import co.cask.cdap.etl.api.batch.BatchSinkContext; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMRSinkTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private DMaaPMRSink dMaaPMRSink; + + @Before + public void before() { + dMaaPMRSink = new DMaaPMRSink(getTestDMaaPMRSinkPluginConfig()); + } + + @Test + public void testConfigurePipeline() throws Exception { + final PipelineConfigurer pipelineConfigurer = Mockito.mock(PipelineConfigurer.class); + final StageConfigurer stageConfigurer = Mockito.mock(StageConfigurer.class); + when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); + when(stageConfigurer.getInputSchema()).thenReturn(getDMaaPMRSinkTestSchema()); + dMaaPMRSink.configurePipeline(pipelineConfigurer); + verify(stageConfigurer, times(1)).getInputSchema(); + } + + @Test(expected = CDAPSettingsException.class) + public void testConfigurePipelineWithInvalidSchema() throws Exception { + final PipelineConfigurer pipelineConfigurer = Mockito.mock(PipelineConfigurer.class); + final StageConfigurer stageConfigurer = Mockito.mock(StageConfigurer.class); + when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); + when(stageConfigurer.getInputSchema()).thenReturn(Schema.recordOf( + "DMaaPMRSinkInvalidSchema", + Schema.Field.of("message1", Schema.of(Schema.Type.STRING)), + Schema.Field.of("field1", Schema.of(Schema.Type.STRING)) + )); + dMaaPMRSink.configurePipeline(pipelineConfigurer); + } + + @Test + public void testPrepareRun() throws Exception { + final BatchSinkContext batchSinkContext = Mockito.mock(BatchSinkContext.class); + dMaaPMRSink.prepareRun(batchSinkContext); + } + + @Test + @SuppressWarnings("unchecked") + public void testTransform() throws Exception { + final StructuredRecord structuredRecord = Mockito.mock(StructuredRecord.class); + final Emitter emitter = Mockito.mock(Emitter.class); + final String incomingTestMessage = "test message"; + when(structuredRecord.get( + eq(getTestDMaaPMRSinkPluginConfig().getMessageColumnName()))).thenReturn(incomingTestMessage); + doNothing().when(emitter).emit(any()); + dMaaPMRSink.transform(structuredRecord, emitter); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java new file mode 100644 index 0000000..5c80baa --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java @@ -0,0 +1,80 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; + +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author Rajiv Singla . Creation Date: 1/23/2017. + */ +public class DMaaPMRSinkPluginConfigTest extends BaseAnalyticsCDAPPluginsUnitTest { + + @Test + public void testDMaaPMRSinkPluginConfigDefaults() throws Exception { + final DMaaPMRSinkPluginConfig sinkPluginConfig = new DMaaPMRSinkPluginConfig + (DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME, DMAAP_MR_SINK_PLUGIN_HOST_NAME, + DMAAP_MR_SINK_PLUGIN_TOPIC_NAME, DMAAP_MR_SINK_MESSAGE_COLUMN_NAME); + + assertThat(sinkPluginConfig.getReferenceName(), is(DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME)); + assertThat(sinkPluginConfig.getHostName(), is(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); + assertThat(sinkPluginConfig.getTopicName(), is(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); + assertThat(sinkPluginConfig.getMessageColumnName(), is(DMAAP_MR_SINK_MESSAGE_COLUMN_NAME)); + assertNull(sinkPluginConfig.getPortNumber()); + assertNull(sinkPluginConfig.getProtocol()); + assertNull(sinkPluginConfig.getUserName()); + assertNull(sinkPluginConfig.getUserPassword()); + assertNull(sinkPluginConfig.getContentType()); + assertNull(sinkPluginConfig.getMaxBatchSize()); + assertNull(sinkPluginConfig.getMaxRecoveryQueueSize()); + } + + @Test + public void testDMaaPMRSinkPluginConfigCustom() throws Exception { + final DMaaPMRSinkPluginConfig sinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); + assertThat(sinkPluginConfig.getReferenceName(), is(DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME)); + assertThat(sinkPluginConfig.getHostName(), is(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); + assertThat(sinkPluginConfig.getTopicName(), is(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); + assertThat(sinkPluginConfig.getPortNumber(), is(DMAAP_MR_SINK_PLUGIN_PORT_NUMBER)); + assertThat(sinkPluginConfig.getProtocol(), is(DMAAP_MR_SINK_PLUGIN_PROTOCOL)); + assertThat(sinkPluginConfig.getUserName(), is(DMAAP_MR_SINK_PLUGIN_USERNAME)); + assertThat(sinkPluginConfig.getUserPassword(), is(DMAAP_MR_SINK_PLUGIN_PASSWORD)); + assertThat(sinkPluginConfig.getContentType(), is(DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE)); + assertThat(sinkPluginConfig.getMessageColumnName(), is(DMAAP_MR_SINK_MESSAGE_COLUMN_NAME)); + assertThat(sinkPluginConfig.getMaxBatchSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE)); + assertThat(sinkPluginConfig.getMaxRecoveryQueueSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE)); + } + + @Test + public void testValidToString() throws Exception { + final TestDMaaPMRSinkPluginConfig sinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); + assertNotNull(sinkPluginConfig.toString()); + assertTrue(sinkPluginConfig.toString().contains(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); + } + + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java new file mode 100644 index 0000000..bb4f7f6 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java @@ -0,0 +1,84 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; + +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + + +/** + * @author Rajiv Singla . Creation Date: 1/23/2017. + */ +public class DMaaPMRSourcePluginConfigTest extends BaseAnalyticsCDAPPluginsUnitTest { + + @Test + public void testDMaaPMRSourcePluginConfigDefaults() throws Exception { + final DMaaPMRSourcePluginConfig sourcePluginConfig = new DMaaPMRSourcePluginConfig + (DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME, DMAAP_MR_SOURCE_PLUGIN_HOST_NAME, + DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME, DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL); + + assertThat(sourcePluginConfig.getReferenceName(), is(DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME)); + assertThat(sourcePluginConfig.getHostName(), is(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); + assertThat(sourcePluginConfig.getTopicName(), is(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME)); + assertThat(sourcePluginConfig.getPollingInterval(), is(DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL)); + assertNull(sourcePluginConfig.getPortNumber()); + assertNull(sourcePluginConfig.getProtocol()); + assertNull(sourcePluginConfig.getUserName()); + assertNull(sourcePluginConfig.getUserPassword()); + assertNull(sourcePluginConfig.getContentType()); + assertNull(sourcePluginConfig.getConsumerGroup()); + assertNull(sourcePluginConfig.getConsumerId()); + assertNull(sourcePluginConfig.getMessageLimit()); + assertNull(sourcePluginConfig.getTimeoutMS()); + } + + @Test + public void testDMaaPMRSourcePluginConfigCustom() throws Exception { + final TestDMaaPMRSourcePluginConfig sourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + assertThat(sourcePluginConfig.getReferenceName(), is(DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME)); + assertThat(sourcePluginConfig.getHostName(), is(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); + assertThat(sourcePluginConfig.getTopicName(), is(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME)); + assertThat(sourcePluginConfig.getPollingInterval(), is(DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL)); + assertThat(sourcePluginConfig.getPortNumber(), is(DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER)); + assertThat(sourcePluginConfig.getProtocol(), is(DMAAP_MR_SOURCE_PLUGIN_PROTOCOL)); + assertThat(sourcePluginConfig.getUserName(), is(DMAAP_MR_SOURCE_PLUGIN_USERNAME)); + assertThat(sourcePluginConfig.getUserPassword(), is(DMAAP_MR_SOURCE_PLUGIN_PASSWORD)); + assertThat(sourcePluginConfig.getContentType(), is(DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE)); + assertThat(sourcePluginConfig.getConsumerGroup(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP)); + assertThat(sourcePluginConfig.getConsumerId(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID)); + assertThat(sourcePluginConfig.getMessageLimit(), is(DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT)); + assertThat(sourcePluginConfig.getTimeoutMS(), is(DMAAP_MR_SOURCE_PLUGIN_TIMEOUT)); + } + + @Test + public void testValidToString() throws Exception { + final TestDMaaPMRSourcePluginConfig sourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + assertNotNull(sourcePluginConfig.toString()); + assertTrue(sourcePluginConfig.toString().contains(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java new file mode 100644 index 0000000..8cba5e1 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java @@ -0,0 +1,76 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; + +import javax.annotation.Nullable; + +/** + * Test {@link DMaaPMRSinkPluginConfig} for testing purposes only + *

+ * @author Rajiv Singla . Creation Date: 1/23/2017. + */ +public class TestDMaaPMRSinkPluginConfig extends DMaaPMRSinkPluginConfig { + + public void setReferenceName(String referenceName) { + this.referenceName = referenceName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public void setPortNumber(@Nullable Integer portNumber) { + this.portNumber = portNumber; + } + + public void setTopicName(String topicName) { + this.topicName = topicName; + } + + public void setProtocol(@Nullable String protocol) { + this.protocol = protocol; + } + + public void setUserName(@Nullable String userName) { + this.userName = userName; + } + + public void setUserPassword(@Nullable String userPassword) { + this.userPassword = userPassword; + } + + public void setContentType(@Nullable String contentType) { + this.contentType = contentType; + } + + public void setMaxBatchSize(@Nullable Integer maxBatchSize) { + this.maxBatchSize = maxBatchSize; + } + + public void setMaxRecoveryQueueSize(@Nullable Integer maxRecoveryQueueSize) { + this.maxRecoveryQueueSize = maxRecoveryQueueSize; + } + + public void setMessageColumnName(String messageColumnName) { + this.messageColumnName = messageColumnName; + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java new file mode 100644 index 0000000..c33f313 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java @@ -0,0 +1,84 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; + +import javax.annotation.Nullable; + +/** + * Test {@link DMaaPMRSourcePluginConfig} for testing purposes only + *

+ * @author Rajiv Singla . Creation Date: 1/23/2017. + */ +public class TestDMaaPMRSourcePluginConfig extends DMaaPMRSourcePluginConfig { + + public void setReferenceName(String referenceName) { + this.referenceName = referenceName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public void setPortNumber(@Nullable Integer portNumber) { + this.portNumber = portNumber; + } + + public void setTopicName(String topicName) { + this.topicName = topicName; + } + + public void setPollingInterval(Integer pollingInterval) { + this.pollingInterval = pollingInterval; + } + + public void setProtocol(@Nullable String protocol) { + this.protocol = protocol; + } + + public void setUserName(@Nullable String userName) { + this.userName = userName; + } + + public void setUserPassword(@Nullable String userPassword) { + this.userPassword = userPassword; + } + + public void setContentType(@Nullable String contentType) { + this.contentType = contentType; + } + + public void setConsumerId(@Nullable String consumerId) { + this.consumerId = consumerId; + } + + public void setConsumerGroup(@Nullable String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public void setTimeoutMS(@Nullable Integer timeoutMS) { + this.timeoutMS = timeoutMS; + } + + public void setMessageLimit(@Nullable Integer messageLimit) { + this.messageLimit = messageLimit; + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java new file mode 100644 index 0000000..efdd7ae --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java @@ -0,0 +1,50 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter; + +/** + * @author Rajiv Singla . Creation Date: 3/3/2017. + */ +public class TestJsonPathFilterPluginConfig extends JsonPathFilterPluginConfig { + + public TestJsonPathFilterPluginConfig(final String referenceName, final String incomingJsonFieldName, + final String outputSchemaFieldName, final String jsonFilterMappings, + final String schema) { + super(referenceName, incomingJsonFieldName, outputSchemaFieldName, jsonFilterMappings, schema); + } + + + public void setIncomingJsonFieldName(String incomingJsonFieldName) { + this.incomingJsonFieldName = incomingJsonFieldName; + } + + public void setOutputSchemaFieldName(String outputSchemaFieldName) { + this.outputSchemaFieldName = outputSchemaFieldName; + } + + public void setJsonFilterMappings(String jsonFilterMappings) { + this.jsonFilterMappings = jsonFilterMappings; + } + + public void setSchema(String schema) { + this.schema = schema; + } +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java new file mode 100644 index 0000000..6bf3252 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java @@ -0,0 +1,56 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca; + +/** + * @author Rajiv Singla . Creation Date: 2/17/2017. + */ +public class TestSimpleTCAPluginConfig extends SimpleTCAPluginConfig { + + public TestSimpleTCAPluginConfig(String vesMessageFieldName, String policyJson, String alertFieldName, + String messageTypeFieldName, String schema, Boolean enableAlertCEFFormat) { + super(vesMessageFieldName, policyJson, alertFieldName, messageTypeFieldName, schema, enableAlertCEFFormat); + } + + public void setVesMessageFieldName(String vesMessageFieldName) { + this.vesMessageFieldName = vesMessageFieldName; + } + + public void setPolicyJson(String policyJson) { + this.policyJson = policyJson; + } + + public void setAlertFieldName(String alertFieldName) { + this.alertFieldName = alertFieldName; + } + + public void setMessageTypeFieldName(String messageTypeFieldName) { + this.messageTypeFieldName = messageTypeFieldName; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public void setEnableAlertCEFFormat(Boolean enableAlertCEFFormat) { + this.enableAlertCEFFormat = enableAlertCEFFormat; + } +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java new file mode 100644 index 0000000..d14e184 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java @@ -0,0 +1,63 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap; + +import co.cask.cdap.api.data.schema.Schema; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; + +import java.util.LinkedList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author Rajiv Singla . Creation Date: 1/25/2017. + */ +public class DMaaPSourceOutputSchemaTest extends BaseAnalyticsCDAPPluginsUnitTest { + + + @Test + public void testGetSchemaColumnName() throws Exception { + assertThat(DMaaPSourceOutputSchema.TIMESTAMP.getSchemaColumnName(), is("ts")); + assertThat(DMaaPSourceOutputSchema.RESPONSE_CODE.getSchemaColumnName(), is("responseCode")); + assertThat(DMaaPSourceOutputSchema.RESPONSE_MESSAGE.getSchemaColumnName(), is("responseMessage")); + assertThat(DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName(), is("message")); + } + + @Test + public void testGetSchema() throws Exception { + final Schema schema = DMaaPSourceOutputSchema.getSchema(); + final List fields = schema.getFields(); + final List fieldNames = new LinkedList<>(); + for (Schema.Field field : fields) { + fieldNames.add(field.getName()); + } + assertThat(fieldNames, hasItems( + DMaaPSourceOutputSchema.TIMESTAMP.getSchemaColumnName(), + DMaaPSourceOutputSchema.RESPONSE_CODE.getSchemaColumnName(), + DMaaPSourceOutputSchema.RESPONSE_MESSAGE.getSchemaColumnName(), + DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName())); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java new file mode 100644 index 0000000..762e65a --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java @@ -0,0 +1,229 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.it; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.api.dataset.table.Table; +import co.cask.cdap.api.plugin.PluginClass; +import co.cask.cdap.api.plugin.PluginPropertyField; +import co.cask.cdap.common.utils.Tasks; +import co.cask.cdap.datapipeline.DataPipelineApp; +import co.cask.cdap.datapipeline.SmartWorkflow; +import co.cask.cdap.etl.api.batch.SparkCompute; +import co.cask.cdap.etl.mock.batch.MockSink; +import co.cask.cdap.etl.mock.batch.MockSource; +import co.cask.cdap.etl.mock.test.HydratorTestBase; +import co.cask.cdap.etl.proto.v2.ETLBatchConfig; +import co.cask.cdap.etl.proto.v2.ETLPlugin; +import co.cask.cdap.etl.proto.v2.ETLStage; +import co.cask.cdap.proto.artifact.AppRequest; +import co.cask.cdap.proto.artifact.ArtifactSummary; +import co.cask.cdap.proto.id.ApplicationId; +import co.cask.cdap.proto.id.ArtifactId; +import co.cask.cdap.proto.id.NamespaceId; +import co.cask.cdap.test.ApplicationManager; +import co.cask.cdap.test.DataSetManager; +import co.cask.cdap.test.WorkflowManager; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.common.persistance.tca.TCACalculatorMessageType; +import org.onap.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.sparkcompute.tca.SimpleTCAPlugin; +import org.onap.dcae.apod.analytics.common.validation.DCAEValidator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +/** + * Integration Test which used CDAP Hydrator Test Base to Test Simple TCA Plugin + * + * @author Rajiv Singla . Creation Date: 2/17/2017. + */ +public class SimpleTCAPluginCDAPIT extends HydratorTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(SimpleTCAPluginCDAPIT.class); + + private static final String CDAP_PLUGIN_VERSION = "3.0-SNAPSHOT"; + private static final String CDAP_PLUGIN_ARTIFACT_NAME = "dcae-analytics-cdap-plugins"; + + protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", + "4.0.0"); + protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "4.0.0"); + + private static Schema sourceSchema = Schema.recordOf("CEFMessageSourceSchema", + Schema.Field.of("message", Schema.of(Schema.Type.STRING)) + ); + + final Schema outputSchema = Schema.recordOf( + "outputSchema", + Schema.Field.of("message", Schema.of(Schema.Type.STRING)), + Schema.Field.of("alert", Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of("tcaMessageType", Schema.of(Schema.Type.STRING)) + ); + + @BeforeClass + public static void setupTest() throws Exception { + + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); + + + // Enable the below code if you want to run the test in Intelli IDEA editor + // addPluginArtifact(NamespaceId.DEFAULT.artifact("spark-plugins", "1.0.0"), DATAPIPELINE_ARTIFACT_ID, + // SimpleTCAPlugin.class, SimpleTCAPluginConfig.class); + + // Enable the below code if you want to run the test via command line + ArtifactId dcaeAnalyticsCdapPluginsArtifact = NamespaceId.DEFAULT.artifact( + CDAP_PLUGIN_ARTIFACT_NAME, CDAP_PLUGIN_VERSION); + + addPluginArtifact(dcaeAnalyticsCdapPluginsArtifact, DATAPIPELINE_ARTIFACT_ID, + ImmutableSet.of(getSimpleTCAPluginClass()), SimpleTCAPlugin.class, SimpleTCAPluginConfig.class, + CDAPAppSettingsValidator.class, DCAEValidator.class); + } + + private static PluginClass getSimpleTCAPluginClass() { + final HashMap properties = new HashMap<>(); + properties.put("vesMessageFieldName", new PluginPropertyField("vesMessageFieldName", "", + "string", false, false)); + properties.put("referenceName", new PluginPropertyField("referenceName", "", + "string", false, false)); + properties.put("policyJson", new PluginPropertyField("policyJson", "", "string", false, false)); + properties.put("alertFieldName", new PluginPropertyField("alertFieldName", "", "string", false, false)); + properties.put("messageTypeFieldName", new PluginPropertyField( + "messageTypeFieldName", "", "string", false, false)); + properties.put("enableAlertCEFFormat", new PluginPropertyField( + "enableAlertCEFFormat", "", "string", false, false)); + properties.put("schema", new PluginPropertyField( + "schema", "", "string", false, false)); + + return new PluginClass("sparkcompute", "SimpleTCAPlugin", "", SimpleTCAPlugin.class.getName(), + "pluginConfig", properties); + } + + + @AfterClass + public static void cleanup() { + } + + @Test + @SuppressWarnings("deprecation") + public void testTransform() throws Exception { + + LOG.info("Starting Test Transform"); + + final String policyString = getFileContentAsString("/data/json/policy/tca_policy.json"); + final String cefMessage = getFileContentAsString("/data/json/cef/cef_message.json"); + + final Map tcaProperties = new ImmutableMap.Builder() + .put("vesMessageFieldName", "message") + .put("referenceName", "SimpleTcaPlugin") + .put("policyJson", policyString) + .put("alertFieldName", "alert") + .put("messageTypeFieldName", "tcaMessageType") + .put("enableAlertCEFFormat", "true") + .put("schema", outputSchema.toString()) + .build(); + + final ETLPlugin mockSourcePlugin = MockSource.getPlugin("messages", sourceSchema); + final ETLPlugin tcaPlugin = + new ETLPlugin("SimpleTCAPlugin", SparkCompute.PLUGIN_TYPE, tcaProperties, null); + final ETLPlugin mockSink = MockSink.getPlugin("tcaOutput"); + + final ETLBatchConfig etlBatchConfig = ETLBatchConfig.builder("* * * * *") + .addStage(new ETLStage("source", mockSourcePlugin)) + .addStage(new ETLStage("simpleTCAPlugin", tcaPlugin)) + .addStage(new ETLStage("sink", mockSink)) + .addConnection("source", "simpleTCAPlugin") + .addConnection("simpleTCAPlugin", "sink") + .build(); + + AppRequest appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlBatchConfig); + ApplicationId appId = NamespaceId.DEFAULT.app("TestSimpleTCAPlugin"); + ApplicationManager appManager = deployApplication(appId.toId(), appRequest); + + List sourceMessages = new ArrayList<>(); + StructuredRecord.Builder builder = StructuredRecord.builder(sourceSchema); + builder.set("message", cefMessage); + sourceMessages.add(builder.build()); + + // write records to source + DataSetManager inputManager = getDataset(NamespaceId.DEFAULT.dataset("messages")); + MockSource.writeInput(inputManager, sourceMessages); + + // manually trigger the pipeline + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.start(); + workflowManager.waitForFinish(5, TimeUnit.MINUTES); + + final DataSetManager
outputManager = getDataset("tcaOutput"); + + Tasks.waitFor( + TCACalculatorMessageType.COMPLIANT.name(), + new Callable() { + @Override + public String call() throws Exception { + outputManager.flush(); + List tcaOutputMessageType = new LinkedList<>(); + for (StructuredRecord outputRecord : MockSink.readOutput(outputManager)) { + tcaOutputMessageType.add(outputRecord.get("tcaMessageType").toString()); + final List fields = outputRecord.getSchema().getFields(); + LOG.debug("====>> Printing output Structured Record Contents: {}", outputRecord); + for (Schema.Field field : fields) { + LOG.debug("Field Name: {} - Field Type: {} ---> Field Value: {}", + field.getName(), field.getSchema().getType(), + outputRecord.get(field.getName())); + } + + } + return tcaOutputMessageType.get(0); + } + }, + 4, + TimeUnit.MINUTES); + + } + + private static String getFileContentAsString(final String fileLocation) throws Exception { + final URI tcaPolicyURI = + SimpleTCAPluginCDAPIT.class.getResource(fileLocation).toURI(); + List lines = Files.readAllLines(Paths.get(tcaPolicyURI), Charset.defaultCharset()); + return Joiner.on("").join(lines); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java new file mode 100644 index 0000000..f7e379b --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java @@ -0,0 +1,119 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.sparkcompute.tca; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.StageConfigurer; +import co.cask.cdap.etl.api.batch.SparkExecutionPluginContext; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.common.persistance.tca.TCACalculatorMessageType; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.TestSimpleTCAPluginConfig; + +import java.util.LinkedList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author Rajiv Singla . Creation Date: 2/17/2017. + */ +public class SimpleTCAPluginTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private SimpleTCAPlugin simpleTCAPlugin; + + @Before + public void before() { + final TestSimpleTCAPluginConfig testSimpleTCAPluginConfig = getTestSimpleTCAPluginConfig(); + Schema outputSchema = Schema.recordOf( + "TestSimpleTCAPluginInputSchema", + Schema.Field.of("message", Schema.of(Schema.Type.STRING)), + Schema.Field.of("alert", Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of("tcaMessageType", Schema.of(Schema.Type.STRING)) + ); + testSimpleTCAPluginConfig.setSchema(outputSchema.toString()); + simpleTCAPlugin = new SimpleTCAPlugin(testSimpleTCAPluginConfig); + } + + @Test + public void testConfigurePipeline() throws Exception { + final PipelineConfigurer pipelineConfigurer = mock(PipelineConfigurer.class); + final StageConfigurer stageConfigurer = mock(StageConfigurer.class); + when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); + when(stageConfigurer.getInputSchema()).thenReturn(getSimpleTCAPluginInputSchema()); + simpleTCAPlugin.configurePipeline(pipelineConfigurer); + verify(stageConfigurer, times(1)).getInputSchema(); + } + + @Test + public void testTransform() throws Exception { + + JavaSparkContext javaSparkContext = new JavaSparkContext("local", "test"); + + Schema sourceSchema = Schema.recordOf("CEFMessageSourceSchema", + Schema.Field.of("message", Schema.of(Schema.Type.STRING)) + ); + + // Inapplicable Message Structured Record + final StructuredRecord inapplicableSR = + StructuredRecord.builder(sourceSchema).set("message", "test").build(); + // compliant + final StructuredRecord compliantSR = + StructuredRecord.builder(sourceSchema).set("message", + fromStream(CEF_MESSAGE_JSON_FILE_LOCATION)).build(); + // non compliant + final String nonCompliantCEF = fromStream(CEF_NON_COMPLIANT_MESSAGE_JSON_FILE_LOCATION); + final StructuredRecord nonCompliantSR = + StructuredRecord.builder(sourceSchema).set("message", nonCompliantCEF).build(); + + final List records = new LinkedList<>(); + records.add(inapplicableSR); + records.add(compliantSR); + records.add(nonCompliantSR); + + final JavaRDD input = + javaSparkContext.parallelize(records); + final SparkExecutionPluginContext context = Mockito.mock(SparkExecutionPluginContext.class); + final MockStageMetrics stageMetrics = Mockito.mock(MockStageMetrics.class); + when(context.getMetrics()).thenReturn(stageMetrics); + final List outputRecord = simpleTCAPlugin.transform(context, input).collect(); + assertNotNull(outputRecord); + assertThat(outputRecord.size(), is(3)); + + assertTrue(outputRecord.get(0).get("tcaMessageType").equals(TCACalculatorMessageType.INAPPLICABLE.toString())); + assertTrue(outputRecord.get(1).get("tcaMessageType").equals(TCACalculatorMessageType.COMPLIANT.toString())); + assertTrue(outputRecord.get(2).get("tcaMessageType").equals(TCACalculatorMessageType.NON_COMPLIANT.toString())); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java new file mode 100644 index 0000000..aca5581 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java @@ -0,0 +1,75 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import com.google.common.collect.ImmutableList; +import org.apache.spark.storage.StorageLevel; +import org.junit.Test; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; +import org.onap.dcae.apod.analytics.dmaap.domain.response.DMaaPMRSubscriberResponse; +import org.onap.dcae.apod.analytics.dmaap.service.subscriber.DMaaPMRSubscriber; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author Rajiv Singla . Creation Date: 1/24/2017. + */ +public class DMaaPMRReceiverTest extends BaseAnalyticsCDAPPluginsUnitTest { + + + @Test + public void testStoreStructuredRecords() throws Exception { + + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + final TestDMaaPMRReceiver dMaaPMRReceiver = + new TestDMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), testDMaaPMRSourcePluginConfig); + + final DMaaPMRSubscriber dMaaPMRSubscriber = Mockito.mock(DMaaPMRSubscriber.class); + final DMaaPMRSubscriberResponse subscriberResponse = Mockito.mock(DMaaPMRSubscriberResponse.class); + when(dMaaPMRSubscriber.fetchMessages()).thenReturn(subscriberResponse); + when(subscriberResponse.getFetchedMessages()).thenReturn(ImmutableList.of("Test Message")); + when(subscriberResponse.getResponseCode()).thenReturn(200); + when(subscriberResponse.getResponseMessage()).thenReturn("OK"); + dMaaPMRReceiver.storeStructuredRecords(dMaaPMRSubscriber); + verify(dMaaPMRSubscriber, times(1)).fetchMessages(); + verify(subscriberResponse, times(1)).getFetchedMessages(); + } + + @Test + public void testStoreStructuredRecordsWhenSubscriberThrowsException() throws Exception { + + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + final TestDMaaPMRReceiver dMaaPMRReceiver = + new TestDMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), testDMaaPMRSourcePluginConfig); + + final DMaaPMRSubscriber dMaaPMRSubscriber = Mockito.mock(DMaaPMRSubscriber.class); + final DMaaPMRSubscriberResponse subscriberResponse = Mockito.mock(DMaaPMRSubscriberResponse.class); + when(dMaaPMRSubscriber.fetchMessages()).thenThrow(DCAEAnalyticsRuntimeException.class); + dMaaPMRReceiver.storeStructuredRecords(dMaaPMRSubscriber); + verify(dMaaPMRSubscriber, times(1)).fetchMessages(); + verify(subscriberResponse, times(0)).getFetchedMessages(); + } +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java new file mode 100644 index 0000000..7b1f876 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java @@ -0,0 +1,91 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.StageConfigurer; +import co.cask.cdap.etl.api.streaming.StreamingContext; +import org.apache.spark.streaming.api.java.JavaDStream; +import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; +import org.apache.spark.streaming.api.java.JavaStreamingContext; +import org.apache.spark.streaming.receiver.Receiver; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; + +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.when; + +/** + * @author Rajiv Singla . Creation Date: 1/24/2017. + */ +public class DMaaPMRSourceTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private PipelineConfigurer pipelineConfigurer; + + @Before + public void before() { + pipelineConfigurer = Mockito.mock(PipelineConfigurer.class); + final StageConfigurer stageConfigurer = Mockito.mock(StageConfigurer.class); + when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); + doNothing().when(stageConfigurer).setOutputSchema(any(Schema.class)); + } + + @Test + public void testDMaaPMRSourceConfigurePipelineWithValidPluginSettings() throws Exception { + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + final DMaaPMRSource dMaaPMRSource = new DMaaPMRSource(testDMaaPMRSourcePluginConfig); + dMaaPMRSource.configurePipeline(pipelineConfigurer); + assertNotNull(dMaaPMRSource); + } + + @Test(expected = CDAPSettingsException.class) + public void testDMaaPMRSourceConfigurePipelineWithInvalidPluginSettings() throws Exception { + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + // blank out DMaaP MR Source Host + testDMaaPMRSourcePluginConfig.setHostName(null); + final DMaaPMRSource dMaaPMRSource = new DMaaPMRSource(testDMaaPMRSourcePluginConfig); + dMaaPMRSource.configurePipeline(pipelineConfigurer); + } + + + @Test + @SuppressWarnings("unchecked") + public void testGetStream() throws Exception { + final StreamingContext streamingContext = Mockito.mock(StreamingContext.class); + final JavaStreamingContext javaStreamingContext = Mockito.mock(JavaStreamingContext.class); + final JavaReceiverInputDStream dMaaPMRReceiver = Mockito.mock(JavaReceiverInputDStream.class); + when(streamingContext.getSparkStreamingContext()).thenReturn(javaStreamingContext); + when(javaStreamingContext.receiverStream(any(Receiver.class))).thenReturn(dMaaPMRReceiver); + + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + final DMaaPMRSource dMaaPMRSource = new DMaaPMRSource(testDMaaPMRSourcePluginConfig); + final JavaDStream stream = dMaaPMRSource.getStream(streamingContext); + assertNotNull(stream); + } +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java new file mode 100644 index 0000000..cd3e4bd --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java @@ -0,0 +1,81 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.data.format.StructuredRecord; +import org.apache.spark.storage.StorageLevel; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap.DMaaPSourceOutputSchema; + +import java.util.concurrent.TimeUnit; + +/** + * @author Rajiv Singla . Creation Date: 2/20/2017. + */ +public class MockDMaaPMRReceiverTest extends BaseAnalyticsCDAPPluginsUnitTest { + + protected class TestMockDMaaPMRReceiverTest extends MockDMaaPMRReceiver { + + private boolean canStop = false; + + public TestMockDMaaPMRReceiverTest(StorageLevel storageLevel, DMaaPMRSourcePluginConfig pluginConfig) { + super(storageLevel, pluginConfig); + } + + @Override + public boolean isStopped() { + return canStop; + } + + @Override + public void store(StructuredRecord dataItem) { + LOG.debug("Mocking storing dataItem - {}", + dataItem.get(DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName())); + } + + public void setCanStop(boolean canStop) { + this.canStop = canStop; + } + } + + @Test + public void testStoreStructuredRecords() throws Exception { + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + testDMaaPMRSourcePluginConfig.setPollingInterval(100); + final TestMockDMaaPMRReceiverTest mockDMaaPMRReceiver = new TestMockDMaaPMRReceiverTest(StorageLevel + .MEMORY_ONLY(), + testDMaaPMRSourcePluginConfig); + new Thread(new Runnable() { + @Override + public void run() { + mockDMaaPMRReceiver.storeStructuredRecords(null); + } + }).start(); + TimeUnit.MILLISECONDS.sleep(1000); + LOG.info("Killing Mock Subscriber after 1 ms"); + mockDMaaPMRReceiver.setCanStop(true); + + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java new file mode 100644 index 0000000..6a842d0 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java @@ -0,0 +1,74 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.etl.api.streaming.StreamingContext; +import org.apache.spark.streaming.api.java.JavaDStream; +import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; +import org.apache.spark.streaming.api.java.JavaStreamingContext; +import org.apache.spark.streaming.receiver.Receiver; +import org.junit.Test; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; + +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +/** + * @author Rajiv Singla . Creation Date: 2/20/2017. + */ +@SuppressWarnings("unchecked") +public class MockDMaaPMRSourceTest extends BaseAnalyticsCDAPPluginsUnitTest { + + @Test + public void testGetStream() throws Exception { + final StreamingContext streamingContext = Mockito.mock(StreamingContext.class); + final JavaStreamingContext javaStreamingContext = Mockito.mock(JavaStreamingContext.class); + final JavaReceiverInputDStream dMaaPMRReceiver = Mockito.mock(JavaReceiverInputDStream.class); + when(streamingContext.getSparkStreamingContext()).thenReturn(javaStreamingContext); + when(javaStreamingContext.receiverStream(any(Receiver.class))).thenReturn(dMaaPMRReceiver); + + MockDMaaPMRSource mockDMaaPMRSource = new MockDMaaPMRSource(getTestDMaaPMRSourcePluginConfig()); + final JavaDStream stream = mockDMaaPMRSource.getStream(streamingContext); + assertNotNull(stream); + } + + @Test(expected = CDAPSettingsException.class) + public void testConfigurePipelineWhenPollingIntervalNotPresent() throws Exception { + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + testDMaaPMRSourcePluginConfig.setPollingInterval(null); + final MockDMaaPMRSource mockDMaaPMRSource = new MockDMaaPMRSource(testDMaaPMRSourcePluginConfig); + mockDMaaPMRSource.configurePipeline(null); + } + + @Test + public void testConfigurePipelineWhenPollingIntervalIsPresent() throws Exception { + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + final MockDMaaPMRSource mockDMaaPMRSource = new MockDMaaPMRSource(testDMaaPMRSourcePluginConfig); + mockDMaaPMRSource.configurePipeline(null); + assertNotNull(mockDMaaPMRSource); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java new file mode 100644 index 0000000..0d87496 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java @@ -0,0 +1,58 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.streaming.dmaap; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.metrics.Metrics; +import org.apache.spark.storage.StorageLevel; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; + +/** + * Test implementation for {@link DMaaPMRReceiver} + *

+ * @author Rajiv Singla . Creation Date: 1/24/2017. + */ +public class TestDMaaPMRReceiver extends DMaaPMRReceiver { + + protected static Metrics metrics; + + static { + metrics = Mockito.mock(Metrics.class); + doNothing().when(metrics).count(anyString(), anyInt()); + doNothing().when(metrics).gauge(anyString(), anyInt()); + } + + + public TestDMaaPMRReceiver(StorageLevel storageLevel, DMaaPMRSourcePluginConfig pluginConfig) { + + super(storageLevel, pluginConfig, metrics); + } + + @Override + public void store(StructuredRecord dataItem) { + // do nothing + } +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java new file mode 100644 index 0000000..8bf91c8 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java @@ -0,0 +1,84 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.transform.filter; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import co.cask.cdap.etl.api.Emitter; +import co.cask.cdap.etl.api.PipelineConfigurer; +import co.cask.cdap.etl.api.StageConfigurer; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; + +import java.util.Date; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author Rajiv Singla . Creation Date: 3/3/2017. + */ +public class JsonPathFilterTest extends BaseAnalyticsCDAPPluginsUnitTest { + + + @Test + public void testInitializeWhenFilterMappingIsValid() throws Exception { + final JsonPathFilter jsonPathFilter = new JsonPathFilter(getJsonPathFilterPluginConfig()); + jsonPathFilter.initialize(null); + } + + + @Test + public void configurePipeline() throws Exception { + final JsonPathFilter jsonPathFilter = new JsonPathFilter(getJsonPathFilterPluginConfig()); + final PipelineConfigurer pipelineConfigurer = mock(PipelineConfigurer.class); + final StageConfigurer stageConfigurer = mock(StageConfigurer.class); + when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); + when(stageConfigurer.getInputSchema()).thenReturn(getSimpleTCAPluginInputSchema()); + doNothing().when(stageConfigurer).setOutputSchema(any(Schema.class)); + jsonPathFilter.configurePipeline(pipelineConfigurer); + verify(stageConfigurer, times(1)).setOutputSchema(any(Schema.class)); + } + + @Test + public void testTransform() throws Exception { + final JsonPathFilter jsonPathFilter = new JsonPathFilter(getJsonPathFilterPluginConfig()); + jsonPathFilter.initialize(null); + final StructuredRecord inputSR = StructuredRecord.builder(getJsonFilterPluginInputSchema()) + .set("ts", new Date().getTime()) + .set("responseCode", 200) + .set("responseMessage", "OK") + .set("message", fromStream(CEF_MESSAGE_JSON_FILE_LOCATION)) + .build(); + + final Emitter emitter = Mockito.mock(Emitter.class); + doNothing().when(emitter).emit(ArgumentMatchers.any(StructuredRecord.class)); + jsonPathFilter.transform(inputSR, emitter); + verify(emitter, times(1)).emit(any(StructuredRecord.class)); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java new file mode 100644 index 0000000..2792e17 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java @@ -0,0 +1,171 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.utils; + +import co.cask.cdap.api.data.format.StructuredRecord; +import co.cask.cdap.api.data.schema.Schema; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class CDAPPluginUtilsTest extends BaseAnalyticsCDAPPluginsUnitTest { + + + @Test + public void testValidateSchemaContainsFieldsWhenSchemaIsNotNull() throws Exception { + final Schema dMaaPMRSinkTestSchema = getDMaaPMRSinkTestSchema(); + CDAPPluginUtils.validateSchemaContainsFields(dMaaPMRSinkTestSchema, "message"); + } + + @Test + public void testValidateSchemaContainsFieldsWhenInputSchemaIsNull() throws Exception { + CDAPPluginUtils.validateSchemaContainsFields(null, "message"); + } + + @Test + public void testCreateStructuredRecord() throws Exception { + final StructuredRecord testMessage = CDAPPluginUtils.createDMaaPMRResponseStructuredRecord("testMessage"); + assertNotNull(testMessage); + } + + + @Test + public void testCreateOutputStructuredRecordBuilder() throws Exception { + + final String messageFieldName = "message"; + final String firstInputFieldName = "inputField1"; + final String secondInputFieldName = "inputField2"; + + + final Schema inputSchema = Schema.recordOf( + "inputSchema", + Schema.Field.of(messageFieldName, Schema.of(Schema.Type.STRING)), + Schema.Field.of(firstInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(secondInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))) + ); + + final String addedFieldName = "addedField"; + final Schema outputSchema = Schema.recordOf( + "outputSchema", + Schema.Field.of(messageFieldName, Schema.of(Schema.Type.STRING)), + Schema.Field.of(firstInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(addedFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))) // added field + // missing second Input Field + ); + + // input structured record + final String messageFieldValue = "Message String"; + final String firstFieldValue = "Input Field 1"; + final String secondFieldValue = "Input Field 2"; + final StructuredRecord inputSR = StructuredRecord.builder(inputSchema) + .set(messageFieldName, messageFieldValue) + .set(firstInputFieldName, firstFieldValue) + .set(secondInputFieldName, secondFieldValue) + .build(); + + final StructuredRecord.Builder outputStructuredRecordBuilder = + CDAPPluginUtils.createOutputStructuredRecordBuilder(outputSchema, inputSR); + + final String addedFieldValue = "Added Field Value"; + final StructuredRecord outputSR = outputStructuredRecordBuilder + .set(addedFieldName, addedFieldValue) + .build(); + + assertThat("Added Field field value copied correctly", + outputSR.get(addedFieldName).toString(), is(addedFieldValue)); + + assertThat("Output SR has message field copied correctly", + outputSR.get(messageFieldName).toString(), is(messageFieldValue)); + + assertThat("First Field value copied correctly", + outputSR.get(firstInputFieldName).toString(), is(firstFieldValue)); + + assertNull("Second Field value is null as output schema does not have the field", + outputSR.get(secondInputFieldName)); + + } + + + @Test + public void testAddFieldValueToStructuredRecordBuilder() throws Exception { + + final String messageFieldName = "message"; + final String firstInputFieldName = "inputField1"; + final String addedFieldName = "addedField"; + final String firstFieldValue = "Input Field 1"; + final String addedFieldValue = "Added Field Value"; + final Schema outputSchema = Schema.recordOf( + "outputSchema", + Schema.Field.of(messageFieldName, Schema.of(Schema.Type.STRING)), + Schema.Field.of(firstInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(addedFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))) // added field + ); + + final StructuredRecord.Builder outputSRBuilder = StructuredRecord.builder(outputSchema) + .set(messageFieldName, "Some message") + .set(firstInputFieldName, firstFieldValue); + + final StructuredRecord.Builder addedFieldSRBuilder = CDAPPluginUtils.addFieldValueToStructuredRecordBuilder( + outputSRBuilder, outputSchema, addedFieldName, addedFieldValue); + + // Try adding field to output Structured record that is not in output schema + final String nonExistentFieldName = "fieldNotInOutputSchema"; + final String nonExistentFieldValue = "Some Value"; + final StructuredRecord outputSR = CDAPPluginUtils.addFieldValueToStructuredRecordBuilder( + addedFieldSRBuilder, outputSchema, nonExistentFieldName, nonExistentFieldValue).build(); + + assertThat("Output SR must contain added Field which is in output schema", + outputSR.get(addedFieldName).toString(), is(addedFieldValue)); + assertNull("Output SR must not contain field that is not in output schema", + outputSR.get(nonExistentFieldName)); + + } + + @Test(expected = DCAEAnalyticsRuntimeException.class) + public void testValidateSchemaFieldTypeWhenInputSchemaIsNotValidJson() throws Exception { + CDAPPluginUtils.validateSchemaFieldType("Invalid Schema", "field1", Schema.Type.STRING); + } + + @Test(expected = DCAEAnalyticsRuntimeException.class) + public void testSetOutputSchemaWhenOutputSchemaIsNotValidJson() throws Exception { + CDAPPluginUtils.setOutputSchema(null, "Invalid output Schema"); + } + + @Test(expected = DCAEAnalyticsRuntimeException.class) + public void testExtractFieldMappingsWhenFieldMappingValueIsEmpty() throws Exception { + CDAPPluginUtils.extractFieldMappings("path1:,path2:value2"); + } + + @Test(expected = DCAEAnalyticsRuntimeException.class) + public void testExtractFieldMappingsWhenFieldMappingAreBlank() throws Exception { + CDAPPluginUtils.extractFieldMappings("path1: ,path2:value2"); + } + + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java new file mode 100644 index 0000000..9b6f7f8 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java @@ -0,0 +1,57 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.utils; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.dmaap.domain.config.DMaaPMRPublisherConfig; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPSinkConfigMapperTest extends BaseAnalyticsCDAPPluginsUnitTest { + + + @Test + public void testMapToPublisherConfig() throws Exception { + + final Configuration testConfiguration = getTestConfiguration(); + final DMaaPMRPublisherConfig publisherConfig = DMaaPSinkConfigMapper.map(testConfiguration); + + assertNotNull(publisherConfig); + assertThat(publisherConfig.getHostName(), is(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); + assertThat(publisherConfig.getTopicName(), is(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); + assertThat(publisherConfig.getPortNumber(), is(DMAAP_MR_SINK_PLUGIN_PORT_NUMBER)); + assertThat(publisherConfig.getProtocol(), is(DMAAP_MR_SINK_PLUGIN_PROTOCOL)); + assertThat(publisherConfig.getUserName(), is(DMAAP_MR_SINK_PLUGIN_USERNAME)); + assertThat(publisherConfig.getUserPassword(), is(DMAAP_MR_SINK_PLUGIN_PASSWORD)); + assertThat(publisherConfig.getContentType(), is(DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE)); + assertThat(publisherConfig.getMaxBatchSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE)); + assertThat(publisherConfig.getMaxRecoveryQueueSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE)); + + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java new file mode 100644 index 0000000..6e79f47 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java @@ -0,0 +1,64 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.utils; + +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.dmaap.domain.config.DMaaPMRSubscriberConfig; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author Rajiv Singla . Creation Date: 1/24/2017. + */ +public class DMaaPSourceConfigMapperTest extends BaseAnalyticsCDAPPluginsUnitTest { + + @Test + public void testMapToSubscriberConfig() throws Exception { + + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + final DMaaPMRSubscriberConfig subscriberConfig = DMaaPSourceConfigMapper.map(testDMaaPMRSourcePluginConfig); + + assertNotNull(subscriberConfig); + assertThat(subscriberConfig.getHostName(), is(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); + assertThat(subscriberConfig.getTopicName(), is(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME)); + assertThat(subscriberConfig.getPortNumber(), is(DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER)); + assertThat(subscriberConfig.getProtocol(), is(DMAAP_MR_SOURCE_PLUGIN_PROTOCOL)); + assertThat(subscriberConfig.getUserName(), is(DMAAP_MR_SOURCE_PLUGIN_USERNAME)); + assertThat(subscriberConfig.getUserPassword(), is(DMAAP_MR_SOURCE_PLUGIN_PASSWORD)); + assertThat(subscriberConfig.getContentType(), is(DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE)); + assertThat(subscriberConfig.getConsumerGroup(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP)); + assertThat(subscriberConfig.getConsumerId(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID)); + assertThat(subscriberConfig.getMessageLimit(), is(DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT)); + assertThat(subscriberConfig.getTimeoutMS(), is(DMAAP_MR_SOURCE_PLUGIN_TIMEOUT)); + } + + @Test(expected = IllegalStateException.class) + public void testMapToSubscriberConfigWhenSubscriberHostNameIsEmpty() throws Exception { + final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + testDMaaPMRSourcePluginConfig.setHostName(null); + DMaaPSourceConfigMapper.map(testDMaaPMRSourcePluginConfig); + + } +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java new file mode 100644 index 0000000..904a74b --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java @@ -0,0 +1,86 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import org.junit.Before; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSinkPluginConfig; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMRSinkPluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private TestDMaaPMRSinkPluginConfig sinkPluginConfig; + private DMaaPMRSinkPluginConfigValidator sinkPluginConfigValidator; + + @Before + public void before() { + sinkPluginConfigValidator = new DMaaPMRSinkPluginConfigValidator(); + sinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); + } + + @Test + public void validateAppSettingsWithValidDMaaPSinkConfig() throws Exception { + final GenericValidationResponse validationResponse = + sinkPluginConfigValidator.validateAppSettings(sinkPluginConfig); + assertFalse(validationResponse.hasErrors()); + } + + + @Test + public void validateAppSettingsWithValidDMaaPSinkConfigWhenHostNameIsNotPresent() throws Exception { + sinkPluginConfig.setHostName(null); + assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); + } + + @Test + public void validateAppSettingsWithValidDMaaPSinkConfigWhenHostPortIsNotPresent() throws Exception { + sinkPluginConfig.setPortNumber(null); + assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); + } + + @Test + public void validateAppSettingsWithValidDMaaPSinkConfigWhenTopicNameIsNotPresent() throws Exception { + sinkPluginConfig.setTopicName(null); + assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); + } + + @Test + public void validateAppSettingsWithValidDMaaPSinkConfigWhenColumnNameIsNotPresent() throws Exception { + sinkPluginConfig.setMessageColumnName(null); + assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); + } + + private static void assertResponseHasErrors(final TestDMaaPMRSinkPluginConfig sinkPluginConfig, + final DMaaPMRSinkPluginConfigValidator validator) { + final GenericValidationResponse validationResponse = validator.validateAppSettings(sinkPluginConfig); + assertTrue(validationResponse.hasErrors()); + } + + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java new file mode 100644 index 0000000..b4357e8 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java @@ -0,0 +1,85 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import org.junit.Before; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Rajiv Singla . Creation Date: 1/30/2017. + */ +public class DMaaPMRSourcePluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private TestDMaaPMRSourcePluginConfig sourcePluginConfig; + private DMaaPMRSourcePluginConfigValidator sourcePluginConfigValidator; + + @Before + public void before() { + sourcePluginConfigValidator = new DMaaPMRSourcePluginConfigValidator(); + sourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); + } + + @Test + public void validateAppSettingsWithValidDMaaPSourceConfig() throws Exception { + final GenericValidationResponse validationResponse = + sourcePluginConfigValidator.validateAppSettings(sourcePluginConfig); + assertFalse(validationResponse.hasErrors()); + } + + + @Test + public void validateAppSettingsWithValidDMaaPSourceConfigWhenHostNameIsNotPresent() throws Exception { + sourcePluginConfig.setHostName(null); + assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); + } + + @Test + public void validateAppSettingsWithValidDMaaPSourceConfigWhenHostPortIsNotPresent() throws Exception { + sourcePluginConfig.setPortNumber(null); + assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); + } + + @Test + public void validateAppSettingsWithValidDMaaPSourceConfigWhenTopicNameIsNotPresent() throws Exception { + sourcePluginConfig.setTopicName(null); + assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); + } + + @Test + public void validateAppSettingsWithValidDMaaPSourcePollingIntervalIsNotPresent() throws Exception { + sourcePluginConfig.setPollingInterval(null); + assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); + } + + private static void assertResponseHasErrors(final TestDMaaPMRSourcePluginConfig sourcePluginConfig, + final DMaaPMRSourcePluginConfigValidator validator) { + final GenericValidationResponse validationResponse = validator.validateAppSettings(sourcePluginConfig); + assertTrue(validationResponse.hasErrors()); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java new file mode 100644 index 0000000..4eff03c --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java @@ -0,0 +1,107 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import org.junit.Before; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter.JsonPathFilterPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.filter.TestJsonPathFilterPluginConfig; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Rajiv Singla . Creation Date: 3/3/2017. + */ +public class JsonPathFilterPluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private TestJsonPathFilterPluginConfig jsonPathFilterPluginConfig; + private JsonPathFilterPluginConfigValidator jsonPathFilterPluginConfigValidator; + + @Before + public void before() { + jsonPathFilterPluginConfig = getJsonPathFilterPluginConfig(); + jsonPathFilterPluginConfigValidator = new JsonPathFilterPluginConfigValidator(); + } + + + @Test + public void testValidateAppSettingsWhenNoValidationErrors() throws Exception { + final GenericValidationResponse validationResponse = + jsonPathFilterPluginConfigValidator.validateAppSettings(jsonPathFilterPluginConfig); + assertFalse(validationResponse.hasErrors()); + } + + @Test + public void testValidateAppSettingsWhenFilterMappingsAreEmpty() throws Exception { + jsonPathFilterPluginConfig.setJsonFilterMappings(""); + assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenOutputSchemaIsNotPresent() throws Exception { + jsonPathFilterPluginConfig.setSchema(null); + assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenOutputSchemaFilterMatchedFieldIsNotBoolean() throws Exception { + final String outputSchemaWithMatchedFieldNotBoolean = + "{\"type\":\"record\"," + + "\"name\":\"etlSchemaBody\",\"fields\":" + + "[" + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"filterMatched\",\"type\":[\"string\",\"null\"]}," + + "{\"name\":\"responseCode\",\"type\":\"int\"}," + + "{\"name\":\"responseMessage\",\"type\":\"string\"}," + + "{\"name\":\"message\",\"type\":\"string\"}" + + "]" + + "}"; + jsonPathFilterPluginConfig.setSchema(outputSchemaWithMatchedFieldNotBoolean); + assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenOutputSchemaFilterMatchedFieldIsNotNullable() throws Exception { + final String outputSchemaWithMatchedFieldNotNullable = + "{\"type\":\"record\"," + + "\"name\":\"etlSchemaBody\",\"fields\":" + + "[" + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"filterMatched\",\"type\":\"boolean\"}," + + "{\"name\":\"responseCode\",\"type\":\"int\"}," + + "{\"name\":\"responseMessage\",\"type\":\"string\"}," + + "{\"name\":\"message\",\"type\":\"string\"}" + + "]" + + "}"; + jsonPathFilterPluginConfig.setSchema(outputSchemaWithMatchedFieldNotNullable); + assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); + } + + private static void assertResponseHasErrors(final TestJsonPathFilterPluginConfig jsonPluginConfig, + final JsonPathFilterPluginConfigValidator validator) { + final GenericValidationResponse validationResponse = validator.validateAppSettings(jsonPluginConfig); + assertTrue(validationResponse.hasErrors()); + } + +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java new file mode 100644 index 0000000..284bec3 --- /dev/null +++ b/dcae-analytics-cdap-plugins/src/test/java/org/onap/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java @@ -0,0 +1,157 @@ +/* + * ===============================LICENSE_START====================================== + * dcae-analytics + * ================================================================================ + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================LICENSE_END=========================================== + */ + +package org.onap.dcae.apod.analytics.cdap.plugins.validator; + +import org.junit.Before; +import org.junit.Test; +import org.onap.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; +import org.onap.dcae.apod.analytics.cdap.plugins.domain.config.tca.TestSimpleTCAPluginConfig; +import org.onap.dcae.apod.analytics.common.validation.GenericValidationResponse; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Rajiv Singla . Creation Date: 2/21/2017. + */ +public class SimpleTCAPluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { + + private TestSimpleTCAPluginConfig testSimpleTCAPluginConfig; + private SimpleTCAPluginConfigValidator simpleTCAPluginConfigValidator; + + @Before + public void before() { + testSimpleTCAPluginConfig = getTestSimpleTCAPluginConfig(); + simpleTCAPluginConfigValidator = new SimpleTCAPluginConfigValidator(); + } + + @Test + public void testValidateAppSettingsWhenAllSettingsAreValid() throws Exception { + final GenericValidationResponse validationResponse = + simpleTCAPluginConfigValidator.validateAppSettings(testSimpleTCAPluginConfig); + assertFalse(validationResponse.hasErrors()); + } + + @Test + public void testValidateAppSettingsWhenVESMessageFieldNameIsMissing() throws Exception { + testSimpleTCAPluginConfig.setVesMessageFieldName(null); + assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenPolicyJsonIsMissing() throws Exception { + testSimpleTCAPluginConfig.setPolicyJson(null); + assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenAlertFieldNameIsMissing() throws Exception { + testSimpleTCAPluginConfig.setAlertFieldName(null); + assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenOutputSchemaIsNull() throws Exception { + testSimpleTCAPluginConfig.setSchema(null); + assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenMessageTypeFieldNameIsMissing() throws Exception { + testSimpleTCAPluginConfig.setMessageTypeFieldName(null); + assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenAlertFieldIsNullableInOutputSchema() throws Exception { + testSimpleTCAPluginConfig.setSchema( + "{\"type\":\"record\"," + + "\"name\":\"etlSchemaBody\"," + + "\"fields\":[" + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"responseCode\",\"type\":\"int\"}," + + "{\"name\":\"responseMessage\",\"type\":\"string\"}," + + "{\"name\":\"message\",\"type\":\"string\"}," + + "{\"name\":\"alert\",\"type\":[\"string\",\"null\"]}," + + "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); + final GenericValidationResponse validationResponse = + simpleTCAPluginConfigValidator.validateAppSettings(testSimpleTCAPluginConfig); + assertFalse(validationResponse.hasErrors()); + + } + + @Test + public void testValidateAppSettingsWhenAlertFieldIsNotPresentInOutputSchema() throws Exception { + testSimpleTCAPluginConfig.setSchema( + "{\"type\":\"record\"," + + "\"name\":\"etlSchemaBody\"," + + "\"fields\":[" + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"responseCode\",\"type\":\"int\"}," + + "{\"name\":\"responseMessage\",\"type\":\"string\"}," + + "{\"name\":\"message\",\"type\":\"string\"}," + + "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); + final GenericValidationResponse validationResponse = + simpleTCAPluginConfigValidator.validateAppSettings(testSimpleTCAPluginConfig); + assertFalse(validationResponse.hasErrors()); + + } + + @Test + public void testValidateAppSettingsWhenAlertFieldIsNullableButNotStringTypeInOutputSchema() throws Exception { + testSimpleTCAPluginConfig.setSchema( + "{\"type\":\"record\"," + + "\"name\":\"etlSchemaBody\"," + + "\"fields\":[" + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"responseCode\",\"type\":\"int\"}," + + "{\"name\":\"responseMessage\",\"type\":\"string\"}," + + "{\"name\":\"message\",\"type\":\"string\"}," + + "{\"name\":\"alert\",\"type\":[\"int\",\"null\"]}," + + "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); + assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); + } + + @Test + public void testValidateAppSettingsWhenAlertFieldNameIsNotNullableInOutputSchema() throws Exception { + testSimpleTCAPluginConfig.setSchema( + "{\"type\":\"record\"," + + "\"name\":\"etlSchemaBody\"," + + "\"fields\":[" + + "{\"name\":\"ts\",\"type\":\"long\"}," + + "{\"name\":\"responseCode\",\"type\":\"int\"}," + + "{\"name\":\"responseMessage\",\"type\":\"string\"}," + + "{\"name\":\"message\",\"type\":\"string\"}," + + "{\"name\":\"alert\",\"type\":\"string\"}," + + "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); + assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); + } + + + + private static void assertResponseHasErrors(final TestSimpleTCAPluginConfig pluginConfig, + final SimpleTCAPluginConfigValidator validator) { + final GenericValidationResponse validationResponse = validator.validateAppSettings(pluginConfig); + assertTrue(validationResponse.hasErrors()); + LOG.debug("Validation Error Message: {}", validationResponse.getAllErrorMessage()); + } +} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java deleted file mode 100644 index cba456b..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/BaseAnalyticsCDAPPluginsUnitTest.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins; - -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.StageMetrics; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Suppliers; -import org.apache.hadoop.conf.Configuration; -import org.openecomp.dcae.apod.analytics.cdap.common.CDAPPluginConstants; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSinkPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter.TestJsonPathFilterPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.TestSimpleTCAPluginConfig; -import org.openecomp.dcae.apod.analytics.model.util.json.AnalyticsModelObjectMapperSupplier; -import org.openecomp.dcae.apod.analytics.test.BaseDCAEAnalyticsUnitTest; - -import java.io.IOException; -import java.io.Serializable; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * @author Rajiv Singla . Creation Date: 1/23/2017. - */ -public abstract class BaseAnalyticsCDAPPluginsUnitTest extends BaseDCAEAnalyticsUnitTest { - - protected static final ObjectMapper ANALYTICS_MODEL_OBJECT_MAPPER = - Suppliers.memoize(new AnalyticsModelObjectMapperSupplier()).get(); - - protected static final String TCA_POLICY_JSON_FILE_LOCATION = "data/json/policy/tca_policy.json"; - protected static final String CEF_MESSAGE_JSON_FILE_LOCATION = "data/json/cef/cef_message.json"; - protected static final String CEF_NON_COMPLIANT_MESSAGE_JSON_FILE_LOCATION = - "data/json/cef/cef_message_with_threshold_violation.json"; - - - protected static final String DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME = "testDMaaPMRSource"; - protected static final String DMAAP_MR_SOURCE_PLUGIN_HOST_NAME = "dcae-msrt-mtl1-ftl.homer.com"; - protected static final Integer DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER = 3905; - protected static final String DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME = "com.dcae.dmaap.FTL.DcaeTestVESSub"; - protected static final Integer DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL = 1000; - protected static final String DMAAP_MR_SOURCE_PLUGIN_PROTOCOL = "https"; - protected static final String DMAAP_MR_SOURCE_PLUGIN_USERNAME = "username"; - protected static final String DMAAP_MR_SOURCE_PLUGIN_PASSWORD = "password"; - protected static final String DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE = "application/json"; - protected static final String DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP = "G1"; - protected static final String DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID = "C1"; - protected static final Integer DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT = 100; - protected static final Integer DMAAP_MR_SOURCE_PLUGIN_TIMEOUT = 10000; - - - protected static final String DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME = "testDMaaPMRSINK"; - protected static final String DMAAP_MR_SINK_PLUGIN_HOST_NAME = "dcae-msrt-mtl1-ftl.homer.com"; - protected static final Integer DMAAP_MR_SINK_PLUGIN_PORT_NUMBER = 3905; - protected static final String DMAAP_MR_SINK_PLUGIN_TOPIC_NAME = "com.dcae.dmaap.FTL.DcaeTestVESPub"; - protected static final String DMAAP_MR_SINK_PLUGIN_PROTOCOL = "https"; - protected static final String DMAAP_MR_SINK_PLUGIN_USERNAME = "username"; - protected static final String DMAAP_MR_SINK_PLUGIN_PASSWORD = "password"; - protected static final String DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE = "application/json"; - protected static final String DMAAP_MR_SINK_MESSAGE_COLUMN_NAME = "message"; - protected static final Integer DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE = 10; - protected static final Integer DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE = 100; - - protected static final String VES_MESSAGE_FIELD_NAME = "message"; - protected static final String TCA_PLUGIN_ALERT_FIELD_NAME = "alert"; - protected static final String TCA_PLUGIN_MESSAGE_TYPE_FIELD_NAME = "tcaMessageType"; - - - protected static final String JSON_PATH_FILTER_PLUGIN_REFERENCE_NAME = "JsonPathFilter"; - protected static final String JSON_PATH_FILTER_PLUGIN_INCOMING_JSON_FIELD_NAME = "message"; - protected static final String JSON_PATH_FILTER_PLUGIN_OUTPUT_SCHEMA_FILED_NAME = "filterMatched"; - protected static final String JSON_PATH_FILTER_PLUGIN_JSON_FILTER_MAPPINGS = - "$.event.commonEventHeader.domain:measurementsForVfScaling," + - "$.event.commonEventHeader.eventName:vLoadBalancer;vFirewall"; - protected static final String JSON_PATH_FILTER_PLUGIN_JSON_FILTER_OUTPUT_SCHEMA = - "{\"type\":\"record\"," + - "\"name\":\"etlSchemaBody\",\"fields\":" + - "[" + - "{\"name\":\"ts\",\"type\":\"long\"}," + - "{\"name\":\"filterMatched\",\"type\":[\"boolean\",\"null\"]}," + - "{\"name\":\"responseCode\",\"type\":\"int\"}," + - "{\"name\":\"responseMessage\",\"type\":\"string\"}," + - "{\"name\":\"message\",\"type\":\"string\"}" + - "]" + - "}"; - - protected static class MockStageMetrics implements StageMetrics, Serializable { - - @Override - public void count(String metricName, int delta) { - LOG.debug("Mocking metric count, MetricName: {}, Delta: {}", metricName, delta); - } - - @Override - public void gauge(String metricName, long value) { - LOG.debug("Mocking metric guage, MetricName: {}, Value: {}", metricName, value); - } - - @Override - public void pipelineCount(String metricName, int delta) { - LOG.debug("Mocking metric pipelineCount, MetricName: {}, Delta: {}", metricName, delta); - } - - @Override - public void pipelineGauge(String metricName, long value) { - LOG.debug("Mocking metric guage, pipelineGauge: {}, Value: {}", metricName, value); - } - } - - protected static TestDMaaPMRSourcePluginConfig getTestDMaaPMRSourcePluginConfig() { - final TestDMaaPMRSourcePluginConfig sourcePluginConfig = new TestDMaaPMRSourcePluginConfig(); - sourcePluginConfig.setReferenceName(DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME); - sourcePluginConfig.setHostName(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME); - sourcePluginConfig.setPortNumber(DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER); - sourcePluginConfig.setTopicName(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME); - sourcePluginConfig.setPollingInterval(DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL); - sourcePluginConfig.setProtocol(DMAAP_MR_SOURCE_PLUGIN_PROTOCOL); - sourcePluginConfig.setUserName(DMAAP_MR_SOURCE_PLUGIN_USERNAME); - sourcePluginConfig.setUserPassword(DMAAP_MR_SOURCE_PLUGIN_PASSWORD); - sourcePluginConfig.setContentType(DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE); - sourcePluginConfig.setConsumerGroup(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP); - sourcePluginConfig.setConsumerId(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID); - sourcePluginConfig.setMessageLimit(DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT); - sourcePluginConfig.setTimeoutMS(DMAAP_MR_SOURCE_PLUGIN_TIMEOUT); - return sourcePluginConfig; - } - - protected static TestDMaaPMRSinkPluginConfig getTestDMaaPMRSinkPluginConfig() { - final TestDMaaPMRSinkPluginConfig sinkPluginConfig = new TestDMaaPMRSinkPluginConfig(); - sinkPluginConfig.setReferenceName(DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME); - sinkPluginConfig.setHostName(DMAAP_MR_SINK_PLUGIN_HOST_NAME); - sinkPluginConfig.setPortNumber(DMAAP_MR_SINK_PLUGIN_PORT_NUMBER); - sinkPluginConfig.setTopicName(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME); - sinkPluginConfig.setProtocol(DMAAP_MR_SINK_PLUGIN_PROTOCOL); - sinkPluginConfig.setUserName(DMAAP_MR_SINK_PLUGIN_USERNAME); - sinkPluginConfig.setUserPassword(DMAAP_MR_SINK_PLUGIN_PASSWORD); - sinkPluginConfig.setContentType(DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE); - sinkPluginConfig.setMessageColumnName(DMAAP_MR_SINK_MESSAGE_COLUMN_NAME); - sinkPluginConfig.setMaxBatchSize(DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE); - sinkPluginConfig.setMaxRecoveryQueueSize(DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE); - return sinkPluginConfig; - } - - - protected static Configuration getTestConfiguration() { - final Configuration configuration = new Configuration(); - final Map sinkConfigurationMap = createSinkConfigurationMap(); - for (Map.Entry property : sinkConfigurationMap.entrySet()) { - configuration.set(property.getKey(), property.getValue()); - } - return configuration; - } - - protected static Map createSinkConfigurationMap() { - - Map sinkConfig = new LinkedHashMap<>(); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.HOST_NAME, DMAAP_MR_SINK_PLUGIN_HOST_NAME); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.TOPIC_NAME, DMAAP_MR_SINK_PLUGIN_TOPIC_NAME); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.PORT_NUMBER, - DMAAP_MR_SINK_PLUGIN_PORT_NUMBER.toString()); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.PROTOCOL, DMAAP_MR_SINK_PLUGIN_PROTOCOL); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.USER_NAME, DMAAP_MR_SINK_PLUGIN_USERNAME); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.USER_PASS, DMAAP_MR_SINK_PLUGIN_PASSWORD); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.CONTENT_TYPE, - DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.MAX_BATCH_SIZE, - DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE.toString()); - sinkConfig.put(CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields.MAX_RECOVER_QUEUE_SIZE, - DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE.toString()); - return sinkConfig; - } - - protected static Schema getDMaaPMRSinkTestSchema() { - return Schema.recordOf( - "DMaaPMRSinkTestSchema", - Schema.Field.of("message", Schema.of(Schema.Type.STRING)), - Schema.Field.of("field1", Schema.of(Schema.Type.STRING)) - ); - } - - - protected static TestSimpleTCAPluginConfig getTestSimpleTCAPluginConfig() { - final String policyJson; - try { - policyJson = fromStream(TCA_POLICY_JSON_FILE_LOCATION); - } catch (IOException e) { - throw new RuntimeException("Error while parsing policy", e); - } - return new TestSimpleTCAPluginConfig(VES_MESSAGE_FIELD_NAME, policyJson, TCA_PLUGIN_ALERT_FIELD_NAME, - TCA_PLUGIN_MESSAGE_TYPE_FIELD_NAME, getSimpleTCAPluginInputSchema().toString(), false); - } - - protected static Schema getSimpleTCAPluginInputSchema() { - return Schema.recordOf( - "TestSimpleTCAPluginInputSchema", - Schema.Field.of("message", Schema.of(Schema.Type.STRING)), - Schema.Field.of("inputField1", Schema.nullableOf(Schema.of(Schema.Type.STRING))), - Schema.Field.of("inputField2", Schema.nullableOf(Schema.of(Schema.Type.STRING))) - ); - } - - protected static Schema getJsonFilterPluginInputSchema() { - return Schema.recordOf( - "TestJsonFilterInputSchema", - Schema.Field.of("ts", Schema.of(Schema.Type.LONG)), - Schema.Field.of("responseCode", Schema.of(Schema.Type.INT)), - Schema.Field.of("responseMessage", Schema.of(Schema.Type.STRING)), - Schema.Field.of("message", Schema.of(Schema.Type.STRING)) - ); - } - - protected static TestJsonPathFilterPluginConfig getJsonPathFilterPluginConfig() { - return new TestJsonPathFilterPluginConfig(JSON_PATH_FILTER_PLUGIN_REFERENCE_NAME, - JSON_PATH_FILTER_PLUGIN_INCOMING_JSON_FIELD_NAME, - JSON_PATH_FILTER_PLUGIN_OUTPUT_SCHEMA_FILED_NAME, - JSON_PATH_FILTER_PLUGIN_JSON_FILTER_MAPPINGS, - JSON_PATH_FILTER_PLUGIN_JSON_FILTER_OUTPUT_SCHEMA); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java deleted file mode 100644 index beeb19b..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatProviderTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.common.CDAPPluginConstants.DMaaPMRSinkHadoopConfigFields; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSinkPluginConfig; -import org.openecomp.dcae.apod.analytics.common.AnalyticsConstants; - -import java.util.Map; - -import static org.junit.Assert.assertTrue; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMROutputFormatProviderTest extends BaseAnalyticsCDAPPluginsUnitTest { - - - @Test - public void testDMaaPMROutputFormatProviderWhenConfigIsMissingNonRequiredValues() throws Exception { - final TestDMaaPMRSinkPluginConfig sinkPluginConfig = new TestDMaaPMRSinkPluginConfig(); - sinkPluginConfig.setHostName(DMAAP_MR_SINK_PLUGIN_HOST_NAME); - sinkPluginConfig.setTopicName(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME); - final DMaaPMROutputFormatProvider dMaaPMROutputFormatProvider = - new DMaaPMROutputFormatProvider(sinkPluginConfig); - final Map outputFormatConfiguration = - dMaaPMROutputFormatProvider.getOutputFormatConfiguration(); - final String hostName = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.HOST_NAME); - assertTrue(hostName.equals(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); - final String topicName = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.TOPIC_NAME); - assertTrue(topicName.equals(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); - final String portNumber = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.PORT_NUMBER); - assertTrue(portNumber.equals(AnalyticsConstants.DEFAULT_PORT_NUMBER.toString())); - final String protocol = outputFormatConfiguration.get(DMaaPMRSinkHadoopConfigFields.PROTOCOL); - assertTrue(protocol.equals(AnalyticsConstants.DEFAULT_PROTOCOL)); - } - - @Test - public void testGetOutputFormatClassName() throws Exception { - final DMaaPMROutputFormatProvider dMaaPMROutputFormatProvider = - new DMaaPMROutputFormatProvider(getTestDMaaPMRSinkPluginConfig()); - final String outputFormatClassName = dMaaPMROutputFormatProvider.getOutputFormatClassName(); - assertTrue(outputFormatClassName.equals(DMaaPMROutputFormat.class.getName())); - } - - @Test - public void testGetOutputFormatConfiguration() throws Exception { - final TestDMaaPMRSinkPluginConfig testDMaaPMRSinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); - final DMaaPMROutputFormatProvider dMaaPMROutputFormatProvider = - new DMaaPMROutputFormatProvider(testDMaaPMRSinkPluginConfig); - final Map outputFormatConfiguration = - dMaaPMROutputFormatProvider.getOutputFormatConfiguration(); - assertTrue(outputFormatConfiguration.size() == 9); - - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java deleted file mode 100644 index 97e16c3..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMROutputFormatTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.when; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMROutputFormatTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private DMaaPMROutputFormat dMaaPMROutputFormat; - - @Before - public void before() { - dMaaPMROutputFormat = new DMaaPMROutputFormat(); - } - - @Test - public void testGetRecordWriter() throws Exception { - final TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class); - when(taskAttemptContext.getConfiguration()).thenReturn(getTestConfiguration()); - final RecordWriter recordWriter = dMaaPMROutputFormat.getRecordWriter(taskAttemptContext); - assertNotNull(recordWriter); - final JobContext jobContext = Mockito.mock(JobContext.class); - dMaaPMROutputFormat.checkOutputSpecs(jobContext); - } - - @Test - public void testGetOutputCommitter() throws Exception { - final TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class); - final OutputCommitter outputCommitter = dMaaPMROutputFormat.getOutputCommitter(taskAttemptContext); - assertTrue(outputCommitter.getClass().equals(DMaaPMROutputFormat.NoOpOutputCommitter.class)); - final JobContext jobContext = Mockito.mock(JobContext.class); - outputCommitter.setupJob(jobContext); - outputCommitter.setupTask(taskAttemptContext); - assertFalse(outputCommitter.needsTaskCommit(taskAttemptContext)); - outputCommitter.commitJob(jobContext); - outputCommitter.commitTask(taskAttemptContext); - outputCommitter.abortTask(taskAttemptContext); - - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java deleted file mode 100644 index 611392c..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRRecordWriterTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.dmaap.service.publisher.DMaaPMRPublisher; - -import java.util.Arrays; - -import static org.mockito.Mockito.times; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMRRecordWriterTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private DMaaPMRPublisher publisher; - private DMaaPMRRecordWriter dMaaPMRRecordWriter; - - @Before - public void before() { - publisher = Mockito.mock(DMaaPMRPublisher.class); - dMaaPMRRecordWriter = new DMaaPMRRecordWriter(publisher); - } - - @Test - public void testWrite() throws Exception { - final String testMessage = "test Message"; - dMaaPMRRecordWriter.write(testMessage, null); - Mockito.verify(publisher, times(1)).publish(Arrays.asList(testMessage)); - } - - @Test - public void testClose() throws Exception { - final TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class); - dMaaPMRRecordWriter.close(taskAttemptContext); - Mockito.verify(publisher, times(1)).flush(); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java deleted file mode 100644 index 3ad16d9..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/batch/sink/dmaap/DMaaPMRSinkTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.batch.sink.dmaap; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.Emitter; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.StageConfigurer; -import co.cask.cdap.etl.api.batch.BatchSinkContext; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMRSinkTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private DMaaPMRSink dMaaPMRSink; - - @Before - public void before() { - dMaaPMRSink = new DMaaPMRSink(getTestDMaaPMRSinkPluginConfig()); - } - - @Test - public void testConfigurePipeline() throws Exception { - final PipelineConfigurer pipelineConfigurer = Mockito.mock(PipelineConfigurer.class); - final StageConfigurer stageConfigurer = Mockito.mock(StageConfigurer.class); - when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); - when(stageConfigurer.getInputSchema()).thenReturn(getDMaaPMRSinkTestSchema()); - dMaaPMRSink.configurePipeline(pipelineConfigurer); - verify(stageConfigurer, times(1)).getInputSchema(); - } - - @Test(expected = CDAPSettingsException.class) - public void testConfigurePipelineWithInvalidSchema() throws Exception { - final PipelineConfigurer pipelineConfigurer = Mockito.mock(PipelineConfigurer.class); - final StageConfigurer stageConfigurer = Mockito.mock(StageConfigurer.class); - when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); - when(stageConfigurer.getInputSchema()).thenReturn(Schema.recordOf( - "DMaaPMRSinkInvalidSchema", - Schema.Field.of("message1", Schema.of(Schema.Type.STRING)), - Schema.Field.of("field1", Schema.of(Schema.Type.STRING)) - )); - dMaaPMRSink.configurePipeline(pipelineConfigurer); - } - - @Test - public void testPrepareRun() throws Exception { - final BatchSinkContext batchSinkContext = Mockito.mock(BatchSinkContext.class); - dMaaPMRSink.prepareRun(batchSinkContext); - } - - @Test - @SuppressWarnings("unchecked") - public void testTransform() throws Exception { - final StructuredRecord structuredRecord = Mockito.mock(StructuredRecord.class); - final Emitter emitter = Mockito.mock(Emitter.class); - final String incomingTestMessage = "test message"; - when(structuredRecord.get( - eq(getTestDMaaPMRSinkPluginConfig().getMessageColumnName()))).thenReturn(incomingTestMessage); - doNothing().when(emitter).emit(any()); - dMaaPMRSink.transform(structuredRecord, emitter); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java deleted file mode 100644 index d1fffa3..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSinkPluginConfigTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; - -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Rajiv Singla . Creation Date: 1/23/2017. - */ -public class DMaaPMRSinkPluginConfigTest extends BaseAnalyticsCDAPPluginsUnitTest { - - @Test - public void testDMaaPMRSinkPluginConfigDefaults() throws Exception { - final DMaaPMRSinkPluginConfig sinkPluginConfig = new DMaaPMRSinkPluginConfig - (DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME, DMAAP_MR_SINK_PLUGIN_HOST_NAME, - DMAAP_MR_SINK_PLUGIN_TOPIC_NAME, DMAAP_MR_SINK_MESSAGE_COLUMN_NAME); - - assertThat(sinkPluginConfig.getReferenceName(), is(DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME)); - assertThat(sinkPluginConfig.getHostName(), is(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); - assertThat(sinkPluginConfig.getTopicName(), is(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); - assertThat(sinkPluginConfig.getMessageColumnName(), is(DMAAP_MR_SINK_MESSAGE_COLUMN_NAME)); - assertNull(sinkPluginConfig.getPortNumber()); - assertNull(sinkPluginConfig.getProtocol()); - assertNull(sinkPluginConfig.getUserName()); - assertNull(sinkPluginConfig.getUserPassword()); - assertNull(sinkPluginConfig.getContentType()); - assertNull(sinkPluginConfig.getMaxBatchSize()); - assertNull(sinkPluginConfig.getMaxRecoveryQueueSize()); - } - - @Test - public void testDMaaPMRSinkPluginConfigCustom() throws Exception { - final DMaaPMRSinkPluginConfig sinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); - assertThat(sinkPluginConfig.getReferenceName(), is(DMAAP_MR_SINK_PLUGIN_REFERENCE_NAME)); - assertThat(sinkPluginConfig.getHostName(), is(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); - assertThat(sinkPluginConfig.getTopicName(), is(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); - assertThat(sinkPluginConfig.getPortNumber(), is(DMAAP_MR_SINK_PLUGIN_PORT_NUMBER)); - assertThat(sinkPluginConfig.getProtocol(), is(DMAAP_MR_SINK_PLUGIN_PROTOCOL)); - assertThat(sinkPluginConfig.getUserName(), is(DMAAP_MR_SINK_PLUGIN_USERNAME)); - assertThat(sinkPluginConfig.getUserPassword(), is(DMAAP_MR_SINK_PLUGIN_PASSWORD)); - assertThat(sinkPluginConfig.getContentType(), is(DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE)); - assertThat(sinkPluginConfig.getMessageColumnName(), is(DMAAP_MR_SINK_MESSAGE_COLUMN_NAME)); - assertThat(sinkPluginConfig.getMaxBatchSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE)); - assertThat(sinkPluginConfig.getMaxRecoveryQueueSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE)); - } - - @Test - public void testValidToString() throws Exception { - final TestDMaaPMRSinkPluginConfig sinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); - assertNotNull(sinkPluginConfig.toString()); - assertTrue(sinkPluginConfig.toString().contains(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); - } - - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java deleted file mode 100644 index 2302210..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/DMaaPMRSourcePluginConfigTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; - -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - - -/** - * @author Rajiv Singla . Creation Date: 1/23/2017. - */ -public class DMaaPMRSourcePluginConfigTest extends BaseAnalyticsCDAPPluginsUnitTest { - - @Test - public void testDMaaPMRSourcePluginConfigDefaults() throws Exception { - final DMaaPMRSourcePluginConfig sourcePluginConfig = new DMaaPMRSourcePluginConfig - (DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME, DMAAP_MR_SOURCE_PLUGIN_HOST_NAME, - DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME, DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL); - - assertThat(sourcePluginConfig.getReferenceName(), is(DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME)); - assertThat(sourcePluginConfig.getHostName(), is(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); - assertThat(sourcePluginConfig.getTopicName(), is(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME)); - assertThat(sourcePluginConfig.getPollingInterval(), is(DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL)); - assertNull(sourcePluginConfig.getPortNumber()); - assertNull(sourcePluginConfig.getProtocol()); - assertNull(sourcePluginConfig.getUserName()); - assertNull(sourcePluginConfig.getUserPassword()); - assertNull(sourcePluginConfig.getContentType()); - assertNull(sourcePluginConfig.getConsumerGroup()); - assertNull(sourcePluginConfig.getConsumerId()); - assertNull(sourcePluginConfig.getMessageLimit()); - assertNull(sourcePluginConfig.getTimeoutMS()); - } - - @Test - public void testDMaaPMRSourcePluginConfigCustom() throws Exception { - final TestDMaaPMRSourcePluginConfig sourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - assertThat(sourcePluginConfig.getReferenceName(), is(DMAAP_MR_SOURCE_PLUGIN_REFERENCE_NAME)); - assertThat(sourcePluginConfig.getHostName(), is(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); - assertThat(sourcePluginConfig.getTopicName(), is(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME)); - assertThat(sourcePluginConfig.getPollingInterval(), is(DMAAP_MR_SOURCE_PLUGIN_POLLING_INTERVAL)); - assertThat(sourcePluginConfig.getPortNumber(), is(DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER)); - assertThat(sourcePluginConfig.getProtocol(), is(DMAAP_MR_SOURCE_PLUGIN_PROTOCOL)); - assertThat(sourcePluginConfig.getUserName(), is(DMAAP_MR_SOURCE_PLUGIN_USERNAME)); - assertThat(sourcePluginConfig.getUserPassword(), is(DMAAP_MR_SOURCE_PLUGIN_PASSWORD)); - assertThat(sourcePluginConfig.getContentType(), is(DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE)); - assertThat(sourcePluginConfig.getConsumerGroup(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP)); - assertThat(sourcePluginConfig.getConsumerId(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID)); - assertThat(sourcePluginConfig.getMessageLimit(), is(DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT)); - assertThat(sourcePluginConfig.getTimeoutMS(), is(DMAAP_MR_SOURCE_PLUGIN_TIMEOUT)); - } - - @Test - public void testValidToString() throws Exception { - final TestDMaaPMRSourcePluginConfig sourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - assertNotNull(sourcePluginConfig.toString()); - assertTrue(sourcePluginConfig.toString().contains(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java deleted file mode 100644 index 47c832e..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSinkPluginConfig.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; - -import javax.annotation.Nullable; - -/** - * Test {@link DMaaPMRSinkPluginConfig} for testing purposes only - *

- * @author Rajiv Singla . Creation Date: 1/23/2017. - */ -public class TestDMaaPMRSinkPluginConfig extends DMaaPMRSinkPluginConfig { - - public void setReferenceName(String referenceName) { - this.referenceName = referenceName; - } - - public void setHostName(String hostName) { - this.hostName = hostName; - } - - public void setPortNumber(@Nullable Integer portNumber) { - this.portNumber = portNumber; - } - - public void setTopicName(String topicName) { - this.topicName = topicName; - } - - public void setProtocol(@Nullable String protocol) { - this.protocol = protocol; - } - - public void setUserName(@Nullable String userName) { - this.userName = userName; - } - - public void setUserPassword(@Nullable String userPassword) { - this.userPassword = userPassword; - } - - public void setContentType(@Nullable String contentType) { - this.contentType = contentType; - } - - public void setMaxBatchSize(@Nullable Integer maxBatchSize) { - this.maxBatchSize = maxBatchSize; - } - - public void setMaxRecoveryQueueSize(@Nullable Integer maxRecoveryQueueSize) { - this.maxRecoveryQueueSize = maxRecoveryQueueSize; - } - - public void setMessageColumnName(String messageColumnName) { - this.messageColumnName = messageColumnName; - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java deleted file mode 100644 index 68cd470..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/dmaap/TestDMaaPMRSourcePluginConfig.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap; - -import javax.annotation.Nullable; - -/** - * Test {@link DMaaPMRSourcePluginConfig} for testing purposes only - *

- * @author Rajiv Singla . Creation Date: 1/23/2017. - */ -public class TestDMaaPMRSourcePluginConfig extends DMaaPMRSourcePluginConfig { - - public void setReferenceName(String referenceName) { - this.referenceName = referenceName; - } - - public void setHostName(String hostName) { - this.hostName = hostName; - } - - public void setPortNumber(@Nullable Integer portNumber) { - this.portNumber = portNumber; - } - - public void setTopicName(String topicName) { - this.topicName = topicName; - } - - public void setPollingInterval(Integer pollingInterval) { - this.pollingInterval = pollingInterval; - } - - public void setProtocol(@Nullable String protocol) { - this.protocol = protocol; - } - - public void setUserName(@Nullable String userName) { - this.userName = userName; - } - - public void setUserPassword(@Nullable String userPassword) { - this.userPassword = userPassword; - } - - public void setContentType(@Nullable String contentType) { - this.contentType = contentType; - } - - public void setConsumerId(@Nullable String consumerId) { - this.consumerId = consumerId; - } - - public void setConsumerGroup(@Nullable String consumerGroup) { - this.consumerGroup = consumerGroup; - } - - public void setTimeoutMS(@Nullable Integer timeoutMS) { - this.timeoutMS = timeoutMS; - } - - public void setMessageLimit(@Nullable Integer messageLimit) { - this.messageLimit = messageLimit; - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java deleted file mode 100644 index d503c7e..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/filter/TestJsonPathFilterPluginConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter; - -/** - * @author Rajiv Singla . Creation Date: 3/3/2017. - */ -public class TestJsonPathFilterPluginConfig extends JsonPathFilterPluginConfig { - - public TestJsonPathFilterPluginConfig(final String referenceName, final String incomingJsonFieldName, - final String outputSchemaFieldName, final String jsonFilterMappings, - final String schema) { - super(referenceName, incomingJsonFieldName, outputSchemaFieldName, jsonFilterMappings, schema); - } - - - public void setIncomingJsonFieldName(String incomingJsonFieldName) { - this.incomingJsonFieldName = incomingJsonFieldName; - } - - public void setOutputSchemaFieldName(String outputSchemaFieldName) { - this.outputSchemaFieldName = outputSchemaFieldName; - } - - public void setJsonFilterMappings(String jsonFilterMappings) { - this.jsonFilterMappings = jsonFilterMappings; - } - - public void setSchema(String schema) { - this.schema = schema; - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java deleted file mode 100644 index f06dcf0..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/config/tca/TestSimpleTCAPluginConfig.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca; - -/** - * @author Rajiv Singla . Creation Date: 2/17/2017. - */ -public class TestSimpleTCAPluginConfig extends SimpleTCAPluginConfig { - - public TestSimpleTCAPluginConfig(String vesMessageFieldName, String policyJson, String alertFieldName, - String messageTypeFieldName, String schema, Boolean enableAlertCEFFormat) { - super(vesMessageFieldName, policyJson, alertFieldName, messageTypeFieldName, schema, enableAlertCEFFormat); - } - - public void setVesMessageFieldName(String vesMessageFieldName) { - this.vesMessageFieldName = vesMessageFieldName; - } - - public void setPolicyJson(String policyJson) { - this.policyJson = policyJson; - } - - public void setAlertFieldName(String alertFieldName) { - this.alertFieldName = alertFieldName; - } - - public void setMessageTypeFieldName(String messageTypeFieldName) { - this.messageTypeFieldName = messageTypeFieldName; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public void setEnableAlertCEFFormat(Boolean enableAlertCEFFormat) { - this.enableAlertCEFFormat = enableAlertCEFFormat; - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java deleted file mode 100644 index 2f902b1..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/domain/schema/dmaap/DMaaPSourceOutputSchemaTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap; - -import co.cask.cdap.api.data.schema.Schema; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; - -import java.util.LinkedList; -import java.util.List; - -import static org.hamcrest.CoreMatchers.hasItems; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Rajiv Singla . Creation Date: 1/25/2017. - */ -public class DMaaPSourceOutputSchemaTest extends BaseAnalyticsCDAPPluginsUnitTest { - - - @Test - public void testGetSchemaColumnName() throws Exception { - assertThat(DMaaPSourceOutputSchema.TIMESTAMP.getSchemaColumnName(), is("ts")); - assertThat(DMaaPSourceOutputSchema.RESPONSE_CODE.getSchemaColumnName(), is("responseCode")); - assertThat(DMaaPSourceOutputSchema.RESPONSE_MESSAGE.getSchemaColumnName(), is("responseMessage")); - assertThat(DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName(), is("message")); - } - - @Test - public void testGetSchema() throws Exception { - final Schema schema = DMaaPSourceOutputSchema.getSchema(); - final List fields = schema.getFields(); - final List fieldNames = new LinkedList<>(); - for (Schema.Field field : fields) { - fieldNames.add(field.getName()); - } - assertThat(fieldNames, hasItems( - DMaaPSourceOutputSchema.TIMESTAMP.getSchemaColumnName(), - DMaaPSourceOutputSchema.RESPONSE_CODE.getSchemaColumnName(), - DMaaPSourceOutputSchema.RESPONSE_MESSAGE.getSchemaColumnName(), - DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName())); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java deleted file mode 100644 index f43a3df..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/it/SimpleTCAPluginCDAPIT.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.it; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.api.dataset.table.Table; -import co.cask.cdap.api.plugin.PluginClass; -import co.cask.cdap.api.plugin.PluginPropertyField; -import co.cask.cdap.common.utils.Tasks; -import co.cask.cdap.datapipeline.DataPipelineApp; -import co.cask.cdap.datapipeline.SmartWorkflow; -import co.cask.cdap.etl.api.batch.SparkCompute; -import co.cask.cdap.etl.mock.batch.MockSink; -import co.cask.cdap.etl.mock.batch.MockSource; -import co.cask.cdap.etl.mock.test.HydratorTestBase; -import co.cask.cdap.etl.proto.v2.ETLBatchConfig; -import co.cask.cdap.etl.proto.v2.ETLPlugin; -import co.cask.cdap.etl.proto.v2.ETLStage; -import co.cask.cdap.proto.artifact.AppRequest; -import co.cask.cdap.proto.artifact.ArtifactSummary; -import co.cask.cdap.proto.id.ApplicationId; -import co.cask.cdap.proto.id.ArtifactId; -import co.cask.cdap.proto.id.NamespaceId; -import co.cask.cdap.test.ApplicationManager; -import co.cask.cdap.test.DataSetManager; -import co.cask.cdap.test.WorkflowManager; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.common.persistance.tca.TCACalculatorMessageType; -import org.openecomp.dcae.apod.analytics.cdap.common.validation.CDAPAppSettingsValidator; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.sparkcompute.tca.SimpleTCAPlugin; -import org.openecomp.dcae.apod.analytics.common.validation.DCAEValidator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URI; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; - -/** - * Integration Test which used CDAP Hydrator Test Base to Test Simple TCA Plugin - * - * @author Rajiv Singla . Creation Date: 2/17/2017. - */ -public class SimpleTCAPluginCDAPIT extends HydratorTestBase { - - private static final Logger LOG = LoggerFactory.getLogger(SimpleTCAPluginCDAPIT.class); - - private static final String CDAP_PLUGIN_VERSION = "3.0-SNAPSHOT"; - private static final String CDAP_PLUGIN_ARTIFACT_NAME = "dcae-analytics-cdap-plugins"; - - protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", - "4.0.0"); - protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "4.0.0"); - - private static Schema sourceSchema = Schema.recordOf("CEFMessageSourceSchema", - Schema.Field.of("message", Schema.of(Schema.Type.STRING)) - ); - - final Schema outputSchema = Schema.recordOf( - "outputSchema", - Schema.Field.of("message", Schema.of(Schema.Type.STRING)), - Schema.Field.of("alert", Schema.nullableOf(Schema.of(Schema.Type.STRING))), - Schema.Field.of("tcaMessageType", Schema.of(Schema.Type.STRING)) - ); - - @BeforeClass - public static void setupTest() throws Exception { - - setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); - - - // Enable the below code if you want to run the test in Intelli IDEA editor - // addPluginArtifact(NamespaceId.DEFAULT.artifact("spark-plugins", "1.0.0"), DATAPIPELINE_ARTIFACT_ID, - // SimpleTCAPlugin.class, SimpleTCAPluginConfig.class); - - // Enable the below code if you want to run the test via command line - ArtifactId dcaeAnalyticsCdapPluginsArtifact = NamespaceId.DEFAULT.artifact( - CDAP_PLUGIN_ARTIFACT_NAME, CDAP_PLUGIN_VERSION); - - addPluginArtifact(dcaeAnalyticsCdapPluginsArtifact, DATAPIPELINE_ARTIFACT_ID, - ImmutableSet.of(getSimpleTCAPluginClass()), SimpleTCAPlugin.class, SimpleTCAPluginConfig.class, - CDAPAppSettingsValidator.class, DCAEValidator.class); - } - - private static PluginClass getSimpleTCAPluginClass() { - final HashMap properties = new HashMap<>(); - properties.put("vesMessageFieldName", new PluginPropertyField("vesMessageFieldName", "", - "string", false, false)); - properties.put("referenceName", new PluginPropertyField("referenceName", "", - "string", false, false)); - properties.put("policyJson", new PluginPropertyField("policyJson", "", "string", false, false)); - properties.put("alertFieldName", new PluginPropertyField("alertFieldName", "", "string", false, false)); - properties.put("messageTypeFieldName", new PluginPropertyField( - "messageTypeFieldName", "", "string", false, false)); - properties.put("enableAlertCEFFormat", new PluginPropertyField( - "enableAlertCEFFormat", "", "string", false, false)); - properties.put("schema", new PluginPropertyField( - "schema", "", "string", false, false)); - - return new PluginClass("sparkcompute", "SimpleTCAPlugin", "", SimpleTCAPlugin.class.getName(), - "pluginConfig", properties); - } - - - @AfterClass - public static void cleanup() { - } - - @Test - @SuppressWarnings("deprecation") - public void testTransform() throws Exception { - - LOG.info("Starting Test Transform"); - - final String policyString = getFileContentAsString("/data/json/policy/tca_policy.json"); - final String cefMessage = getFileContentAsString("/data/json/cef/cef_message.json"); - - final Map tcaProperties = new ImmutableMap.Builder() - .put("vesMessageFieldName", "message") - .put("referenceName", "SimpleTcaPlugin") - .put("policyJson", policyString) - .put("alertFieldName", "alert") - .put("messageTypeFieldName", "tcaMessageType") - .put("enableAlertCEFFormat", "true") - .put("schema", outputSchema.toString()) - .build(); - - final ETLPlugin mockSourcePlugin = MockSource.getPlugin("messages", sourceSchema); - final ETLPlugin tcaPlugin = - new ETLPlugin("SimpleTCAPlugin", SparkCompute.PLUGIN_TYPE, tcaProperties, null); - final ETLPlugin mockSink = MockSink.getPlugin("tcaOutput"); - - final ETLBatchConfig etlBatchConfig = ETLBatchConfig.builder("* * * * *") - .addStage(new ETLStage("source", mockSourcePlugin)) - .addStage(new ETLStage("simpleTCAPlugin", tcaPlugin)) - .addStage(new ETLStage("sink", mockSink)) - .addConnection("source", "simpleTCAPlugin") - .addConnection("simpleTCAPlugin", "sink") - .build(); - - AppRequest appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlBatchConfig); - ApplicationId appId = NamespaceId.DEFAULT.app("TestSimpleTCAPlugin"); - ApplicationManager appManager = deployApplication(appId.toId(), appRequest); - - List sourceMessages = new ArrayList<>(); - StructuredRecord.Builder builder = StructuredRecord.builder(sourceSchema); - builder.set("message", cefMessage); - sourceMessages.add(builder.build()); - - // write records to source - DataSetManager

inputManager = getDataset(NamespaceId.DEFAULT.dataset("messages")); - MockSource.writeInput(inputManager, sourceMessages); - - // manually trigger the pipeline - WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); - workflowManager.start(); - workflowManager.waitForFinish(5, TimeUnit.MINUTES); - - final DataSetManager
outputManager = getDataset("tcaOutput"); - - Tasks.waitFor( - TCACalculatorMessageType.COMPLIANT.name(), - new Callable() { - @Override - public String call() throws Exception { - outputManager.flush(); - List tcaOutputMessageType = new LinkedList<>(); - for (StructuredRecord outputRecord : MockSink.readOutput(outputManager)) { - tcaOutputMessageType.add(outputRecord.get("tcaMessageType").toString()); - final List fields = outputRecord.getSchema().getFields(); - LOG.debug("====>> Printing output Structured Record Contents: {}", outputRecord); - for (Schema.Field field : fields) { - LOG.debug("Field Name: {} - Field Type: {} ---> Field Value: {}", - field.getName(), field.getSchema().getType(), - outputRecord.get(field.getName())); - } - - } - return tcaOutputMessageType.get(0); - } - }, - 4, - TimeUnit.MINUTES); - - } - - private static String getFileContentAsString(final String fileLocation) throws Exception { - final URI tcaPolicyURI = - SimpleTCAPluginCDAPIT.class.getResource(fileLocation).toURI(); - List lines = Files.readAllLines(Paths.get(tcaPolicyURI), Charset.defaultCharset()); - return Joiner.on("").join(lines); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java deleted file mode 100644 index a588eb4..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/sparkcompute/tca/SimpleTCAPluginTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.sparkcompute.tca; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.StageConfigurer; -import co.cask.cdap.etl.api.batch.SparkExecutionPluginContext; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.common.persistance.tca.TCACalculatorMessageType; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.TestSimpleTCAPluginConfig; - -import java.util.LinkedList; -import java.util.List; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author Rajiv Singla . Creation Date: 2/17/2017. - */ -public class SimpleTCAPluginTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private SimpleTCAPlugin simpleTCAPlugin; - - @Before - public void before() { - final TestSimpleTCAPluginConfig testSimpleTCAPluginConfig = getTestSimpleTCAPluginConfig(); - Schema outputSchema = Schema.recordOf( - "TestSimpleTCAPluginInputSchema", - Schema.Field.of("message", Schema.of(Schema.Type.STRING)), - Schema.Field.of("alert", Schema.nullableOf(Schema.of(Schema.Type.STRING))), - Schema.Field.of("tcaMessageType", Schema.of(Schema.Type.STRING)) - ); - testSimpleTCAPluginConfig.setSchema(outputSchema.toString()); - simpleTCAPlugin = new SimpleTCAPlugin(testSimpleTCAPluginConfig); - } - - @Test - public void testConfigurePipeline() throws Exception { - final PipelineConfigurer pipelineConfigurer = mock(PipelineConfigurer.class); - final StageConfigurer stageConfigurer = mock(StageConfigurer.class); - when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); - when(stageConfigurer.getInputSchema()).thenReturn(getSimpleTCAPluginInputSchema()); - simpleTCAPlugin.configurePipeline(pipelineConfigurer); - verify(stageConfigurer, times(1)).getInputSchema(); - } - - @Test - public void testTransform() throws Exception { - - JavaSparkContext javaSparkContext = new JavaSparkContext("local", "test"); - - Schema sourceSchema = Schema.recordOf("CEFMessageSourceSchema", - Schema.Field.of("message", Schema.of(Schema.Type.STRING)) - ); - - // Inapplicable Message Structured Record - final StructuredRecord inapplicableSR = - StructuredRecord.builder(sourceSchema).set("message", "test").build(); - // compliant - final StructuredRecord compliantSR = - StructuredRecord.builder(sourceSchema).set("message", - fromStream(CEF_MESSAGE_JSON_FILE_LOCATION)).build(); - // non compliant - final String nonCompliantCEF = fromStream(CEF_NON_COMPLIANT_MESSAGE_JSON_FILE_LOCATION); - final StructuredRecord nonCompliantSR = - StructuredRecord.builder(sourceSchema).set("message", nonCompliantCEF).build(); - - final List records = new LinkedList<>(); - records.add(inapplicableSR); - records.add(compliantSR); - records.add(nonCompliantSR); - - final JavaRDD input = - javaSparkContext.parallelize(records); - final SparkExecutionPluginContext context = Mockito.mock(SparkExecutionPluginContext.class); - final MockStageMetrics stageMetrics = Mockito.mock(MockStageMetrics.class); - when(context.getMetrics()).thenReturn(stageMetrics); - final List outputRecord = simpleTCAPlugin.transform(context, input).collect(); - assertNotNull(outputRecord); - assertThat(outputRecord.size(), is(3)); - - assertTrue(outputRecord.get(0).get("tcaMessageType").equals(TCACalculatorMessageType.INAPPLICABLE.toString())); - assertTrue(outputRecord.get(1).get("tcaMessageType").equals(TCACalculatorMessageType.COMPLIANT.toString())); - assertTrue(outputRecord.get(2).get("tcaMessageType").equals(TCACalculatorMessageType.NON_COMPLIANT.toString())); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java deleted file mode 100644 index 40fadcc..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRReceiverTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import com.google.common.collect.ImmutableList; -import org.apache.spark.storage.StorageLevel; -import org.junit.Test; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; -import org.openecomp.dcae.apod.analytics.dmaap.domain.response.DMaaPMRSubscriberResponse; -import org.openecomp.dcae.apod.analytics.dmaap.service.subscriber.DMaaPMRSubscriber; - -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author Rajiv Singla . Creation Date: 1/24/2017. - */ -public class DMaaPMRReceiverTest extends BaseAnalyticsCDAPPluginsUnitTest { - - - @Test - public void testStoreStructuredRecords() throws Exception { - - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - final TestDMaaPMRReceiver dMaaPMRReceiver = - new TestDMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), testDMaaPMRSourcePluginConfig); - - final DMaaPMRSubscriber dMaaPMRSubscriber = Mockito.mock(DMaaPMRSubscriber.class); - final DMaaPMRSubscriberResponse subscriberResponse = Mockito.mock(DMaaPMRSubscriberResponse.class); - when(dMaaPMRSubscriber.fetchMessages()).thenReturn(subscriberResponse); - when(subscriberResponse.getFetchedMessages()).thenReturn(ImmutableList.of("Test Message")); - when(subscriberResponse.getResponseCode()).thenReturn(200); - when(subscriberResponse.getResponseMessage()).thenReturn("OK"); - dMaaPMRReceiver.storeStructuredRecords(dMaaPMRSubscriber); - verify(dMaaPMRSubscriber, times(1)).fetchMessages(); - verify(subscriberResponse, times(1)).getFetchedMessages(); - } - - @Test - public void testStoreStructuredRecordsWhenSubscriberThrowsException() throws Exception { - - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - final TestDMaaPMRReceiver dMaaPMRReceiver = - new TestDMaaPMRReceiver(StorageLevel.MEMORY_ONLY(), testDMaaPMRSourcePluginConfig); - - final DMaaPMRSubscriber dMaaPMRSubscriber = Mockito.mock(DMaaPMRSubscriber.class); - final DMaaPMRSubscriberResponse subscriberResponse = Mockito.mock(DMaaPMRSubscriberResponse.class); - when(dMaaPMRSubscriber.fetchMessages()).thenThrow(DCAEAnalyticsRuntimeException.class); - dMaaPMRReceiver.storeStructuredRecords(dMaaPMRSubscriber); - verify(dMaaPMRSubscriber, times(1)).fetchMessages(); - verify(subscriberResponse, times(0)).getFetchedMessages(); - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java deleted file mode 100644 index 0b5ac87..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/DMaaPMRSourceTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.StageConfigurer; -import co.cask.cdap.etl.api.streaming.StreamingContext; -import org.apache.spark.streaming.api.java.JavaDStream; -import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; -import org.apache.spark.streaming.api.java.JavaStreamingContext; -import org.apache.spark.streaming.receiver.Receiver; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; - -import static org.junit.Assert.assertNotNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.when; - -/** - * @author Rajiv Singla . Creation Date: 1/24/2017. - */ -public class DMaaPMRSourceTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private PipelineConfigurer pipelineConfigurer; - - @Before - public void before() { - pipelineConfigurer = Mockito.mock(PipelineConfigurer.class); - final StageConfigurer stageConfigurer = Mockito.mock(StageConfigurer.class); - when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); - doNothing().when(stageConfigurer).setOutputSchema(any(Schema.class)); - } - - @Test - public void testDMaaPMRSourceConfigurePipelineWithValidPluginSettings() throws Exception { - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - final DMaaPMRSource dMaaPMRSource = new DMaaPMRSource(testDMaaPMRSourcePluginConfig); - dMaaPMRSource.configurePipeline(pipelineConfigurer); - assertNotNull(dMaaPMRSource); - } - - @Test(expected = CDAPSettingsException.class) - public void testDMaaPMRSourceConfigurePipelineWithInvalidPluginSettings() throws Exception { - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - // blank out DMaaP MR Source Host - testDMaaPMRSourcePluginConfig.setHostName(null); - final DMaaPMRSource dMaaPMRSource = new DMaaPMRSource(testDMaaPMRSourcePluginConfig); - dMaaPMRSource.configurePipeline(pipelineConfigurer); - } - - - @Test - @SuppressWarnings("unchecked") - public void testGetStream() throws Exception { - final StreamingContext streamingContext = Mockito.mock(StreamingContext.class); - final JavaStreamingContext javaStreamingContext = Mockito.mock(JavaStreamingContext.class); - final JavaReceiverInputDStream dMaaPMRReceiver = Mockito.mock(JavaReceiverInputDStream.class); - when(streamingContext.getSparkStreamingContext()).thenReturn(javaStreamingContext); - when(javaStreamingContext.receiverStream(any(Receiver.class))).thenReturn(dMaaPMRReceiver); - - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - final DMaaPMRSource dMaaPMRSource = new DMaaPMRSource(testDMaaPMRSourcePluginConfig); - final JavaDStream stream = dMaaPMRSource.getStream(streamingContext); - assertNotNull(stream); - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java deleted file mode 100644 index d2221ea..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRReceiverTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.data.format.StructuredRecord; -import org.apache.spark.storage.StorageLevel; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.schema.dmaap.DMaaPSourceOutputSchema; - -import java.util.concurrent.TimeUnit; - -/** - * @author Rajiv Singla . Creation Date: 2/20/2017. - */ -public class MockDMaaPMRReceiverTest extends BaseAnalyticsCDAPPluginsUnitTest { - - protected class TestMockDMaaPMRReceiverTest extends MockDMaaPMRReceiver { - - private boolean canStop = false; - - public TestMockDMaaPMRReceiverTest(StorageLevel storageLevel, DMaaPMRSourcePluginConfig pluginConfig) { - super(storageLevel, pluginConfig); - } - - @Override - public boolean isStopped() { - return canStop; - } - - @Override - public void store(StructuredRecord dataItem) { - LOG.debug("Mocking storing dataItem - {}", - dataItem.get(DMaaPSourceOutputSchema.FETCHED_MESSAGE.getSchemaColumnName())); - } - - public void setCanStop(boolean canStop) { - this.canStop = canStop; - } - } - - @Test - public void testStoreStructuredRecords() throws Exception { - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - testDMaaPMRSourcePluginConfig.setPollingInterval(100); - final TestMockDMaaPMRReceiverTest mockDMaaPMRReceiver = new TestMockDMaaPMRReceiverTest(StorageLevel - .MEMORY_ONLY(), - testDMaaPMRSourcePluginConfig); - new Thread(new Runnable() { - @Override - public void run() { - mockDMaaPMRReceiver.storeStructuredRecords(null); - } - }).start(); - TimeUnit.MILLISECONDS.sleep(1000); - LOG.info("Killing Mock Subscriber after 1 ms"); - mockDMaaPMRReceiver.setCanStop(true); - - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java deleted file mode 100644 index 7888d59..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/MockDMaaPMRSourceTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.etl.api.streaming.StreamingContext; -import org.apache.spark.streaming.api.java.JavaDStream; -import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; -import org.apache.spark.streaming.api.java.JavaStreamingContext; -import org.apache.spark.streaming.receiver.Receiver; -import org.junit.Test; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.common.exception.CDAPSettingsException; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; - -import static org.junit.Assert.assertNotNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -/** - * @author Rajiv Singla . Creation Date: 2/20/2017. - */ -@SuppressWarnings("unchecked") -public class MockDMaaPMRSourceTest extends BaseAnalyticsCDAPPluginsUnitTest { - - @Test - public void testGetStream() throws Exception { - final StreamingContext streamingContext = Mockito.mock(StreamingContext.class); - final JavaStreamingContext javaStreamingContext = Mockito.mock(JavaStreamingContext.class); - final JavaReceiverInputDStream dMaaPMRReceiver = Mockito.mock(JavaReceiverInputDStream.class); - when(streamingContext.getSparkStreamingContext()).thenReturn(javaStreamingContext); - when(javaStreamingContext.receiverStream(any(Receiver.class))).thenReturn(dMaaPMRReceiver); - - MockDMaaPMRSource mockDMaaPMRSource = new MockDMaaPMRSource(getTestDMaaPMRSourcePluginConfig()); - final JavaDStream stream = mockDMaaPMRSource.getStream(streamingContext); - assertNotNull(stream); - } - - @Test(expected = CDAPSettingsException.class) - public void testConfigurePipelineWhenPollingIntervalNotPresent() throws Exception { - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - testDMaaPMRSourcePluginConfig.setPollingInterval(null); - final MockDMaaPMRSource mockDMaaPMRSource = new MockDMaaPMRSource(testDMaaPMRSourcePluginConfig); - mockDMaaPMRSource.configurePipeline(null); - } - - @Test - public void testConfigurePipelineWhenPollingIntervalIsPresent() throws Exception { - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - final MockDMaaPMRSource mockDMaaPMRSource = new MockDMaaPMRSource(testDMaaPMRSourcePluginConfig); - mockDMaaPMRSource.configurePipeline(null); - assertNotNull(mockDMaaPMRSource); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java deleted file mode 100644 index 2eab27b..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/streaming/dmaap/TestDMaaPMRReceiver.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.streaming.dmaap; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.metrics.Metrics; -import org.apache.spark.storage.StorageLevel; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; - -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; - -/** - * Test implementation for {@link DMaaPMRReceiver} - *

- * @author Rajiv Singla . Creation Date: 1/24/2017. - */ -public class TestDMaaPMRReceiver extends DMaaPMRReceiver { - - protected static Metrics metrics; - - static { - metrics = Mockito.mock(Metrics.class); - doNothing().when(metrics).count(anyString(), anyInt()); - doNothing().when(metrics).gauge(anyString(), anyInt()); - } - - - public TestDMaaPMRReceiver(StorageLevel storageLevel, DMaaPMRSourcePluginConfig pluginConfig) { - - super(storageLevel, pluginConfig, metrics); - } - - @Override - public void store(StructuredRecord dataItem) { - // do nothing - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java deleted file mode 100644 index a0feb99..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/transform/filter/JsonPathFilterTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.transform.filter; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import co.cask.cdap.etl.api.Emitter; -import co.cask.cdap.etl.api.PipelineConfigurer; -import co.cask.cdap.etl.api.StageConfigurer; -import org.junit.Test; -import org.mockito.ArgumentMatchers; -import org.mockito.Mockito; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; - -import java.util.Date; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author Rajiv Singla . Creation Date: 3/3/2017. - */ -public class JsonPathFilterTest extends BaseAnalyticsCDAPPluginsUnitTest { - - - @Test - public void testInitializeWhenFilterMappingIsValid() throws Exception { - final JsonPathFilter jsonPathFilter = new JsonPathFilter(getJsonPathFilterPluginConfig()); - jsonPathFilter.initialize(null); - } - - - @Test - public void configurePipeline() throws Exception { - final JsonPathFilter jsonPathFilter = new JsonPathFilter(getJsonPathFilterPluginConfig()); - final PipelineConfigurer pipelineConfigurer = mock(PipelineConfigurer.class); - final StageConfigurer stageConfigurer = mock(StageConfigurer.class); - when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); - when(stageConfigurer.getInputSchema()).thenReturn(getSimpleTCAPluginInputSchema()); - doNothing().when(stageConfigurer).setOutputSchema(any(Schema.class)); - jsonPathFilter.configurePipeline(pipelineConfigurer); - verify(stageConfigurer, times(1)).setOutputSchema(any(Schema.class)); - } - - @Test - public void testTransform() throws Exception { - final JsonPathFilter jsonPathFilter = new JsonPathFilter(getJsonPathFilterPluginConfig()); - jsonPathFilter.initialize(null); - final StructuredRecord inputSR = StructuredRecord.builder(getJsonFilterPluginInputSchema()) - .set("ts", new Date().getTime()) - .set("responseCode", 200) - .set("responseMessage", "OK") - .set("message", fromStream(CEF_MESSAGE_JSON_FILE_LOCATION)) - .build(); - - final Emitter emitter = Mockito.mock(Emitter.class); - doNothing().when(emitter).emit(ArgumentMatchers.any(StructuredRecord.class)); - jsonPathFilter.transform(inputSR, emitter); - verify(emitter, times(1)).emit(any(StructuredRecord.class)); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java deleted file mode 100644 index fbecb81..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/CDAPPluginUtilsTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.utils; - -import co.cask.cdap.api.data.format.StructuredRecord; -import co.cask.cdap.api.data.schema.Schema; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.common.exception.DCAEAnalyticsRuntimeException; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class CDAPPluginUtilsTest extends BaseAnalyticsCDAPPluginsUnitTest { - - - @Test - public void testValidateSchemaContainsFieldsWhenSchemaIsNotNull() throws Exception { - final Schema dMaaPMRSinkTestSchema = getDMaaPMRSinkTestSchema(); - CDAPPluginUtils.validateSchemaContainsFields(dMaaPMRSinkTestSchema, "message"); - } - - @Test - public void testValidateSchemaContainsFieldsWhenInputSchemaIsNull() throws Exception { - CDAPPluginUtils.validateSchemaContainsFields(null, "message"); - } - - @Test - public void testCreateStructuredRecord() throws Exception { - final StructuredRecord testMessage = CDAPPluginUtils.createDMaaPMRResponseStructuredRecord("testMessage"); - assertNotNull(testMessage); - } - - - @Test - public void testCreateOutputStructuredRecordBuilder() throws Exception { - - final String messageFieldName = "message"; - final String firstInputFieldName = "inputField1"; - final String secondInputFieldName = "inputField2"; - - - final Schema inputSchema = Schema.recordOf( - "inputSchema", - Schema.Field.of(messageFieldName, Schema.of(Schema.Type.STRING)), - Schema.Field.of(firstInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))), - Schema.Field.of(secondInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))) - ); - - final String addedFieldName = "addedField"; - final Schema outputSchema = Schema.recordOf( - "outputSchema", - Schema.Field.of(messageFieldName, Schema.of(Schema.Type.STRING)), - Schema.Field.of(firstInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))), - Schema.Field.of(addedFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))) // added field - // missing second Input Field - ); - - // input structured record - final String messageFieldValue = "Message String"; - final String firstFieldValue = "Input Field 1"; - final String secondFieldValue = "Input Field 2"; - final StructuredRecord inputSR = StructuredRecord.builder(inputSchema) - .set(messageFieldName, messageFieldValue) - .set(firstInputFieldName, firstFieldValue) - .set(secondInputFieldName, secondFieldValue) - .build(); - - final StructuredRecord.Builder outputStructuredRecordBuilder = - CDAPPluginUtils.createOutputStructuredRecordBuilder(outputSchema, inputSR); - - final String addedFieldValue = "Added Field Value"; - final StructuredRecord outputSR = outputStructuredRecordBuilder - .set(addedFieldName, addedFieldValue) - .build(); - - assertThat("Added Field field value copied correctly", - outputSR.get(addedFieldName).toString(), is(addedFieldValue)); - - assertThat("Output SR has message field copied correctly", - outputSR.get(messageFieldName).toString(), is(messageFieldValue)); - - assertThat("First Field value copied correctly", - outputSR.get(firstInputFieldName).toString(), is(firstFieldValue)); - - assertNull("Second Field value is null as output schema does not have the field", - outputSR.get(secondInputFieldName)); - - } - - - @Test - public void testAddFieldValueToStructuredRecordBuilder() throws Exception { - - final String messageFieldName = "message"; - final String firstInputFieldName = "inputField1"; - final String addedFieldName = "addedField"; - final String firstFieldValue = "Input Field 1"; - final String addedFieldValue = "Added Field Value"; - final Schema outputSchema = Schema.recordOf( - "outputSchema", - Schema.Field.of(messageFieldName, Schema.of(Schema.Type.STRING)), - Schema.Field.of(firstInputFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))), - Schema.Field.of(addedFieldName, Schema.nullableOf(Schema.of(Schema.Type.STRING))) // added field - ); - - final StructuredRecord.Builder outputSRBuilder = StructuredRecord.builder(outputSchema) - .set(messageFieldName, "Some message") - .set(firstInputFieldName, firstFieldValue); - - final StructuredRecord.Builder addedFieldSRBuilder = CDAPPluginUtils.addFieldValueToStructuredRecordBuilder( - outputSRBuilder, outputSchema, addedFieldName, addedFieldValue); - - // Try adding field to output Structured record that is not in output schema - final String nonExistentFieldName = "fieldNotInOutputSchema"; - final String nonExistentFieldValue = "Some Value"; - final StructuredRecord outputSR = CDAPPluginUtils.addFieldValueToStructuredRecordBuilder( - addedFieldSRBuilder, outputSchema, nonExistentFieldName, nonExistentFieldValue).build(); - - assertThat("Output SR must contain added Field which is in output schema", - outputSR.get(addedFieldName).toString(), is(addedFieldValue)); - assertNull("Output SR must not contain field that is not in output schema", - outputSR.get(nonExistentFieldName)); - - } - - @Test(expected = DCAEAnalyticsRuntimeException.class) - public void testValidateSchemaFieldTypeWhenInputSchemaIsNotValidJson() throws Exception { - CDAPPluginUtils.validateSchemaFieldType("Invalid Schema", "field1", Schema.Type.STRING); - } - - @Test(expected = DCAEAnalyticsRuntimeException.class) - public void testSetOutputSchemaWhenOutputSchemaIsNotValidJson() throws Exception { - CDAPPluginUtils.setOutputSchema(null, "Invalid output Schema"); - } - - @Test(expected = DCAEAnalyticsRuntimeException.class) - public void testExtractFieldMappingsWhenFieldMappingValueIsEmpty() throws Exception { - CDAPPluginUtils.extractFieldMappings("path1:,path2:value2"); - } - - @Test(expected = DCAEAnalyticsRuntimeException.class) - public void testExtractFieldMappingsWhenFieldMappingAreBlank() throws Exception { - CDAPPluginUtils.extractFieldMappings("path1: ,path2:value2"); - } - - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java deleted file mode 100644 index b05a5d2..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSinkConfigMapperTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.utils; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.dmaap.domain.config.DMaaPMRPublisherConfig; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPSinkConfigMapperTest extends BaseAnalyticsCDAPPluginsUnitTest { - - - @Test - public void testMapToPublisherConfig() throws Exception { - - final Configuration testConfiguration = getTestConfiguration(); - final DMaaPMRPublisherConfig publisherConfig = DMaaPSinkConfigMapper.map(testConfiguration); - - assertNotNull(publisherConfig); - assertThat(publisherConfig.getHostName(), is(DMAAP_MR_SINK_PLUGIN_HOST_NAME)); - assertThat(publisherConfig.getTopicName(), is(DMAAP_MR_SINK_PLUGIN_TOPIC_NAME)); - assertThat(publisherConfig.getPortNumber(), is(DMAAP_MR_SINK_PLUGIN_PORT_NUMBER)); - assertThat(publisherConfig.getProtocol(), is(DMAAP_MR_SINK_PLUGIN_PROTOCOL)); - assertThat(publisherConfig.getUserName(), is(DMAAP_MR_SINK_PLUGIN_USERNAME)); - assertThat(publisherConfig.getUserPassword(), is(DMAAP_MR_SINK_PLUGIN_PASSWORD)); - assertThat(publisherConfig.getContentType(), is(DMAAP_MR_SINK_PLUGIN_CONTENT_TYPE)); - assertThat(publisherConfig.getMaxBatchSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_BATCH_SIZE)); - assertThat(publisherConfig.getMaxRecoveryQueueSize(), is(DMAAP_MR_SINK_PLUGIN_MAX_RECOVERY_QUEUE_SIZE)); - - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java deleted file mode 100644 index e84194f..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/utils/DMaaPSourceConfigMapperTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.utils; - -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.dmaap.domain.config.DMaaPMRSubscriberConfig; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author Rajiv Singla . Creation Date: 1/24/2017. - */ -public class DMaaPSourceConfigMapperTest extends BaseAnalyticsCDAPPluginsUnitTest { - - @Test - public void testMapToSubscriberConfig() throws Exception { - - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - final DMaaPMRSubscriberConfig subscriberConfig = DMaaPSourceConfigMapper.map(testDMaaPMRSourcePluginConfig); - - assertNotNull(subscriberConfig); - assertThat(subscriberConfig.getHostName(), is(DMAAP_MR_SOURCE_PLUGIN_HOST_NAME)); - assertThat(subscriberConfig.getTopicName(), is(DMAAP_MR_SOURCE_PLUGIN_TOPIC_NAME)); - assertThat(subscriberConfig.getPortNumber(), is(DMAAP_MR_SOURCE_PLUGIN_PORT_NUMBER)); - assertThat(subscriberConfig.getProtocol(), is(DMAAP_MR_SOURCE_PLUGIN_PROTOCOL)); - assertThat(subscriberConfig.getUserName(), is(DMAAP_MR_SOURCE_PLUGIN_USERNAME)); - assertThat(subscriberConfig.getUserPassword(), is(DMAAP_MR_SOURCE_PLUGIN_PASSWORD)); - assertThat(subscriberConfig.getContentType(), is(DMAAP_MR_SOURCE_PLUGIN_CONTENT_TYPE)); - assertThat(subscriberConfig.getConsumerGroup(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_GROUP)); - assertThat(subscriberConfig.getConsumerId(), is(DMAAP_MR_SOURCE_PLUGIN_CONSUMER_ID)); - assertThat(subscriberConfig.getMessageLimit(), is(DMAAP_MR_SOURCE_PLUGIN_MESSAGE_LIMIT)); - assertThat(subscriberConfig.getTimeoutMS(), is(DMAAP_MR_SOURCE_PLUGIN_TIMEOUT)); - } - - @Test(expected = IllegalStateException.class) - public void testMapToSubscriberConfigWhenSubscriberHostNameIsEmpty() throws Exception { - final TestDMaaPMRSourcePluginConfig testDMaaPMRSourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - testDMaaPMRSourcePluginConfig.setHostName(null); - DMaaPSourceConfigMapper.map(testDMaaPMRSourcePluginConfig); - - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java deleted file mode 100644 index 05c0ddd..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSinkPluginConfigValidatorTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import org.junit.Before; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSinkPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSinkPluginConfig; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMRSinkPluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private TestDMaaPMRSinkPluginConfig sinkPluginConfig; - private DMaaPMRSinkPluginConfigValidator sinkPluginConfigValidator; - - @Before - public void before() { - sinkPluginConfigValidator = new DMaaPMRSinkPluginConfigValidator(); - sinkPluginConfig = getTestDMaaPMRSinkPluginConfig(); - } - - @Test - public void validateAppSettingsWithValidDMaaPSinkConfig() throws Exception { - final GenericValidationResponse validationResponse = - sinkPluginConfigValidator.validateAppSettings(sinkPluginConfig); - assertFalse(validationResponse.hasErrors()); - } - - - @Test - public void validateAppSettingsWithValidDMaaPSinkConfigWhenHostNameIsNotPresent() throws Exception { - sinkPluginConfig.setHostName(null); - assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); - } - - @Test - public void validateAppSettingsWithValidDMaaPSinkConfigWhenHostPortIsNotPresent() throws Exception { - sinkPluginConfig.setPortNumber(null); - assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); - } - - @Test - public void validateAppSettingsWithValidDMaaPSinkConfigWhenTopicNameIsNotPresent() throws Exception { - sinkPluginConfig.setTopicName(null); - assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); - } - - @Test - public void validateAppSettingsWithValidDMaaPSinkConfigWhenColumnNameIsNotPresent() throws Exception { - sinkPluginConfig.setMessageColumnName(null); - assertResponseHasErrors(sinkPluginConfig, sinkPluginConfigValidator); - } - - private static void assertResponseHasErrors(final TestDMaaPMRSinkPluginConfig sinkPluginConfig, - final DMaaPMRSinkPluginConfigValidator validator) { - final GenericValidationResponse validationResponse = validator.validateAppSettings(sinkPluginConfig); - assertTrue(validationResponse.hasErrors()); - } - - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java deleted file mode 100644 index 0401bb7..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/DMaaPMRSourcePluginConfigValidatorTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import org.junit.Before; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.DMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.dmaap.TestDMaaPMRSourcePluginConfig; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author Rajiv Singla . Creation Date: 1/30/2017. - */ -public class DMaaPMRSourcePluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private TestDMaaPMRSourcePluginConfig sourcePluginConfig; - private DMaaPMRSourcePluginConfigValidator sourcePluginConfigValidator; - - @Before - public void before() { - sourcePluginConfigValidator = new DMaaPMRSourcePluginConfigValidator(); - sourcePluginConfig = getTestDMaaPMRSourcePluginConfig(); - } - - @Test - public void validateAppSettingsWithValidDMaaPSourceConfig() throws Exception { - final GenericValidationResponse validationResponse = - sourcePluginConfigValidator.validateAppSettings(sourcePluginConfig); - assertFalse(validationResponse.hasErrors()); - } - - - @Test - public void validateAppSettingsWithValidDMaaPSourceConfigWhenHostNameIsNotPresent() throws Exception { - sourcePluginConfig.setHostName(null); - assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); - } - - @Test - public void validateAppSettingsWithValidDMaaPSourceConfigWhenHostPortIsNotPresent() throws Exception { - sourcePluginConfig.setPortNumber(null); - assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); - } - - @Test - public void validateAppSettingsWithValidDMaaPSourceConfigWhenTopicNameIsNotPresent() throws Exception { - sourcePluginConfig.setTopicName(null); - assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); - } - - @Test - public void validateAppSettingsWithValidDMaaPSourcePollingIntervalIsNotPresent() throws Exception { - sourcePluginConfig.setPollingInterval(null); - assertResponseHasErrors(sourcePluginConfig, sourcePluginConfigValidator); - } - - private static void assertResponseHasErrors(final TestDMaaPMRSourcePluginConfig sourcePluginConfig, - final DMaaPMRSourcePluginConfigValidator validator) { - final GenericValidationResponse validationResponse = validator.validateAppSettings(sourcePluginConfig); - assertTrue(validationResponse.hasErrors()); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java deleted file mode 100644 index 7708dcd..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/JsonPathFilterPluginConfigValidatorTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import org.junit.Before; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter.JsonPathFilterPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.filter.TestJsonPathFilterPluginConfig; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author Rajiv Singla . Creation Date: 3/3/2017. - */ -public class JsonPathFilterPluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private TestJsonPathFilterPluginConfig jsonPathFilterPluginConfig; - private JsonPathFilterPluginConfigValidator jsonPathFilterPluginConfigValidator; - - @Before - public void before() { - jsonPathFilterPluginConfig = getJsonPathFilterPluginConfig(); - jsonPathFilterPluginConfigValidator = new JsonPathFilterPluginConfigValidator(); - } - - - @Test - public void testValidateAppSettingsWhenNoValidationErrors() throws Exception { - final GenericValidationResponse validationResponse = - jsonPathFilterPluginConfigValidator.validateAppSettings(jsonPathFilterPluginConfig); - assertFalse(validationResponse.hasErrors()); - } - - @Test - public void testValidateAppSettingsWhenFilterMappingsAreEmpty() throws Exception { - jsonPathFilterPluginConfig.setJsonFilterMappings(""); - assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenOutputSchemaIsNotPresent() throws Exception { - jsonPathFilterPluginConfig.setSchema(null); - assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenOutputSchemaFilterMatchedFieldIsNotBoolean() throws Exception { - final String outputSchemaWithMatchedFieldNotBoolean = - "{\"type\":\"record\"," + - "\"name\":\"etlSchemaBody\",\"fields\":" + - "[" + - "{\"name\":\"ts\",\"type\":\"long\"}," + - "{\"name\":\"filterMatched\",\"type\":[\"string\",\"null\"]}," + - "{\"name\":\"responseCode\",\"type\":\"int\"}," + - "{\"name\":\"responseMessage\",\"type\":\"string\"}," + - "{\"name\":\"message\",\"type\":\"string\"}" + - "]" + - "}"; - jsonPathFilterPluginConfig.setSchema(outputSchemaWithMatchedFieldNotBoolean); - assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenOutputSchemaFilterMatchedFieldIsNotNullable() throws Exception { - final String outputSchemaWithMatchedFieldNotNullable = - "{\"type\":\"record\"," + - "\"name\":\"etlSchemaBody\",\"fields\":" + - "[" + - "{\"name\":\"ts\",\"type\":\"long\"}," + - "{\"name\":\"filterMatched\",\"type\":\"boolean\"}," + - "{\"name\":\"responseCode\",\"type\":\"int\"}," + - "{\"name\":\"responseMessage\",\"type\":\"string\"}," + - "{\"name\":\"message\",\"type\":\"string\"}" + - "]" + - "}"; - jsonPathFilterPluginConfig.setSchema(outputSchemaWithMatchedFieldNotNullable); - assertResponseHasErrors(jsonPathFilterPluginConfig, jsonPathFilterPluginConfigValidator); - } - - private static void assertResponseHasErrors(final TestJsonPathFilterPluginConfig jsonPluginConfig, - final JsonPathFilterPluginConfigValidator validator) { - final GenericValidationResponse validationResponse = validator.validateAppSettings(jsonPluginConfig); - assertTrue(validationResponse.hasErrors()); - } - -} diff --git a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java b/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java deleted file mode 100644 index 3641c79..0000000 --- a/dcae-analytics-cdap-plugins/src/test/java/org/openecomp/dcae/apod/analytics/cdap/plugins/validator/SimpleTCAPluginConfigValidatorTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * ===============================LICENSE_START====================================== - * dcae-analytics - * ================================================================================ - * Copyright © 2017 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================LICENSE_END=========================================== - */ - -package org.openecomp.dcae.apod.analytics.cdap.plugins.validator; - -import org.junit.Before; -import org.junit.Test; -import org.openecomp.dcae.apod.analytics.cdap.plugins.BaseAnalyticsCDAPPluginsUnitTest; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.SimpleTCAPluginConfig; -import org.openecomp.dcae.apod.analytics.cdap.plugins.domain.config.tca.TestSimpleTCAPluginConfig; -import org.openecomp.dcae.apod.analytics.common.validation.GenericValidationResponse; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author Rajiv Singla . Creation Date: 2/21/2017. - */ -public class SimpleTCAPluginConfigValidatorTest extends BaseAnalyticsCDAPPluginsUnitTest { - - private TestSimpleTCAPluginConfig testSimpleTCAPluginConfig; - private SimpleTCAPluginConfigValidator simpleTCAPluginConfigValidator; - - @Before - public void before() { - testSimpleTCAPluginConfig = getTestSimpleTCAPluginConfig(); - simpleTCAPluginConfigValidator = new SimpleTCAPluginConfigValidator(); - } - - @Test - public void testValidateAppSettingsWhenAllSettingsAreValid() throws Exception { - final GenericValidationResponse validationResponse = - simpleTCAPluginConfigValidator.validateAppSettings(testSimpleTCAPluginConfig); - assertFalse(validationResponse.hasErrors()); - } - - @Test - public void testValidateAppSettingsWhenVESMessageFieldNameIsMissing() throws Exception { - testSimpleTCAPluginConfig.setVesMessageFieldName(null); - assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenPolicyJsonIsMissing() throws Exception { - testSimpleTCAPluginConfig.setPolicyJson(null); - assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenAlertFieldNameIsMissing() throws Exception { - testSimpleTCAPluginConfig.setAlertFieldName(null); - assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenOutputSchemaIsNull() throws Exception { - testSimpleTCAPluginConfig.setSchema(null); - assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenMessageTypeFieldNameIsMissing() throws Exception { - testSimpleTCAPluginConfig.setMessageTypeFieldName(null); - assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenAlertFieldIsNullableInOutputSchema() throws Exception { - testSimpleTCAPluginConfig.setSchema( - "{\"type\":\"record\"," + - "\"name\":\"etlSchemaBody\"," + - "\"fields\":[" + - "{\"name\":\"ts\",\"type\":\"long\"}," + - "{\"name\":\"responseCode\",\"type\":\"int\"}," + - "{\"name\":\"responseMessage\",\"type\":\"string\"}," + - "{\"name\":\"message\",\"type\":\"string\"}," + - "{\"name\":\"alert\",\"type\":[\"string\",\"null\"]}," + - "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); - final GenericValidationResponse validationResponse = - simpleTCAPluginConfigValidator.validateAppSettings(testSimpleTCAPluginConfig); - assertFalse(validationResponse.hasErrors()); - - } - - @Test - public void testValidateAppSettingsWhenAlertFieldIsNotPresentInOutputSchema() throws Exception { - testSimpleTCAPluginConfig.setSchema( - "{\"type\":\"record\"," + - "\"name\":\"etlSchemaBody\"," + - "\"fields\":[" + - "{\"name\":\"ts\",\"type\":\"long\"}," + - "{\"name\":\"responseCode\",\"type\":\"int\"}," + - "{\"name\":\"responseMessage\",\"type\":\"string\"}," + - "{\"name\":\"message\",\"type\":\"string\"}," + - "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); - final GenericValidationResponse validationResponse = - simpleTCAPluginConfigValidator.validateAppSettings(testSimpleTCAPluginConfig); - assertFalse(validationResponse.hasErrors()); - - } - - @Test - public void testValidateAppSettingsWhenAlertFieldIsNullableButNotStringTypeInOutputSchema() throws Exception { - testSimpleTCAPluginConfig.setSchema( - "{\"type\":\"record\"," + - "\"name\":\"etlSchemaBody\"," + - "\"fields\":[" + - "{\"name\":\"ts\",\"type\":\"long\"}," + - "{\"name\":\"responseCode\",\"type\":\"int\"}," + - "{\"name\":\"responseMessage\",\"type\":\"string\"}," + - "{\"name\":\"message\",\"type\":\"string\"}," + - "{\"name\":\"alert\",\"type\":[\"int\",\"null\"]}," + - "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); - assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); - } - - @Test - public void testValidateAppSettingsWhenAlertFieldNameIsNotNullableInOutputSchema() throws Exception { - testSimpleTCAPluginConfig.setSchema( - "{\"type\":\"record\"," + - "\"name\":\"etlSchemaBody\"," + - "\"fields\":[" + - "{\"name\":\"ts\",\"type\":\"long\"}," + - "{\"name\":\"responseCode\",\"type\":\"int\"}," + - "{\"name\":\"responseMessage\",\"type\":\"string\"}," + - "{\"name\":\"message\",\"type\":\"string\"}," + - "{\"name\":\"alert\",\"type\":\"string\"}," + - "{\"name\":\"tcaMessageType\",\"type\":\"string\"}]}"); - assertResponseHasErrors(testSimpleTCAPluginConfig, simpleTCAPluginConfigValidator); - } - - - - private static void assertResponseHasErrors(final TestSimpleTCAPluginConfig pluginConfig, - final SimpleTCAPluginConfigValidator validator) { - final GenericValidationResponse validationResponse = validator.validateAppSettings(pluginConfig); - assertTrue(validationResponse.hasErrors()); - LOG.debug("Validation Error Message: {}", validationResponse.getAllErrorMessage()); - } -} diff --git a/dcae-analytics-cdap-plugins/src/test/resources/logback-test.xml b/dcae-analytics-cdap-plugins/src/test/resources/logback-test.xml index 4857522..78cbdfa 100644 --- a/dcae-analytics-cdap-plugins/src/test/resources/logback-test.xml +++ b/dcae-analytics-cdap-plugins/src/test/resources/logback-test.xml @@ -1,55 +1,55 @@ - - - - - - - - - - - - - - - - - - - - - - - - - %d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + %d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + + + + + + + + + + -- cgit 1.2.3-korg