diff options
author | marcinrzepeckiwroc <marcin.rzepecki@nokia.com> | 2020-07-15 13:26:51 +0200 |
---|---|---|
committer | marcinrzepeckiwroc <marcin.rzepecki@nokia.com> | 2020-07-17 13:43:43 +0200 |
commit | ba384eeb3cc007ff1d48db04fecc9356069456c5 (patch) | |
tree | cbb6695e1ca3ac80e31dc16fd05483fd10b3fa37 /src | |
parent | 977a518c680032572f6c5c59879ead71221e460a (diff) |
implement regex support in PM Mapper filtering mechanism
Issue-ID: DCAEGEN2-2347
Signed-off-by: marcinrzepeckiwroc <marcin.rzepecki@nokia.com>
Change-Id: I3fdbae33ae318800ccc2758a191604fe79c2aaf4
Diffstat (limited to 'src')
8 files changed, 278 insertions, 42 deletions
diff --git a/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java b/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java index 21e4441..6aeeaba 100644 --- a/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java +++ b/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java @@ -59,7 +59,6 @@ import javax.net.ssl.SSLContext; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; @Data @@ -213,6 +212,7 @@ public class App { try { hasMatchingFilter = filterHandler.filterByMeasType(events); if(!hasMatchingFilter) { + logger.unwrap().info("No filter match from all measurement files."); sendEventProcessed(config,event); } } catch (Exception exception) { @@ -223,25 +223,25 @@ public class App { } public static Flux<List<Event>> map(Mapper mapper, List<Event> events, MapperConfig config) { - List<Event> mappedEvents = new ArrayList<>(); + List<Event> mappedEvents; try { mappedEvents = mapper.mapEvents(events); } catch (Exception exception) { logger.unwrap().error("Unable to map XML to VES",exception); sendEventProcessed(config,events.get(0)); - return Flux.<List<Event>>empty(); + return Flux.empty(); } return Flux.just(mappedEvents); } public static Flux<List<Event>> split(MeasSplitter splitter, Event event, MapperConfig config) { - List<Event> splitEvents = new ArrayList<>(); + List<Event> splitEvents; try { splitEvents = splitter.split(event); } catch (Exception exception) { logger.unwrap().error("Unable to split MeasCollecFile",exception); sendEventProcessed(config,event); - return Flux.<List<Event>>empty(); + return Flux.empty(); } return Flux.just(splitEvents); } diff --git a/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java b/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java index fb5772e..7f2a5d6 100644 --- a/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java +++ b/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java @@ -21,8 +21,11 @@ package org.onap.dcaegen2.services.pmmapper.filtering; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.Optional; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import org.apache.commons.io.FilenameUtils; @@ -68,16 +71,9 @@ public class MeasFilterHandler { logger.unwrap().info("Filtering the measurement file by measTypes."); MeasurementData measData = measurementFile.getMeasurementData().get().get(0); List<MeasurementInfo> measInfos = measData.getMeasurementInfo(); - List<MeasurementInfo> filteredMeasInfos = new ArrayList<>(); + List<Pattern> filterPatternList = getPatternsFromFilters(filter); - for (MeasurementInfo currentMeasInfo : measInfos) { - List<String> measTypesNode = currentMeasInfo.getMeasTypes(); - if (measTypesNode != null && !measTypesNode.isEmpty()) { - setMeasInfosFromMeasTypes(currentMeasInfo, filteredMeasInfos, filter.get()); - } else { - setMeasInfoFromMeasType(currentMeasInfo, filteredMeasInfos, filter.get()); - } - } + List<MeasurementInfo> filteredMeasInfos = filterMeasInfos(measInfos, filterPatternList); if (filteredMeasInfos.isEmpty()) { logger.unwrap().info("No filter match from the current measurement file."); @@ -90,27 +86,43 @@ public class MeasFilterHandler { return true; } - /** - * Filters each measInfo node in the list for measTypes that match the given measTypes from filters. - **/ - public boolean filterByMeasType(List<Event> events) { - boolean hasMatchAnyFilter = false; - for (int i = 0; i < events.size(); i++) { - Event currentEvent = events.get(i); - boolean hasMatchingFilter = filterByMeasType(currentEvent); - if (hasMatchingFilter) { - hasMatchAnyFilter = true; + private List<MeasurementInfo> filterMeasInfos(List<MeasurementInfo> measInfos, List<Pattern> filterPatternList) { + List<MeasurementInfo> filteredMeasInfos = new LinkedList<>(); + for (MeasurementInfo currentMeasInfo : measInfos) { + List<String> measTypesNode = currentMeasInfo.getMeasTypes(); + if (measTypesNode != null && !measTypesNode.isEmpty()) { + getFilteredMeasInfosFromMeasTypes(currentMeasInfo, filterPatternList).ifPresent(filteredMeasInfos::add); } else { - events.remove(events.get(i)); + getFilteredMeasInfoFromMeasType(currentMeasInfo, filterPatternList).ifPresent(filteredMeasInfos::add); } } + return filteredMeasInfos; + } - if (!hasMatchAnyFilter) { - logger.unwrap().info("No filter match from all measurement files."); - return false; + private List<Pattern> getPatternsFromFilters(Optional<Filter> filters) { + List<Pattern> patternList = new LinkedList<>(); + for (String filter : filters.get().getMeasTypes()) { + tryToCompileFilter(filter).ifPresent(patternList::add); + } + return patternList; + } + private Optional<Pattern> tryToCompileFilter(String measType) { + try { + return Optional.of(Pattern.compile("^" + measType + "$")); + } catch (PatternSyntaxException exception) { + logger.unwrap().warn("Can not parse measType filter: ", exception); } + return Optional.empty(); + } - return true; + /** + * Filters each measInfo node in the list for measTypes that match the given measTypes from filters. + **/ + public boolean filterByMeasType(List<Event> events) { + List<Event> filteredList = events.stream().filter(this::filterByMeasType).collect(Collectors.toList()); + events.clear(); + events.addAll(filteredList); + return !events.isEmpty(); } private boolean hasNoFilters(Optional<Filter> filter) { @@ -162,9 +174,9 @@ public class MeasFilterHandler { return hasResults; } - private void setMeasInfoFromMeasType(MeasurementInfo currentMeasInfo, List<MeasurementInfo> filteredMeasInfos, Filter filter) { + private Optional<MeasurementInfo> getFilteredMeasInfoFromMeasType(MeasurementInfo currentMeasInfo, List<Pattern> filters) { List<MeasurementInfo.MeasType> filteredMeasTypes = currentMeasInfo.getMeasType().stream() - .filter(mt -> filter.getMeasTypes().contains(mt.getValue())) + .filter(mt -> matchFilters(filters, mt.getValue())) .collect(Collectors.toList()); if(!filteredMeasTypes.isEmpty()) { @@ -173,11 +185,16 @@ public class MeasFilterHandler { .collect(Collectors.toList()); currentMeasInfo.replaceMeasType(filteredMeasTypes); currentMeasInfo.replaceMeasValue(filteredMeasValues); - filteredMeasInfos.add(currentMeasInfo); + return Optional.of(currentMeasInfo); } + return Optional.empty(); + } + + private boolean matchFilters(List<Pattern> filters, String measType) { + return filters.stream().anyMatch(filter -> filter.matcher(measType).matches()); } - private void setMeasInfosFromMeasTypes(MeasurementInfo currentMeasInfo, List<MeasurementInfo> filteredMeasInfos, Filter filter) { + private Optional<MeasurementInfo> getFilteredMeasInfosFromMeasTypes(MeasurementInfo currentMeasInfo, List<Pattern> filters) { MeasValue currentMeasValue = currentMeasInfo.getMeasValue() .get(0); List<String> measTypesNode = currentMeasInfo.getMeasTypes(); @@ -187,8 +204,7 @@ public class MeasFilterHandler { List<String> filteredMeasTypes = new ArrayList<>(); for (int j = 0; j < measTypesNode.size(); j++) { String currentMeasType = measTypesNode.get(j); - List<String> measTypeFilters = filter.getMeasTypes(); - if (measTypeFilters.contains(currentMeasType)) { + if (matchFilters(filters, currentMeasType)) { filteredMeasTypes.add(currentMeasType); filteredMeasResults.add(measResultsNode.get(j)); } @@ -197,8 +213,9 @@ public class MeasFilterHandler { if (!filteredMeasTypes.isEmpty()) { currentMeasInfo.replaceMeasTypes(filteredMeasTypes); currentMeasValue.replaceMeasResults(filteredMeasResults); - filteredMeasInfos.add(currentMeasInfo); + return Optional.of(currentMeasInfo); } + return Optional.empty(); } } diff --git a/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java b/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java index ce051e4..0b8cdfc 100644 --- a/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java +++ b/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java @@ -33,13 +33,14 @@ import static org.mockserver.integration.ClientAndServer.startClientAndServer; import static org.mockserver.model.HttpResponse.response; import static utils.ConfigUtils.getMapperConfigFromFile; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Collections; import com.google.gson.Gson; import io.undertow.server.HttpServerExchange; @@ -261,7 +262,7 @@ class AppTest { @Test void testFilter_success() { Event mockEvent = Mockito.mock(Event.class); - List<Event> mockEvents = Arrays.asList(mockEvent); + List<Event> mockEvents = new LinkedList<>(Collections.singletonList(mockEvent)); MapperConfig mockConfig = Mockito.mock(MapperConfig.class); boolean result = App.filter(new MeasFilterHandler(new MeasConverter()), mockEvents, mockConfig); assertTrue(result); diff --git a/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java b/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java index 0462616..3eea765 100644 --- a/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java +++ b/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -58,7 +59,7 @@ class MeasFilterHandlerTest { private static final Path FILTER_DIRECTORY = Paths.get("src/test/resources/filter_test/"); private static final String baseDir = "src/test/resources/filter_test/"; - private static MeasConverter converter = new MeasConverter(); + private static final MeasConverter converter = new MeasConverter(); private MeasFilterHandler objUnderTest; @Mock private HttpServerExchange exchange; @@ -101,10 +102,25 @@ class MeasFilterHandlerTest { } @Test + void remove_all_events_that_does_not_match_filter() { + String inputPath = baseDir + "lte/meas_type_and_r_manyinfo/test.xml"; + + Filter noMatchFilter = new MeasFilterConfig().new Filter(); + noMatchFilter.setMeasTypes(Arrays.asList("ad", "bs")); + Event eventNoMatch = generateEvent(inputPath, noMatchFilter); + + List<Event> events = new ArrayList<>(); + events.add(eventNoMatch); + events.add(eventNoMatch); + assertFalse(objUnderTest.filterByMeasType(events)); + assertEquals(0, events.size()); + } + + @Test void skip_mapping_when_MeasData_isEmpty() { String inputPath = baseDir + "lte/meas_results/test.xml"; Event event = generateEvent(inputPath, generateValidFilter()); - event.getMeasurement().replacementMeasurementData(Arrays.asList()); + event.getMeasurement().replacementMeasurementData(Collections.emptyList()); assertFalse(objUnderTest.filterByMeasType(event)); } @@ -114,7 +130,7 @@ class MeasFilterHandlerTest { String inputPath = baseDir + "lte/meas_results/test.xml"; Filter emptyMeastypesFilter = new MeasFilterConfig().new Filter(); - emptyMeastypesFilter.setMeasTypes(Arrays.asList()); + emptyMeastypesFilter.setMeasTypes(Collections.emptyList()); Event event = generateEvent(inputPath, emptyMeastypesFilter); MeasurementFile originalMeasCollec = event.getMeasurement(); @@ -148,6 +164,18 @@ class MeasFilterHandlerTest { }); } + @Test + void invalid_pattern_test() { + String inputPath = baseDir + "lte/meas_type_and_r_manyinfo/test.xml"; + Filter matchFilter = new MeasFilterConfig().new Filter(); + matchFilter.setMeasTypes(Arrays.asList("a", "b", "......****")); + Event eventMatch = generateEvent(inputPath, matchFilter); + List<Event> events = new ArrayList<>(); + events.add(eventMatch); + assertTrue(objUnderTest.filterByMeasType(events)); + assertEquals(1, events.size()); + } + @ParameterizedTest @MethodSource("getEvents") void filter_valid_measurements(Event expectedEvent, Event testEvent) { @@ -170,7 +198,7 @@ class MeasFilterHandlerTest { Filter filter; filter = new MeasFilterConfig().new Filter(); filter.setDictionaryVersion("1.0"); - filter.setMeasTypes(Arrays.asList("a", "b")); + filter.setMeasTypes(Arrays.asList("a", "b", "aab.*", ".*3", ".******test")); return filter; } diff --git a/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml new file mode 100644 index 0000000..d1044ad --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml @@ -0,0 +1,42 @@ +<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec"> + <fileHeader fileFormatVersion="32.435 V10.0" vendorName="FooBar Ltd" dnPrefix="some dnPrefix"> + <fileSender localDn="Dublin"/> + <measCollec beginTime="2018-10-02T12:00:00+01:00"/> + </fileHeader> + <measData> + <managedElement localDn="Dublin" swVersion="r0.1"/> + <measInfo measInfoId="this will be filtered out"> + <job jobId="jobId"/> + <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/> + <repPeriod duration="PT900S"/> + <measTypes>zz3</measTypes> + <measValue measObjLdn="objLdn"> + <measResults>27</measResults> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="some measInfoId"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measTypes>a b</measTypes> + <measValue measObjLdn="objLdn"> + <measResults>1 2</measResults> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="some measInfoId2"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measTypes>aab1 b</measTypes> + <measValue measObjLdn="objLdn"> + <measResults>1 2</measResults> + <suspect>false</suspect> + </measValue> + </measInfo> + </measData> + <fileFooter> + <measCollec endTime="2018-10-02T12:15:00+01:00"/> + </fileFooter> +</measCollecFile> diff --git a/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml new file mode 100644 index 0000000..890fee4 --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec"> + <fileHeader dnPrefix="some dnPrefix" vendorName="FooBar Ltd" + fileFormatVersion="32.435 V10.0"> + <fileSender localDn="Dublin"/> + <measCollec beginTime="2018-10-02T12:00:00+01:00"/> + </fileHeader> + <measData> + <managedElement swVersion="r0.1" localDn="Dublin"/> + <measInfo measInfoId="this will be filtered out"> + <job jobId="jobId"/> + <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/> + <repPeriod duration="PT900S"/> + <measTypes>z aa zz3 bb</measTypes> + <measValue measObjLdn="objLdn"> + <measResults>99 1 27 2</measResults> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="some measInfoId"> + <job jobId="jobId"/> + <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/> + <repPeriod duration="PT900S"/> + <measTypes>z a zz b</measTypes> + <measValue measObjLdn="objLdn"> + <measResults>99 1 27 2</measResults> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="some measInfoId2"> + <job jobId="jobId"/> + <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/> + <repPeriod duration="PT900S"/> + <measTypes>z aab1 zz b</measTypes> + <measValue measObjLdn="objLdn"> + <measResults>99 1 27 2</measResults> + <suspect>false</suspect> + </measValue> + </measInfo> + </measData> + <fileFooter> + <measCollec endTime="2018-10-02T12:15:00+01:00"/> + </fileFooter> +</measCollecFile> diff --git a/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml new file mode 100644 index 0000000..2a2a967 --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml @@ -0,0 +1,48 @@ +<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec"> + <fileHeader fileFormatVersion="32.435 V10.0" vendorName="FooBar Ltd" dnPrefix="some dnPrefix"> + <fileSender localDn="Dublin"/> + <measCollec beginTime="2018-10-02T12:00:00+01:00"/> + </fileHeader> + <measData> + <managedElement localDn="Dublin" swVersion="r0.1"/> + <measInfo measInfoId="some measInfoId"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measType p="1">aab1</measType> + <measType p="2">aab2</measType> + <measType p="4">b</measType> + <measValue measObjLdn="some measObjLdn"> + <r p="1">1</r> + <r p="2">99</r> + <r p="4">2</r> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="filter will disregard this measInfo"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measType p="1">aab1</measType> + <measValue measObjLdn="some measObjLdn"> + <r p="1">1</r> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="some measInfoId2"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measType p="1">aab1</measType> + <measType p="3">bb3</measType> + <measValue measObjLdn="some measObjLdn"> + <r p="1">1</r> + <r p="3">2</r> + <suspect>false</suspect> + </measValue> + </measInfo> + </measData> + <fileFooter> + <measCollec endTime="2018-10-02T12:15:00+01:00"/> + </fileFooter> +</measCollecFile> diff --git a/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml new file mode 100644 index 0000000..9aa668e --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml @@ -0,0 +1,56 @@ +<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec"> + <fileHeader fileFormatVersion="32.435 V10.0" vendorName="FooBar Ltd" dnPrefix="some dnPrefix"> + <fileSender localDn="Dublin"/> + <measCollec beginTime="2018-10-02T12:00:00+01:00"/> + </fileHeader> + <measData> + <managedElement localDn="Dublin" swVersion="r0.1"/> + <measInfo measInfoId="some measInfoId"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measType p="1">aab1</measType> + <measType p="2">aab2</measType> + <measType p="3">z</measType> + <measType p="4">b</measType> + <measValue measObjLdn="some measObjLdn"> + <r p="1">1</r> + <r p="2">99</r> + <r p="3">3</r> + <r p="4">2</r> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="filter will disregard this measInfo"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measType p="1">aab1</measType> + <measType p="2">z</measType> + <measType p="3">bb</measType> + <measValue measObjLdn="some measObjLdn"> + <r p="1">1</r> + <r p="2">99</r> + <r p="3">2</r> + <suspect>false</suspect> + </measValue> + </measInfo> + <measInfo measInfoId="some measInfoId2"> + <job jobId="jobId"/> + <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/> + <repPeriod duration="PT900S"/> + <measType p="1">aab1</measType> + <measType p="2">z</measType> + <measType p="3">bb3</measType> + <measValue measObjLdn="some measObjLdn"> + <r p="1">1</r> + <r p="2">99</r> + <r p="3">2</r> + <suspect>false</suspect> + </measValue> + </measInfo> + </measData> + <fileFooter> + <measCollec endTime="2018-10-02T12:15:00+01:00"/> + </fileFooter> +</measCollecFile> |