From ba384eeb3cc007ff1d48db04fecc9356069456c5 Mon Sep 17 00:00:00 2001 From: marcinrzepeckiwroc Date: Wed, 15 Jul 2020 13:26:51 +0200 Subject: implement regex support in PM Mapper filtering mechanism Issue-ID: DCAEGEN2-2347 Signed-off-by: marcinrzepeckiwroc Change-Id: I3fdbae33ae318800ccc2758a191604fe79c2aaf4 --- .../org/onap/dcaegen2/services/pmmapper/App.java | 10 +-- .../pmmapper/filtering/MeasFilterHandler.java | 79 +++++++++++++--------- .../onap/dcaegen2/services/pmmapper/AppTest.java | 5 +- .../pmmapper/filtering/MeasFilterHandlerTest.java | 36 ++++++++-- .../lte/meas_results_manyinfo_regex/expected.xml | 42 ++++++++++++ .../lte/meas_results_manyinfo_regex/test.xml | 44 ++++++++++++ .../meas_type_and_r_manyinfo_regex/expected.xml | 48 +++++++++++++ .../lte/meas_type_and_r_manyinfo_regex/test.xml | 56 +++++++++++++++ 8 files changed, 278 insertions(+), 42 deletions(-) create mode 100644 src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml create mode 100644 src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml create mode 100644 src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml create mode 100644 src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml diff --git a/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java b/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java index 21e4441..6aeeaba 100644 --- a/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java +++ b/src/main/java/org/onap/dcaegen2/services/pmmapper/App.java @@ -59,7 +59,6 @@ import javax.net.ssl.SSLContext; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; @Data @@ -213,6 +212,7 @@ public class App { try { hasMatchingFilter = filterHandler.filterByMeasType(events); if(!hasMatchingFilter) { + logger.unwrap().info("No filter match from all measurement files."); sendEventProcessed(config,event); } } catch (Exception exception) { @@ -223,25 +223,25 @@ public class App { } public static Flux> map(Mapper mapper, List events, MapperConfig config) { - List mappedEvents = new ArrayList<>(); + List mappedEvents; try { mappedEvents = mapper.mapEvents(events); } catch (Exception exception) { logger.unwrap().error("Unable to map XML to VES",exception); sendEventProcessed(config,events.get(0)); - return Flux.>empty(); + return Flux.empty(); } return Flux.just(mappedEvents); } public static Flux> split(MeasSplitter splitter, Event event, MapperConfig config) { - List splitEvents = new ArrayList<>(); + List splitEvents; try { splitEvents = splitter.split(event); } catch (Exception exception) { logger.unwrap().error("Unable to split MeasCollecFile",exception); sendEventProcessed(config,event); - return Flux.>empty(); + return Flux.empty(); } return Flux.just(splitEvents); } diff --git a/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java b/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java index fb5772e..7f2a5d6 100644 --- a/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java +++ b/src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java @@ -21,8 +21,11 @@ package org.onap.dcaegen2.services.pmmapper.filtering; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.Optional; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import org.apache.commons.io.FilenameUtils; @@ -68,16 +71,9 @@ public class MeasFilterHandler { logger.unwrap().info("Filtering the measurement file by measTypes."); MeasurementData measData = measurementFile.getMeasurementData().get().get(0); List measInfos = measData.getMeasurementInfo(); - List filteredMeasInfos = new ArrayList<>(); + List filterPatternList = getPatternsFromFilters(filter); - for (MeasurementInfo currentMeasInfo : measInfos) { - List measTypesNode = currentMeasInfo.getMeasTypes(); - if (measTypesNode != null && !measTypesNode.isEmpty()) { - setMeasInfosFromMeasTypes(currentMeasInfo, filteredMeasInfos, filter.get()); - } else { - setMeasInfoFromMeasType(currentMeasInfo, filteredMeasInfos, filter.get()); - } - } + List filteredMeasInfos = filterMeasInfos(measInfos, filterPatternList); if (filteredMeasInfos.isEmpty()) { logger.unwrap().info("No filter match from the current measurement file."); @@ -90,27 +86,43 @@ public class MeasFilterHandler { return true; } - /** - * Filters each measInfo node in the list for measTypes that match the given measTypes from filters. - **/ - public boolean filterByMeasType(List events) { - boolean hasMatchAnyFilter = false; - for (int i = 0; i < events.size(); i++) { - Event currentEvent = events.get(i); - boolean hasMatchingFilter = filterByMeasType(currentEvent); - if (hasMatchingFilter) { - hasMatchAnyFilter = true; + private List filterMeasInfos(List measInfos, List filterPatternList) { + List filteredMeasInfos = new LinkedList<>(); + for (MeasurementInfo currentMeasInfo : measInfos) { + List measTypesNode = currentMeasInfo.getMeasTypes(); + if (measTypesNode != null && !measTypesNode.isEmpty()) { + getFilteredMeasInfosFromMeasTypes(currentMeasInfo, filterPatternList).ifPresent(filteredMeasInfos::add); } else { - events.remove(events.get(i)); + getFilteredMeasInfoFromMeasType(currentMeasInfo, filterPatternList).ifPresent(filteredMeasInfos::add); } } + return filteredMeasInfos; + } - if (!hasMatchAnyFilter) { - logger.unwrap().info("No filter match from all measurement files."); - return false; + private List getPatternsFromFilters(Optional filters) { + List patternList = new LinkedList<>(); + for (String filter : filters.get().getMeasTypes()) { + tryToCompileFilter(filter).ifPresent(patternList::add); + } + return patternList; + } + private Optional tryToCompileFilter(String measType) { + try { + return Optional.of(Pattern.compile("^" + measType + "$")); + } catch (PatternSyntaxException exception) { + logger.unwrap().warn("Can not parse measType filter: ", exception); } + return Optional.empty(); + } - return true; + /** + * Filters each measInfo node in the list for measTypes that match the given measTypes from filters. + **/ + public boolean filterByMeasType(List events) { + List filteredList = events.stream().filter(this::filterByMeasType).collect(Collectors.toList()); + events.clear(); + events.addAll(filteredList); + return !events.isEmpty(); } private boolean hasNoFilters(Optional filter) { @@ -162,9 +174,9 @@ public class MeasFilterHandler { return hasResults; } - private void setMeasInfoFromMeasType(MeasurementInfo currentMeasInfo, List filteredMeasInfos, Filter filter) { + private Optional getFilteredMeasInfoFromMeasType(MeasurementInfo currentMeasInfo, List filters) { List filteredMeasTypes = currentMeasInfo.getMeasType().stream() - .filter(mt -> filter.getMeasTypes().contains(mt.getValue())) + .filter(mt -> matchFilters(filters, mt.getValue())) .collect(Collectors.toList()); if(!filteredMeasTypes.isEmpty()) { @@ -173,11 +185,16 @@ public class MeasFilterHandler { .collect(Collectors.toList()); currentMeasInfo.replaceMeasType(filteredMeasTypes); currentMeasInfo.replaceMeasValue(filteredMeasValues); - filteredMeasInfos.add(currentMeasInfo); + return Optional.of(currentMeasInfo); } + return Optional.empty(); + } + + private boolean matchFilters(List filters, String measType) { + return filters.stream().anyMatch(filter -> filter.matcher(measType).matches()); } - private void setMeasInfosFromMeasTypes(MeasurementInfo currentMeasInfo, List filteredMeasInfos, Filter filter) { + private Optional getFilteredMeasInfosFromMeasTypes(MeasurementInfo currentMeasInfo, List filters) { MeasValue currentMeasValue = currentMeasInfo.getMeasValue() .get(0); List measTypesNode = currentMeasInfo.getMeasTypes(); @@ -187,8 +204,7 @@ public class MeasFilterHandler { List filteredMeasTypes = new ArrayList<>(); for (int j = 0; j < measTypesNode.size(); j++) { String currentMeasType = measTypesNode.get(j); - List measTypeFilters = filter.getMeasTypes(); - if (measTypeFilters.contains(currentMeasType)) { + if (matchFilters(filters, currentMeasType)) { filteredMeasTypes.add(currentMeasType); filteredMeasResults.add(measResultsNode.get(j)); } @@ -197,8 +213,9 @@ public class MeasFilterHandler { if (!filteredMeasTypes.isEmpty()) { currentMeasInfo.replaceMeasTypes(filteredMeasTypes); currentMeasValue.replaceMeasResults(filteredMeasResults); - filteredMeasInfos.add(currentMeasInfo); + return Optional.of(currentMeasInfo); } + return Optional.empty(); } } diff --git a/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java b/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java index ce051e4..0b8cdfc 100644 --- a/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java +++ b/src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java @@ -33,13 +33,14 @@ import static org.mockserver.integration.ClientAndServer.startClientAndServer; import static org.mockserver.model.HttpResponse.response; import static utils.ConfigUtils.getMapperConfigFromFile; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Collections; import com.google.gson.Gson; import io.undertow.server.HttpServerExchange; @@ -261,7 +262,7 @@ class AppTest { @Test void testFilter_success() { Event mockEvent = Mockito.mock(Event.class); - List mockEvents = Arrays.asList(mockEvent); + List mockEvents = new LinkedList<>(Collections.singletonList(mockEvent)); MapperConfig mockConfig = Mockito.mock(MapperConfig.class); boolean result = App.filter(new MeasFilterHandler(new MeasConverter()), mockEvents, mockConfig); assertTrue(result); diff --git a/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java b/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java index 0462616..3eea765 100644 --- a/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java +++ b/src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -58,7 +59,7 @@ class MeasFilterHandlerTest { private static final Path FILTER_DIRECTORY = Paths.get("src/test/resources/filter_test/"); private static final String baseDir = "src/test/resources/filter_test/"; - private static MeasConverter converter = new MeasConverter(); + private static final MeasConverter converter = new MeasConverter(); private MeasFilterHandler objUnderTest; @Mock private HttpServerExchange exchange; @@ -100,11 +101,26 @@ class MeasFilterHandlerTest { assertEquals(1, events.size()); } + @Test + void remove_all_events_that_does_not_match_filter() { + String inputPath = baseDir + "lte/meas_type_and_r_manyinfo/test.xml"; + + Filter noMatchFilter = new MeasFilterConfig().new Filter(); + noMatchFilter.setMeasTypes(Arrays.asList("ad", "bs")); + Event eventNoMatch = generateEvent(inputPath, noMatchFilter); + + List events = new ArrayList<>(); + events.add(eventNoMatch); + events.add(eventNoMatch); + assertFalse(objUnderTest.filterByMeasType(events)); + assertEquals(0, events.size()); + } + @Test void skip_mapping_when_MeasData_isEmpty() { String inputPath = baseDir + "lte/meas_results/test.xml"; Event event = generateEvent(inputPath, generateValidFilter()); - event.getMeasurement().replacementMeasurementData(Arrays.asList()); + event.getMeasurement().replacementMeasurementData(Collections.emptyList()); assertFalse(objUnderTest.filterByMeasType(event)); } @@ -114,7 +130,7 @@ class MeasFilterHandlerTest { String inputPath = baseDir + "lte/meas_results/test.xml"; Filter emptyMeastypesFilter = new MeasFilterConfig().new Filter(); - emptyMeastypesFilter.setMeasTypes(Arrays.asList()); + emptyMeastypesFilter.setMeasTypes(Collections.emptyList()); Event event = generateEvent(inputPath, emptyMeastypesFilter); MeasurementFile originalMeasCollec = event.getMeasurement(); @@ -148,6 +164,18 @@ class MeasFilterHandlerTest { }); } + @Test + void invalid_pattern_test() { + String inputPath = baseDir + "lte/meas_type_and_r_manyinfo/test.xml"; + Filter matchFilter = new MeasFilterConfig().new Filter(); + matchFilter.setMeasTypes(Arrays.asList("a", "b", "......****")); + Event eventMatch = generateEvent(inputPath, matchFilter); + List events = new ArrayList<>(); + events.add(eventMatch); + assertTrue(objUnderTest.filterByMeasType(events)); + assertEquals(1, events.size()); + } + @ParameterizedTest @MethodSource("getEvents") void filter_valid_measurements(Event expectedEvent, Event testEvent) { @@ -170,7 +198,7 @@ class MeasFilterHandlerTest { Filter filter; filter = new MeasFilterConfig().new Filter(); filter.setDictionaryVersion("1.0"); - filter.setMeasTypes(Arrays.asList("a", "b")); + filter.setMeasTypes(Arrays.asList("a", "b", "aab.*", ".*3", ".******test")); return filter; } diff --git a/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml new file mode 100644 index 0000000..d1044ad --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + zz3 + + 27 + false + + + + + + + a b + + 1 2 + false + + + + + + + aab1 b + + 1 2 + false + + + + + + + diff --git a/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml new file mode 100644 index 0000000..890fee4 --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + z aa zz3 bb + + 99 1 27 2 + false + + + + + + + z a zz b + + 99 1 27 2 + false + + + + + + + z aab1 zz b + + 99 1 27 2 + false + + + + + + + diff --git a/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml new file mode 100644 index 0000000..2a2a967 --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + aab1 + aab2 + b + + 1 + 99 + 2 + false + + + + + + + aab1 + + 1 + false + + + + + + + aab1 + bb3 + + 1 + 2 + false + + + + + + + diff --git a/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml new file mode 100644 index 0000000..9aa668e --- /dev/null +++ b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + aab1 + aab2 + z + b + + 1 + 99 + 3 + 2 + false + + + + + + + aab1 + z + bb + + 1 + 99 + 2 + false + + + + + + + aab1 + z + bb3 + + 1 + 99 + 2 + false + + + + + + + -- cgit 1.2.3-korg