From fa2a5a43c82cd35cca9e7d4b51f83ce70e1e3e59 Mon Sep 17 00:00:00 2001 From: Daniel Cruz Date: Thu, 14 Feb 2019 00:42:45 -0600 Subject: Add Nested JSON Filtering The refactoring of the JsonProtocolFilter allows for complex filter queries for JSON events coming in on network topics. The underlying library used is json-path, which supports path based searching and ruby regex filtering at any level of the JSON document. The JsonProtocolFilter no longer requires a FilterRule class as one json-path filter can contain multiple constraints using "&&" or "||". This eliminates the need to identify the field name as a key with a regex associated with it in the controller properties file (see the original FilterRule implementation). It also simplifies the parsing needed in the DroolsControllerFactory when extracting the filter property and creating the JsonProtocolFilter. JUnit coverage is 100% and all sonar flags were addressed related to the JsonProtocolFilter class. Tested and verified working with the telemetry API in a locally deployed PDP-D. Change-Id: I8bd63db4e497c1ba0b5044b5449ccb7a9e4dbdbc Issue-ID: POLICY-1489 Signed-off-by: Daniel Cruz --- .../protocol/coders/JsonProtocolFilterTest.java | 317 ++++++++------------- .../protocol/coders/ProtocolCoderToolsetTest.java | 45 +-- 2 files changed, 141 insertions(+), 221 deletions(-) (limited to 'policy-management/src/test/java') diff --git a/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/JsonProtocolFilterTest.java b/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/JsonProtocolFilterTest.java index e83e5880..e0baeb6d 100644 --- a/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/JsonProtocolFilterTest.java +++ b/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/JsonProtocolFilterTest.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * ONAP * ================================================================================ - * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,205 +24,140 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.util.ArrayList; -import java.util.List; - import org.junit.Test; -import org.onap.policy.drools.protocol.coders.JsonProtocolFilter.FilterRule; -import org.onap.policy.drools.utils.Pair; - public class JsonProtocolFilterTest { - private static final String NAME1 = "name1"; - private static final String REGEX1 = "regex1"; + private static final String JSON = + "{\"requestID\":\"38adde30-cc22-11e8-a8d5-f2801f1b9fd1\",\"entity\":\"controller\"," + + "\"controllers\":[{\"name\":\"test-controller\"," + + "\"drools\":{\"groupId\":\"org.onap.policy.drools.test\"," + + "\"artifactId\":\"test\",\"version\":\"0.0.1\"},\"operation\":\"update\"}]}"; + + /** + * Tests getting the rule expression of the filter. + */ + @Test + public void getRuleTest() { + assertEquals("$.test", new JsonProtocolFilter("$.test").getRule()); + } + + /** + * Tests setting the rule expression of the filter. + */ + @Test + public void setRuleTest() { + JsonProtocolFilter filter = new JsonProtocolFilter(); + assertEquals(JsonProtocolFilter.MATCH_ANY, filter.getRule()); + filter.setRule("$.test"); + assertEquals("$.test", filter.getRule()); + } + + /** + * Tests that the rule expression will be set to match anything if an empty string is passed. + */ + @Test + public void setRuleEmptyTest() { + assertEquals(JsonProtocolFilter.MATCH_ANY, new JsonProtocolFilter("").getRule()); + } + + /** + * Tests that the rule expression will be set to match anything if a null string is passed. + */ + @Test + public void setRuleNullTest() { + assertEquals(JsonProtocolFilter.MATCH_ANY, new JsonProtocolFilter(null).getRule()); + } + + /** + * Tests accepting a message if all filter rules pass. + */ + @Test + public void acceptPassTest() { + assertTrue(new JsonProtocolFilter( + "$.controllers[?(@.drools.version =~ /\\d\\.\\d\\.\\d/ && @.operation == 'update')]") + .accept(JSON)); + } + + /** + * Tests accepting a message without having to filter if the rule is set to match anything. + */ + @Test + public void acceptAnyTest() { + assertTrue(new JsonProtocolFilter(null).accept(JSON)); + } - private static final String NAME2 = "name2"; - private static final String REGEX2 = "regex2"; + /** + * Tests rejecting a message if one or more of the filter rules fail. + */ + @Test + public void acceptFailTest() { + assertFalse( + new JsonProtocolFilter("$.controllers[?(@.drools.version =~ /\\\\d\\\\.\\\\d\\\\.2/)]") + .accept(JSON)); + } - private static final String NAME3 = "name3"; - private static final String REGEX3 = "regex3"; + /** + * Tests finding field matches for a filter rule corresponding to a topic. + */ + @Test + public void filterPassTest() { + assertEquals("38adde30-cc22-11e8-a8d5-f2801f1b9fd1", new JsonProtocolFilter("$.requestID").filter(JSON).get(0)); + } + + /** + * Tests that an empty list is returned when no matches are found. + */ + @Test + public void filterFailTest() { + assertTrue(new JsonProtocolFilter("$.test").filter(JSON).isEmpty()); + } - private static final String NAME4 = "name4"; - private static final String REGEX4a = "^regex4a.*"; - private static final String REGEX4b = ".*regex4b$"; - + /** + * Tests static method for filtering a JSON string with an arbitrary expression. + */ + @Test + public void staticFilterPassTest() { + assertEquals("controller", JsonProtocolFilter.filter(JSON, "$.entity").get(0)); + } + /** + * Tests that an empty list is returned when the static filter() method does not find any matches. + */ @Test - public void test() { - - // ******************** D E F I N E f i l t e r R u l e O b j e c t s *************************** - // DEFINE one (1) filterRule object (using constructor without parms passed; instead use set methods - FilterRule filterRule1 = new FilterRule(); - filterRule1.setName(NAME1); - filterRule1.setRegex(REGEX1); - assertEquals(filterRule1.getName(), NAME1); - assertEquals(filterRule1.getRegex(), REGEX1); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // DEFINE four (4) filterRule objects (using constructor passing the field values - FilterRule filterRule2 = new FilterRule(NAME2, REGEX2); - assertEquals(filterRule2.getName(), NAME2); - assertEquals(filterRule2.getRegex(), REGEX2); - - FilterRule filterRule3 = new FilterRule(NAME3, REGEX3); - assertEquals(filterRule3.getName(), NAME3); - assertEquals(filterRule3.getRegex(), REGEX3); - - FilterRule filterRule4a = new FilterRule(NAME4, REGEX4a); - assertEquals(filterRule4a.getName(), NAME4); - assertEquals(filterRule4a.getRegex(), REGEX4a); - - FilterRule filterRule4b = new FilterRule(NAME4, REGEX4b); - assertEquals(filterRule4b.getName(), NAME4); - assertEquals(filterRule4b.getRegex(), REGEX4b); - - - - // ************************ D E F I N E f i l t e r L i s t s ************************************ - // DEFINE rawFiltersA - List> rawFiltersA = new ArrayList<>(); - rawFiltersA.add(new Pair(filterRule1.getName(), filterRule1.getRegex())); - rawFiltersA.add(new Pair(filterRule2.getName(), filterRule2.getRegex())); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // DEFINE filtersA - List filtersA = new ArrayList<>(); - for (Pair filterPair: rawFiltersA) { - if (filterPair.first() == null || filterPair.first().isEmpty()) { - continue; - } - filtersA.add(new FilterRule(filterPair.first(), filterPair.second())); - } - - - - // *********** I N S T A N T I A T E J s o n P r o t o c o l F i l t e r O b j e c t s ********** - // INSTANTIATE protocolFilterA (passing raw filters to the 'fromRawFilters' constructor) - JsonProtocolFilter protocolFilterA = JsonProtocolFilter.fromRawFilters(rawFiltersA); - assertTrue(protocolFilterA.isRules()); - assertEquals(protocolFilterA.getRules().toString(), filtersA.toString()); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // INSTANTIATE protocolFilterB (passing filters list to constructor which accepts such) - JsonProtocolFilter protocolFilterB = new JsonProtocolFilter(filtersA); - assertTrue(protocolFilterB.isRules()); - assertEquals(protocolFilterB.getRules(), filtersA); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // INSTANTIATE protocolFilterC (using constructor without parms; add instead using setRules() method - JsonProtocolFilter protocolFilterC = new JsonProtocolFilter(); - protocolFilterC.setRules(filtersA); - assertTrue(protocolFilterC.isRules()); - assertEquals(protocolFilterC.getRules(), filtersA); - - - - // *** D E F I N E o t h e r f i l t e r L i s t s f o r v a l i d a t i o n s ************ - // DEFINE rawFiltersB - List> rawFiltersB = new ArrayList<>(); - rawFiltersB.add(new Pair(filterRule1.getName(), filterRule1.getRegex())); - rawFiltersB.add(new Pair(filterRule2.getName(), filterRule2.getRegex())); - rawFiltersB.add(new Pair(filterRule3.getName(), filterRule3.getRegex())); - rawFiltersB.add(new Pair(filterRule4a.getName(), filterRule4a.getRegex())); - rawFiltersB.add(new Pair(filterRule4b.getName(), filterRule4b.getRegex())); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // DEFINE filtersB - List filtersB = new ArrayList<>(); - for (Pair filterPair: rawFiltersB) { - filtersB.add(new FilterRule(filterPair.first(), filterPair.second())); - } - - - - // *********** A D D T O p r o t o c o l F i l t e r B 3 m o r e f i l t e r s ************ - protocolFilterB.addRule(filterRule3.getName(), filterRule3.getRegex()); - protocolFilterB.addRule(filterRule4a.getName(), filterRule4a.getRegex()); - protocolFilterB.addRule(filterRule4b.getName(), filterRule4b.getRegex()); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // VALIDATE that protocolFilterB now contains filters passed using filtersA and the new ones just added - assertEquals(protocolFilterB.getRules().toString(), filtersB.toString()); - - - - // ************ D E L E T E f i l t e r s f r o m p r o t o c o l F i l t e r B *********** - // DELETE specific filter from protocolFilterB by passing both the name & regex values - - assertTrue(protocolFilterB.getRules(NAME3).size() == 1); - assertTrue(protocolFilterB.getRules(NAME3).get(0).getName().equals(NAME3)); - assertTrue(protocolFilterB.getRules(NAME3).get(0).getRegex().equals(REGEX3)); - - assertTrue(protocolFilterB.getRules(NAME4).size() == 2); - assertTrue(protocolFilterB.getRules(NAME4).get(0).getName().equals(NAME4)); - assertTrue(protocolFilterB.getRules(NAME4).get(0).getRegex().equals(REGEX4a)); - assertTrue(protocolFilterB.getRules(NAME4).get(1).getName().equals(NAME4)); - assertTrue(protocolFilterB.getRules(NAME4).get(1).getRegex().equals(REGEX4b)); - - final String jsonA = "{ \"name1\":\"regex1\",\"name2\":\"regex2\"," - + "\"name3\":\"regex3\",\"name4\":\"regex4a\",\"name4\":\"regex4b\"}"; - final String jsonB = "{ \"name1\":\"regex1\",\"name2\":\"regex2\"," - + "\"name3\":\"regex3\",\"name4\":\"regex4a-regex4b\"}"; - final String jsonC = "{ \"name1\":\"regex1\",\"name2\":\"regex2\"," - + "\"name3\":\"regex3\",\"name4\":\"regex4a\"}"; - final String jsonD = "{ \"name1\":\"regex1\",\"name2\":\"regex2\"," - + "\"name3\":\"regex3\",\"name4\":\"regex4b\"}"; - - assertFalse(protocolFilterB.accept(jsonA)); - assertTrue(protocolFilterB.accept(jsonB)); - assertFalse(protocolFilterB.accept(jsonC)); - assertFalse(protocolFilterB.accept(jsonD)); - - protocolFilterB.deleteRule(NAME4, REGEX4a); - - assertTrue(protocolFilterB.accept(jsonA)); - assertTrue(protocolFilterB.accept(jsonB)); - assertFalse(protocolFilterB.accept(jsonC)); - assertTrue(protocolFilterB.accept(jsonD)); - - protocolFilterB.addRule(NAME4, REGEX4a); - - assertTrue(protocolFilterB.getRules(NAME4).size() == 2); - assertTrue(protocolFilterB.getRules(NAME4).get(0).getName().equals(NAME4)); - assertTrue(protocolFilterB.getRules(NAME4).get(0).getRegex().equals(REGEX4b)); - assertTrue(protocolFilterB.getRules(NAME4).get(1).getName().equals(NAME4)); - assertTrue(protocolFilterB.getRules(NAME4).get(1).getRegex().equals(REGEX4a)); - - assertFalse(protocolFilterB.accept(jsonA)); - assertTrue(protocolFilterB.accept(jsonB)); - assertFalse(protocolFilterB.accept(jsonC)); - assertFalse(protocolFilterB.accept(jsonD)); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // DELETE all filters from protocolFilterB that have a match to the same name value - protocolFilterB.deleteRules(NAME4); - - assertTrue(protocolFilterB.getRules(NAME4).isEmpty()); - assertTrue(protocolFilterB.accept(jsonA)); - assertTrue(protocolFilterB.accept(jsonB)); - assertTrue(protocolFilterB.accept(jsonC)); - assertTrue(protocolFilterB.accept(jsonD)); - - assertTrue(protocolFilterB.getRules(NAME3).size() == 1); - protocolFilterB.addRule(NAME3, REGEX3); - assertTrue(protocolFilterB.getRules(NAME3).size() == 1); - protocolFilterB.deleteRule(NAME3, REGEX3); - assertTrue(protocolFilterB.getRules(NAME3).isEmpty()); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // VALIDATE that protocolFilterB now only contains the filters that were originally passed using filtersA - assertEquals(protocolFilterB.getRules(), filtersA); - - // ************ A C C E P T J S O N I F I T P A S S E S A L L F I L T E R S *********** - // ACCEPT TRUE a JSON that passes all filters - String jsonE = "{ \"name1\":\"regex1\",\"name2\":\"regex2\"}"; - assertTrue(protocolFilterA.accept(jsonE)); - - // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // ACCEPT FALSE a JSON that does NOT pass all filters - String jsonF = "{ \"name1\":\"regex1\"}"; - assertFalse(protocolFilterA.accept(jsonF)); + public void staticFilterFailTest() { + assertTrue(JsonProtocolFilter.filter(JSON, "$.test").isEmpty()); } + /** + * Tests that an exception is thrown if a null JSON string is passed. + */ + @Test(expected = IllegalArgumentException.class) + public void staticFilterNullJsonTest() { + JsonProtocolFilter.filter(null, "[?($ =~ /.*/"); + } + + /** + * Tests that an exception is thrown if an empty JSON string is passed. + */ + @Test(expected = IllegalArgumentException.class) + public void staticFilterEmptyJsonTest() { + JsonProtocolFilter.filter("", "[?($ =~ /.*/"); + } + + /** + * Tests that an exception is thrown if a null expression string is passed. + */ + @Test(expected = IllegalArgumentException.class) + public void staticFilterNullExpressionTest() { + JsonProtocolFilter.filter("{\"hello\":\"world\"}", null); + } + + /** + * Tests that an exception is thrown if an empty expression string is passed. + */ + @Test(expected = IllegalArgumentException.class) + public void staticFilterEmptyExpressionTest() { + JsonProtocolFilter.filter("{\"hello\":\"world\"}", ""); + } } \ No newline at end of file diff --git a/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/ProtocolCoderToolsetTest.java b/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/ProtocolCoderToolsetTest.java index da491261..b7d72f31 100644 --- a/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/ProtocolCoderToolsetTest.java +++ b/policy-management/src/test/java/org/onap/policy/drools/protocol/coders/ProtocolCoderToolsetTest.java @@ -25,7 +25,6 @@ import com.google.gson.GsonBuilder; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; import java.util.Properties; import org.junit.Assert; @@ -39,7 +38,6 @@ import org.onap.policy.drools.controller.DroolsController; import org.onap.policy.drools.controller.internal.MavenDroolsControllerTest; import org.onap.policy.drools.properties.DroolsProperties; import org.onap.policy.drools.protocol.coders.EventProtocolCoder.CoderFilters; -import org.onap.policy.drools.protocol.coders.JsonProtocolFilter.FilterRule; import org.onap.policy.drools.protocol.coders.TopicCoderFilterConfiguration.CustomGsonCoder; import org.onap.policy.drools.util.KieUtils; import org.onap.policy.drools.utils.Triple; @@ -64,7 +62,7 @@ public class ProtocolCoderToolsetTest { /** * Setup. - * + * * @throws IOException throws IO Exception */ @Before @@ -96,7 +94,7 @@ public class ProtocolCoderToolsetTest { /** * Test the Gson toolset. - * + * * @param protocolFilter protocol filter */ public void testGsonToolset(JsonProtocolFilter protocolFilter) { @@ -157,11 +155,9 @@ public class ProtocolCoderToolsetTest { CoderFilters coderFilters = coderToolset.getCoder(Triple.class.getCanonicalName()); Assert.assertTrue(coderFilters.getCodedClass() == Triple.class.getCanonicalName()); Assert.assertTrue(coderFilters.getFilter() == protocolFilter); - Assert.assertTrue(coderFilters.getFilter().getRules("second").size() == 1); - Assert.assertTrue(coderFilters.getFilter().getRules("third").size() == 1); + Assert.assertTrue(coderFilters.getFilter().getRule() != null); - coderFilters.getFilter().getRules("second").get(0).setRegex("^v2$"); - coderFilters.getFilter().getRules("third").get(0).setRegex(".*v3.*"); + coderFilters.getFilter().setRule("[?($.second =~ /^v2$/ && $.third =~ /.*v3.*/)]"); tripleDecoded = (Triple) coderToolset.decode(tripleEncoded); @@ -169,8 +165,8 @@ public class ProtocolCoderToolsetTest { Assert.assertTrue(tripleDecoded.second().equals(triple.second())); Assert.assertTrue(tripleDecoded.third().equals(triple.third())); - coderFilters.getFilter().deleteRules("third"); - Assert.assertTrue(coderFilters.getFilter().getRules("third").isEmpty()); + coderFilters.getFilter().setRule(null); + Assert.assertEquals("[?($ =~ /.*/)]", coderFilters.getFilter().getRule()); tripleDecoded = (Triple) coderToolset.decode(tripleEncoded); @@ -178,7 +174,7 @@ public class ProtocolCoderToolsetTest { Assert.assertTrue(tripleDecoded.second().equals(triple.second())); Assert.assertTrue(tripleDecoded.third().equals(triple.third())); - coderFilters.getFilter().addRule("third", ".*v3.*"); + coderFilters.getFilter().setRule("[?($.third =~ /.*v3.*/)]"); } private String encode(ProtocolCoderToolset coderToolset, Triple triple) { @@ -188,10 +184,8 @@ public class ProtocolCoderToolsetTest { } private void addRemoveCoder(ProtocolCoderToolset coderToolset) { - List filters = new ArrayList<>(); - filters.add(new FilterRule("second", ".*")); - - coderToolset.addCoder(this.getClass().getCanonicalName(), new JsonProtocolFilter(filters), 654321); + coderToolset.addCoder(this.getClass().getCanonicalName(), + new JsonProtocolFilter("[?($.second =~ /.*/)]"), 654321); Assert.assertTrue(coderToolset.getCoders().size() == 2); coderToolset.removeCoders(this.getClass().getCanonicalName()); @@ -199,22 +193,18 @@ public class ProtocolCoderToolsetTest { } private void updateCoderFilterRule(ProtocolCoderToolset coderToolset) { - List filters = new ArrayList<>(); - filters.add(new FilterRule("third", ".*")); - coderToolset.addCoder(Triple.class.getCanonicalName(), new JsonProtocolFilter(filters), 654321); + coderToolset.addCoder(Triple.class.getCanonicalName(), new JsonProtocolFilter("[?($.third =~ /.*/)]"), 654321); Assert.assertTrue(coderToolset.getCoders().size() == 1); Assert.assertTrue(coderToolset.getCoder(Triple.class.getCanonicalName()).getModelClassLoaderHash() == 654321); Assert.assertTrue( - coderToolset.getCoder(Triple.class.getCanonicalName()).getFilter().getRules("third").size() == 1); - - Assert.assertTrue( - coderToolset.getCoder(Triple.class.getCanonicalName()).getFilter().getRules("third").size() == 1); + coderToolset.getCoder( + Triple.class.getCanonicalName()).getFilter().getRule() != null); - Assert.assertTrue(".*".equals(coderToolset.getCoder(Triple.class.getCanonicalName()).getFilter() - .getRules("third").get(0).getRegex())); + Assert.assertTrue("[?($.third =~ /.*/)]".equals(coderToolset.getCoder(Triple.class.getCanonicalName()) + .getFilter().getRule())); } private void validateInitialization(JsonProtocolFilter protocolFilter, ProtocolCoderToolset coderToolset) { @@ -254,11 +244,6 @@ public class ProtocolCoderToolsetTest { } private JsonProtocolFilter createFilterSet() { - List filters = new ArrayList<>(); - filters.add(new FilterRule("first", ".*")); - filters.add(new FilterRule("second", "^blah.*")); - filters.add(new FilterRule("third", "^hello$")); - - return new JsonProtocolFilter(filters); + return new JsonProtocolFilter("[?($.first =~ /.*/ && $.second =~ /^blah.*/ && $.third =~ /^hello$/)]"); } } -- cgit 1.2.3-korg