Merge "Complete filters for Database Fetches"

27 files changed, 440 insertions, 86 deletions

diff --git a/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java b/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java
index 47238fc40..7bdced77b 100644
--- a/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java
+++ b/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java
@@ -20,6 +20,9 @@
 
 package org.onap.policy.models.base;
 
+import org.apache.commons.lang3.ObjectUtils;
+import org.onap.policy.common.utils.validation.Version;
+
 /**
  * An interface that forces a POJO to have getName() and getVersion() methods.
  *
@@ -33,4 +36,43 @@ public interface PfNameVersion {
     public String getVersion();
 
     public void setVersion(final String version);
+
+    /**
+     * COmpare two name version implementation objects.
+     *
+     * @param left the left name/version implementation
+     * @param right the right name/version implementation
+     * @return the comparison resilt
+     */
+    public default int compareNameVersion(final PfNameVersion left, final PfNameVersion right) {
+        if (left == null && right == null) {
+            return 0;
+        }
+
+        if (left == null) {
+            return 1;
+        }
+
+        if (right == null) {
+            return -1;
+        }
+
+        int result = ObjectUtils.compare(left.getName(), right.getName());
+        if (result != 0) {
+            return result;
+        }
+
+        if (left.getVersion() == null && right.getVersion() == null) {
+            return 0;
+        }
+
+        if (left.getVersion() == null) {
+            return 1;
+        }
+
+        if (right.getVersion() == null) {
+            return -1;
+        }
+        return new Version(left.getVersion()).compareTo(new Version(right.getVersion()));
+    }
 }
diff --git a/models-base/src/main/java/org/onap/policy/models/base/PfObjectFiler.java b/models-base/src/main/java/org/onap/policy/models/base/PfObjectFiler.java
deleted file mode 100644
index f1481bf3c..000000000
--- a/models-base/src/main/java/org/onap/policy/models/base/PfObjectFiler.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*-
- * ============LICENSE_START=======================================================
- *  Copyright (C) 2019 Nordix Foundation.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-package org.onap.policy.models.base;
-
-import java.util.List;
-
-/**
- * Interface for filtering a list of concepts.
- *
- * @author Liam Fallon (liam.fallon@est.tech)
- */
-@FunctionalInterface
-public interface PfObjectFiler<T> {
-    /**
-     * Filter an incoming list, removing items that do not match the filter.
-     *
-     * @param originalList the original list
-     * @return the filtered list
-     */
-    public List<T> filter(final List<T> originalList);
-
-}
diff --git a/models-base/src/main/java/org/onap/policy/models/base/PfObjectFilter.java b/models-base/src/main/java/org/onap/policy/models/base/PfObjectFilter.java
new file mode 100644
index 000000000..a377c24be
--- /dev/null
+++ b/models-base/src/main/java/org/onap/policy/models/base/PfObjectFilter.java
@@ -0,0 +1,84 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2019 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.models.base;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import lombok.NonNull;
+
+/**
+ * Interface for filtering a list of concepts.
+ *
+ * @author Liam Fallon (liam.fallon@est.tech)
+ */
+@FunctionalInterface
+public interface PfObjectFilter<T extends Comparable<T>> {
+    /**
+     * Filter an incoming list, removing items that do not match the filter.
+     *
+     * @param originalList the original list
+     * @return the filtered list
+     */
+    public List<T> filter(final List<T> originalList);
+
+    /**
+     * Check if a value matches a regular expression.
+     *
+     * @param value the incoming value to check
+     * @param regexp the regular expression to check against
+     * @return match or not
+     */
+    public default boolean filterOnRegexp(@NonNull final String value, @NonNull final String regexp) {
+        return value.matches(regexp);
+    }
+
+    /**
+     * Sort an incoming list and remove all but the latest version of each concept.
+     *
+     * @param originalList the incoming list
+     * @return the filtered list
+     */
+    public default List<T> latestVersionFilter(final List<T> originalList) {
+        List<T> filteredList = new ArrayList<>();
+        Collections.sort(filteredList);
+
+        List<T> filteredOutList = new ArrayList<>();
+
+        for (int i = 1; i < filteredList.size(); i++) {
+            // Get the current and last element
+            T thisElement = filteredList.get(i);
+            T lastElement = filteredList.get(i - 1);
+
+            // The list is sorted so if the last element name is the same as the current element name, the last element
+            // should be removed
+            if (((PfNameVersion)thisElement).getName().equals(((PfNameVersion)lastElement).getName())) {
+                filteredOutList.add(lastElement);
+            }
+        }
+
+        // We can now remove these elements
+        filteredList.removeAll(filteredOutList);
+
+        return filteredList;
+    }
+}
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml
index c2fd504e9..1dcc5af86 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml
@@ -45,6 +45,7 @@ policy_types:
                 entry_schema:
                     type: string
 data_types:
+  -
     policy.data.affinityProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml
index 93ddd631a..458d152e0 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml
@@ -45,6 +45,7 @@ policy_types:
                 type: policy.data.distanceProperties_properties
                 required: true
 data_types:
+  -
     policy.data.distanceProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml
index 63f0d8ada..be4110517 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml
@@ -36,8 +36,9 @@ policy_types:
                 type: list
                 required: true
                 entry_schema:
-                    type:policy.data.flavorFeatures_properties
+                    type: policy.data.flavorFeatures_properties
 data_types:
+  -
     policy.data.flavorFeatures_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -57,6 +58,7 @@ data_types:
                 required: true
                 entry_schema:
                     type: policy.data.flavorProperties_properties
+  -
     policy.data.directives_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -66,6 +68,7 @@ data_types:
                 type: list
                 entry_schema:
                     type: policy.data.directives_attributes_properties
+  -
     policy.data.directives_attributes_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -73,6 +76,7 @@ data_types:
                 type: string
             attribute_value:
                 type: string
+  -
     policy.data.flavorProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -101,6 +105,7 @@ data_types:
                 required: true
                 entry_schema:
                     type: policy.data.hpa-feature-attributes_properties
+  -
     policy.data.hpa-feature-attributes_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml
index edfac1496..267109861 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml
@@ -40,6 +40,7 @@ policy_types:
                 type: policy.data.objectiveParameter_properties
                 required: true
 data_types:
+  -
     policy.data.objectiveParameter_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml
index 1355eb031..ba4dbec37 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml
@@ -36,8 +36,9 @@ policy_types:
                 type: list
                 required: false
                 entry_schema:
-                -   type:policy.data.pciProperties_properties
+                  type: policy.data.pciProperties_properties
 data_types:
+  -
     policy.data.pciProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml
index f7036dc80..82cd60a46 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml
@@ -31,8 +31,9 @@ policy_types:
                 type: list
                 required: true
                 entry_schema:
-                    type:policy.data.queryProperties_properties
+                    type: policy.data.queryProperties_properties
 data_types:
+  -
     policy.data.queryProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml
index 3c2c2b2c3..ab006fa57 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml
@@ -31,6 +31,7 @@ policy_types:
                 type: policy.data.properties_properties
                 required: true
 data_types:
+  -
     policy.data.properties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml
index 860c37fa0..9c12f7d63 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml
@@ -45,6 +45,7 @@ policy_types:
                 type: policy.data.capacityProperties_properties
                 required: true
 data_types:
+  -
     policy.data.capacityProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml
index 13d4f137c..4bfc24216 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml
@@ -45,8 +45,9 @@ policy_types:
                 type: list
                 required: true
                 entry_schema:
-                    type:policy.data.vnfProperties_properties
+                    type: policy.data.vnfProperties_properties
 data_types:
+  -
     policy.data.vnfProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java
index a59d1af5a..9665fd472 100644
--- a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java
+++ b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java
@@ -40,7 +40,7 @@ import org.onap.policy.models.pdp.enums.PdpState;
  */
 @Data
 @NoArgsConstructor
-public class PdpGroup implements PfNameVersion {
+public class PdpGroup implements PfNameVersion, Comparable<PdpGroup> {
     private String name;
     private String version;
     private String description;
@@ -67,4 +67,9 @@ public class PdpGroup implements PfNameVersion {
         this.properties = (source.properties == null ? null : new LinkedHashMap<>(source.properties));
         this.pdpSubgroups = PfUtils.mapList(source.pdpSubgroups, PdpSubGroup::new);
     }
+
+    @Override
+    public int compareTo(final PdpGroup other) {
+        return compareNameVersion(this, other);
+    }
 }
diff --git a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java
index b49bedefe..c5c0bc541 100644
--- a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java
+++ b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java
@@ -27,20 +27,20 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NonNull;
 
-import org.onap.policy.models.base.PfObjectFiler;
+import org.apache.commons.lang3.ObjectUtils;
+import org.onap.policy.models.base.PfObjectFilter;
 import org.onap.policy.models.pdp.enums.PdpState;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyIdentifier;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeIdentifier;
 
 /**
- * Filter class for searches for {@link PdpGroup} instances.
- * If any fields are null, they are ignored.
+ * Filter class for searches for {@link PdpGroup} instances. If any fields are null, they are ignored.
  *
  * @author Liam Fallon (liam.fallon@est.tech)
  */
 @Builder
 @Data
-public class PdpGroupFilter implements PfObjectFiler<PdpGroup> {
+public class PdpGroupFilter implements PfObjectFilter<PdpGroup> {
     public static final String LATEST_VERSION = "LATEST";
 
     // Regular expression
@@ -60,15 +60,95 @@ public class PdpGroupFilter implements PfObjectFiler<PdpGroup> {
     // Set regular expressions on fields to match policy names and versions
     private ToscaPolicyIdentifier policy;
 
+    private PdpState pdpState;
+
     @Override
     public List<PdpGroup> filter(@NonNull final List<PdpGroup> originalList) {
 
         // @formatter:off
-        return originalList.stream()
-                .filter(p -> name       != null && p.getName()   .matches(name))
-                .filter(p -> version    != null && p.getVersion().matches(version))
-                .filter(p -> groupState != null && p.getPdpGroupState().equals(groupState))
+        List<PdpGroup> returnList = originalList.stream()
+                .filter(p -> filterOnRegexp(p.getName(),    name))
+                .filter(p -> filterOnRegexp(p.getVersion(), version))
+                .filter(p -> ObjectUtils.compare(p.getPdpGroupState(), groupState) == 0)
+                .filter(p -> filterOnPdpType(p, pdpType))
+                .filter(p -> filterOnPolicyType(p, policyType))
+                .filter(p -> filterOnPolicy(p, policy))
                 .collect(Collectors.toList());
         // @formatter:off
+
+        if (LATEST_VERSION.equals(version)) {
+            returnList = this.latestVersionFilter(returnList);
+        }
+
+        return returnList;
+    }
+
+    /**
+     * Filter PDP groups on PDP type.
+     *
+     * @param pdpGroup the PDP group to check
+     * @param pdpType the PDP type to check for
+     * @return true if the filter should let this PDP group through
+     */
+    private boolean filterOnPdpType(final PdpGroup pdpGroup, final String pdpType) {
+        if (pdpType == null) {
+            return true;
+        }
+
+        for (PdpSubGroup pdpSubGroup: pdpGroup.getPdpSubgroups()) {
+            if (pdpSubGroup.getPdpType().equals(pdpType)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Filter PDP groups on policy type.
+     *
+     * @param pdpGroup the PDP group to check
+     * @param policyTypeFilter the policy type regular expressions to check for
+     * @return true if the filter should let this PDP group through
+     */
+    private boolean filterOnPolicyType(final PdpGroup pdpGroup, final ToscaPolicyTypeIdentifier policyTypeFiler) {
+        if (policyTypeFiler == null) {
+            return true;
+        }
+
+        for (PdpSubGroup pdpSubGroup: pdpGroup.getPdpSubgroups()) {
+            for (ToscaPolicyTypeIdentifier foundPolicyType : pdpSubGroup.getSupportedPolicyTypes()) {
+                if (foundPolicyType.getName().matches(policyTypeFiler.getName())
+                        && foundPolicyType.getVersion().matches(policyTypeFiler.getVersion())) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Filter PDP groups on policy.
+     *
+     * @param pdpGroup the PDP group to check
+     * @param policyFilter the policy regular expressions to check for
+     * @return true if the filter should let this PDP group through
+     */
+    private boolean filterOnPolicy(final PdpGroup pdpGroup, final ToscaPolicyIdentifier policyFiler) {
+        if (policyFiler == null) {
+            return true;
+        }
+
+        for (PdpSubGroup pdpSubGroup: pdpGroup.getPdpSubgroups()) {
+            for (ToscaPolicyIdentifier foundPolicy : pdpSubGroup.getPolicies()) {
+                if (foundPolicy.getName().matches(policyFiler.getName())
+                        && foundPolicy.getVersion().matches(policyFiler.getVersion())) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
     }
 }
diff --git a/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java b/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java
index cac506acc..4de1f2ee3 100644
--- a/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java
+++ b/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java
@@ -49,7 +49,7 @@ public class TestPdpGroup {
         // verify with all values
         orig.setDescription("my-descript");
         orig.setName("my-name");
-        orig.setVersion("my-version");
+        orig.setVersion("1.2.3");
         orig.setDescription("my-description");
         orig.setPdpGroupState(PdpState.SAFE);
 
@@ -64,7 +64,7 @@ public class TestPdpGroup {
         props.put("key-B", "value-B");
         orig.setProperties(props);
 
-        assertEquals("PdpGroup(name=my-name, version=my-version, description=my-description, "
+        assertEquals("PdpGroup(name=my-name, version=1.2.3, description=my-description, "
                 + "pdpGroupState=SAFE, properties={key-A=value-A, key-B=value-B}, "
                 + "pdpSubgroups=[PdpSubGroup(pdpType=null, supportedPolicyTypes=[], policies=[], "
                 + "currentInstanceCount=10, desiredInstanceCount=0, properties=null, pdpInstances=[]), "
diff --git a/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyTypePersistenceTest.java b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyTypePersistenceTest.java
new file mode 100644
index 000000000..7f90a0b58
--- /dev/null
+++ b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyTypePersistenceTest.java
@@ -0,0 +1,152 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2019 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.models.provider.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import com.google.gson.GsonBuilder;
+
+import java.util.Base64;
+import java.util.List;
+
+import lombok.NonNull;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.onap.policy.common.utils.coder.StandardCoder;
+import org.onap.policy.common.utils.resources.ResourceUtils;
+import org.onap.policy.models.base.PfModelException;
+import org.onap.policy.models.provider.PolicyModelsProvider;
+import org.onap.policy.models.provider.PolicyModelsProviderFactory;
+import org.onap.policy.models.provider.PolicyModelsProviderParameters;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyType;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * Test persistence of monitoring policies to and from the database.
+ *
+ * @author Liam Fallon (liam.fallon@est.tech)
+ */
+public class PolicyTypePersistenceTest {
+    // Logger for this class
+    private static final Logger LOGGER = LoggerFactory.getLogger(PolicyTypePersistenceTest.class);
+
+    private StandardCoder standardCoder;
+
+    private PolicyModelsProvider databaseProvider;
+
+    // @formatter:off
+    private String[] policyTypeResourceNames = {
+        "policytypes/onap.policies.monitoring.dcaegen2.collectors.datafile.datafile-app-server.yaml",
+        "policytypes/onap.policies.optimization.AffinityPolicy.yaml",
+        "policytypes/onap.policies.optimization.DistancePolicy.yaml",
+        "policytypes/onap.policies.optimization.HpaPolicy.yaml",
+        "policytypes/onap.policies.optimization.OptimizationPolicy.yaml",
+        "policytypes/onap.policies.optimization.PciPolicy.yaml",
+        "policytypes/onap.policies.optimization.QueryPolicy.yaml",
+        "policytypes/onap.policies.optimization.SubscriberPolicy.yaml",
+        "policytypes/onap.policies.optimization.Vim_fit.yaml",
+        "policytypes/onap.policies.optimization.VnfPolicy.yaml",
+        "policytypes/onap.policy.monitoring.cdap.tca.hi.lo.app.yaml"
+    };
+    // @formatter:on
+
+    /**
+     * Initialize provider.
+     *
+     * @throws PfModelException on exceptions in the tests
+     */
+    @Before
+    public void setupParameters() throws PfModelException {
+        PolicyModelsProviderParameters parameters = new PolicyModelsProviderParameters();
+        parameters.setDatabaseUrl("jdbc:h2:mem:testdb");
+        parameters.setDatabaseUser("policy");
+        parameters.setDatabasePassword(Base64.getEncoder().encodeToString("P01icY".getBytes()));
+        parameters.setPersistenceUnit("ToscaConceptTest");
+
+        databaseProvider = new PolicyModelsProviderFactory().createPolicyModelsProvider(parameters);
+    }
+
+    /**
+     * Set up GSON.
+     */
+    @Before
+    public void setupGson() {
+        standardCoder = new StandardCoder();
+    }
+
+    @After
+    public void teardown() throws Exception {
+        databaseProvider.close();
+    }
+
+    @Test
+    public void testPolicyPersistence() {
+        try {
+            for (String policyTypeResourceName : policyTypeResourceNames) {
+                String policyTypeString = ResourceUtils.getResourceAsString(policyTypeResourceName);
+
+                if (policyTypeResourceName.endsWith("yaml")) {
+                    testYamlStringPolicyTypePersistence(policyTypeString);
+                } else {
+                    testJsonStringPolicyTypePersistence(policyTypeString);
+                }
+            }
+        } catch (Exception exc) {
+            LOGGER.warn("error processing policies", exc);
+            fail("test should not throw an exception");
+        }
+    }
+
+    private void testYamlStringPolicyTypePersistence(final String policyTypeString) throws Exception {
+        Object yamlObject = new Yaml().load(policyTypeString);
+        String yamlAsJsonString = new GsonBuilder().setPrettyPrinting().create().toJson(yamlObject);
+
+        testJsonStringPolicyTypePersistence(yamlAsJsonString);
+    }
+
+    /**
+     * Check persistence of a policy.
+     *
+     * @param policyTypeString the policy as a string
+     * @throws Exception any exception thrown
+     */
+    public void testJsonStringPolicyTypePersistence(@NonNull final String policyTypeString) throws Exception {
+        ToscaServiceTemplate serviceTemplate = standardCoder.decode(policyTypeString, ToscaServiceTemplate.class);
+
+        assertNotNull(serviceTemplate);
+        ToscaPolicyType inPolicyType = serviceTemplate.getPolicyTypes().get(0).values().iterator().next();
+
+        databaseProvider.createPolicyTypes(serviceTemplate);
+
+        List<ToscaPolicyType> policyTypeList =
+                databaseProvider.getPolicyTypeList(inPolicyType.getName(), inPolicyType.getVersion());
+
+        assertEquals(1, policyTypeList.size());
+        assertEquals(inPolicyType.getName(), policyTypeList.get(0).getName());
+    }
+}
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java
index 9d327a2ca..e89b31635 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java
@@ -30,6 +30,7 @@ import java.util.Map.Entry;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.NonNull;
+
 import org.onap.policy.models.base.PfNameVersion;
 
 /**
@@ -70,5 +71,4 @@ public class ToscaEntity implements PfNameVersion {
             }
         }
     }
-
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java
index 9c6a375de..284e39c9b 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java
@@ -41,7 +41,7 @@ import lombok.ToString;
 @EqualsAndHashCode(callSuper = true)
 @NoArgsConstructor
 @ToString(callSuper = true)
-public class ToscaPolicy extends ToscaEntity {
+public class ToscaPolicy extends ToscaEntity implements Comparable<ToscaPolicy> {
     private String type;
 
     private String typeVersion;
@@ -84,4 +84,9 @@ public class ToscaPolicy extends ToscaEntity {
     public ToscaPolicyTypeIdentifier getTypeIdentifier() {
         return new ToscaPolicyTypeIdentifier(getType(), getTypeVersion());
     }
+
+    @Override
+    public int compareTo(final ToscaPolicy other) {
+        return compareNameVersion(this, other);
+    }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java
index 7781af236..d29f303f2 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java
@@ -27,7 +27,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NonNull;
 
-import org.onap.policy.models.base.PfObjectFiler;
+import org.onap.policy.models.base.PfObjectFilter;
 
 /**
  * Filter class for searches for {@link ToscaPolicy} instances.
@@ -37,7 +37,7 @@ import org.onap.policy.models.base.PfObjectFiler;
  */
 @Builder
 @Data
-public class ToscaPolicyFilter implements PfObjectFiler<ToscaPolicy> {
+public class ToscaPolicyFilter implements PfObjectFilter<ToscaPolicy> {
     public static final String LATEST_VERSION = "LATEST";
 
     // Regular expression
@@ -47,21 +47,28 @@ public class ToscaPolicyFilter implements PfObjectFiler<ToscaPolicy> {
     private String version;
 
     // Regular expression
-    private String policyTypeName;
+    private String type;
 
     // Regular Expression, set to LATEST_VERRSION to get the latest version
-    private String policyTypeVersion;
+    private String typeVersion;
 
     @Override
     public List<ToscaPolicy> filter(@NonNull final List<ToscaPolicy> originalList) {
 
         // @formatter:off
-        return originalList.stream()
-                .filter(p -> name              != null && p.getName()       .matches(name))
-                .filter(p -> version           != null && p.getVersion()    .matches(version))
-                .filter(p -> policyTypeName    != null && p.getType()       .matches(policyTypeName))
-                .filter(p -> policyTypeVersion != null && p.getTypeVersion().matches(policyTypeVersion))
+        List<ToscaPolicy> returnList = originalList.stream()
+                .filter(p -> filterOnRegexp(p.getName(),        name))
+                .filter(p -> filterOnRegexp(p.getVersion(),     version))
+                .filter(p -> filterOnRegexp(p.getType(),        type))
+                .filter(p -> filterOnRegexp(p.getTypeVersion(), typeVersion))
                 .collect(Collectors.toList());
         // @formatter:off
+
+        if (LATEST_VERSION.equals(version)) {
+            return this.latestVersionFilter(returnList);
+        }
+        else {
+            return returnList;
+        }
     }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java
index d64a5facd..75f17ea5b 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java
@@ -36,6 +36,11 @@ import lombok.NoArgsConstructor;
 @Data
 @EqualsAndHashCode(callSuper = true)
 @NoArgsConstructor
-public class ToscaPolicyType extends ToscaEntity {
+public class ToscaPolicyType extends ToscaEntity implements Comparable<ToscaPolicyType> {
     private Map<String, ToscaProperty> properties;
+
+    @Override
+    public int compareTo(final ToscaPolicyType other) {
+        return compareNameVersion(this, other);
+    }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java
index baa95045c..097fb6139 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java
@@ -27,7 +27,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NonNull;
 
-import org.onap.policy.models.base.PfObjectFiler;
+import org.onap.policy.models.base.PfObjectFilter;
 
 /**
  * Filter class for searches for {@link ToscaPolicyType} instances.
@@ -37,7 +37,7 @@ import org.onap.policy.models.base.PfObjectFiler;
  */
 @Builder
 @Data
-public class ToscaPolicyTypeFilter implements PfObjectFiler<ToscaPolicyType> {
+public class ToscaPolicyTypeFilter implements PfObjectFilter<ToscaPolicyType> {
     public static final String LATEST_VERSION = "LATEST";
 
     // Regular expression
@@ -50,10 +50,17 @@ public class ToscaPolicyTypeFilter implements PfObjectFiler<ToscaPolicyType> {
     public List<ToscaPolicyType> filter(@NonNull final List<ToscaPolicyType> originalList) {
 
         // @formatter:off
-        return originalList.stream()
-                .filter(p -> name    != null && p.getName()   .matches(name))
-                .filter(p -> version != null && p.getVersion().matches(version))
+        List<ToscaPolicyType> returnList = originalList.stream()
+                .filter(p -> filterOnRegexp(p.getName(),    name))
+                .filter(p -> filterOnRegexp(p.getVersion(), version))
                 .collect(Collectors.toList());
         // @formatter:off
+
+        if (LATEST_VERSION.equals(version)) {
+            return this.latestVersionFilter(returnList);
+        }
+        else {
+            return returnList;
+        }
     }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
index 274130a71..a843711ee 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
@@ -157,8 +157,8 @@ public class AuthorativeToscaProvider {
      * @return the policies found
      * @throws PfModelException on errors getting policies
      */
-    public ToscaServiceTemplate getPolicies(@NonNull final PfDao dao, @NonNull final String name,
-            @NonNull final String version) throws PfModelException {
+    public ToscaServiceTemplate getPolicies(@NonNull final PfDao dao, final String name, final String version)
+            throws PfModelException {
 
         return new SimpleToscaProvider().getPolicies(dao, name, version).toAuthorative();
     }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
index f2ab2460f..a39515b49 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
@@ -143,7 +143,7 @@ public class JpaToscaEntityType<T extends ToscaEntity> extends PfConcept impleme
 
     @Override
     public void fromAuthorative(T toscaEntity) {
-        key =  new PfConceptKey();
+        key = new PfConceptKey();
 
         if (toscaEntity.getName() != null) {
             key.setName(toscaEntity.getName());
@@ -158,8 +158,7 @@ public class JpaToscaEntityType<T extends ToscaEntity> extends PfConcept impleme
             // CHeck if the derived from field contains a name-version ID
             if (toscaEntity.getDerivedFrom().contains(":")) {
                 derivedFrom = new PfConceptKey(toscaEntity.getDerivedFrom());
-            }
-            else {
+            } else {
                 derivedFrom = new PfConceptKey(toscaEntity.getDerivedFrom(), PfKey.NULL_KEY_VERSION);
             }
         }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
index 67a833c94..671b5ccac 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
@@ -36,6 +36,7 @@ import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
+import javax.persistence.Lob;
 import javax.persistence.Table;
 
 import lombok.Data;
@@ -79,7 +80,7 @@ public class JpaToscaPolicy extends JpaToscaEntityType<ToscaPolicy> implements P
     private PfConceptKey type;
 
     @ElementCollection
-    @Column(length = 10000)
+    @Lob
     private Map<String, String> properties;
 
     @ElementCollection
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
index fc982965c..a6bc835b4 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
@@ -33,6 +33,7 @@ import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
+import javax.persistence.Lob;
 import javax.persistence.Table;
 
 import lombok.Data;
@@ -68,6 +69,7 @@ public class JpaToscaPolicyType extends JpaToscaEntityType<ToscaPolicyType> impl
     private static final long serialVersionUID = -563659852901842616L;
 
     @ElementCollection
+    @Lob
     private Map<String, JpaToscaProperty> properties;
 
     @ElementCollection
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java
index 38d5c0938..e2b6e6f93 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java
@@ -83,7 +83,6 @@ public class JpaToscaProperty extends PfConcept implements PfAuthorative<ToscaPr
     private String defaultValue;
 
     @Column
-    @NonNull
     private Status status = Status.SUPPORTED;
 
     @ElementCollection
diff --git a/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java b/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java
index 83100cc3c..b8766601b 100644
--- a/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java
+++ b/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java
@@ -200,13 +200,6 @@ public class JpaToscaPropertyTest {
         assertTrue(tp.validate(new PfValidationResult()).isValid());
 
         try {
-            tp.setStatus(null);
-            fail("test should throw an exception");
-        } catch (Exception exc) {
-            assertEquals("status is marked @NonNull but is null", exc.getMessage());
-        }
-
-        try {
             tp.validate(null);
             fail("test should throw an exception");
         } catch (Exception exc) {