Complete filters for Database Fetches

This review completes the implementaiton of the filters
for fetching policy types, policies, and PDP groups
from the database.

It also fixes bugs in Policy type creation.

Yaml in some of the policy type examples modified so that it is syntatically
correct.

Proeprties now stored as a blob in DB as they can be big.

Issue-ID: POLICY-1095
Change-Id: I6aef88ee2905afa58d778d82832f2b55d794fe9c
Signed-off-by: liamfallon <liam.fallon@est.tech>

27 files changed, 440 insertions, 86 deletions

diff --git a/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java b/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java
index 47238fc40..7bdced77b 100644
--- a/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java
+++ b/models-base/src/main/java/org/onap/policy/models/base/PfNameVersion.java
@@ -20,6 +20,9 @@
 
 package org.onap.policy.models.base;
 
+import org.apache.commons.lang3.ObjectUtils;
+import org.onap.policy.common.utils.validation.Version;
+
 /**
  * An interface that forces a POJO to have getName() and getVersion() methods.
  *
@@ -33,4 +36,43 @@ public interface PfNameVersion {
     public String getVersion();
 
     public void setVersion(final String version);
+
+    /**
+     * COmpare two name version implementation objects.
+     *
+     * @param left the left name/version implementation
+     * @param right the right name/version implementation
+     * @return the comparison resilt
+     */
+    public default int compareNameVersion(final PfNameVersion left, final PfNameVersion right) {
+        if (left == null && right == null) {
+            return 0;
+        }
+
+        if (left == null) {
+            return 1;
+        }
+
+        if (right == null) {
+            return -1;
+        }
+
+        int result = ObjectUtils.compare(left.getName(), right.getName());
+        if (result != 0) {
+            return result;
+        }
+
+        if (left.getVersion() == null && right.getVersion() == null) {
+            return 0;
+        }
+
+        if (left.getVersion() == null) {
+            return 1;
+        }
+
+        if (right.getVersion() == null) {
+            return -1;
+        }
+        return new Version(left.getVersion()).compareTo(new Version(right.getVersion()));
+    }
 }
diff --git a/models-base/src/main/java/org/onap/policy/models/base/PfObjectFiler.java b/models-base/src/main/java/org/onap/policy/models/base/PfObjectFiler.java
deleted file mode 100644
index f1481bf3c..000000000
--- a/models-base/src/main/java/org/onap/policy/models/base/PfObjectFiler.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*-
- * ============LICENSE_START=======================================================
- *  Copyright (C) 2019 Nordix Foundation.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-package org.onap.policy.models.base;
-
-import java.util.List;
-
-/**
- * Interface for filtering a list of concepts.
- *
- * @author Liam Fallon (liam.fallon@est.tech)
- */
-@FunctionalInterface
-public interface PfObjectFiler<T> {
-    /**
-     * Filter an incoming list, removing items that do not match the filter.
-     *
-     * @param originalList the original list
-     * @return the filtered list
-     */
-    public List<T> filter(final List<T> originalList);
-
-}
diff --git a/models-base/src/main/java/org/onap/policy/models/base/PfObjectFilter.java b/models-base/src/main/java/org/onap/policy/models/base/PfObjectFilter.java
new file mode 100644
index 000000000..a377c24be
--- /dev/null
+++ b/models-base/src/main/java/org/onap/policy/models/base/PfObjectFilter.java
@@ -0,0 +1,84 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2019 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.models.base;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import lombok.NonNull;
+
+/**
+ * Interface for filtering a list of concepts.
+ *
+ * @author Liam Fallon (liam.fallon@est.tech)
+ */
+@FunctionalInterface
+public interface PfObjectFilter<T extends Comparable<T>> {
+    /**
+     * Filter an incoming list, removing items that do not match the filter.
+     *
+     * @param originalList the original list
+     * @return the filtered list
+     */
+    public List<T> filter(final List<T> originalList);
+
+    /**
+     * Check if a value matches a regular expression.
+     *
+     * @param value the incoming value to check
+     * @param regexp the regular expression to check against
+     * @return match or not
+     */
+    public default boolean filterOnRegexp(@NonNull final String value, @NonNull final String regexp) {
+        return value.matches(regexp);
+    }
+
+    /**
+     * Sort an incoming list and remove all but the latest version of each concept.
+     *
+     * @param originalList the incoming list
+     * @return the filtered list
+     */
+    public default List<T> latestVersionFilter(final List<T> originalList) {
+        List<T> filteredList = new ArrayList<>();
+        Collections.sort(filteredList);
+
+        List<T> filteredOutList = new ArrayList<>();
+
+        for (int i = 1; i < filteredList.size(); i++) {
+            // Get the current and last element
+            T thisElement = filteredList.get(i);
+            T lastElement = filteredList.get(i - 1);
+
+            // The list is sorted so if the last element name is the same as the current element name, the last element
+            // should be removed
+            if (((PfNameVersion)thisElement).getName().equals(((PfNameVersion)lastElement).getName())) {
+                filteredOutList.add(lastElement);
+            }
+        }
+
+        // We can now remove these elements
+        filteredList.removeAll(filteredOutList);
+
+        return filteredList;
+    }
+}
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml
index c2fd504e9..1dcc5af86 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.AffinityPolicy.yaml
@@ -45,6 +45,7 @@ policy_types:
                 entry_schema:
                     type: string
 data_types:
+  -
     policy.data.affinityProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml
index 93ddd631a..458d152e0 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.DistancePolicy.yaml
@@ -45,6 +45,7 @@ policy_types:
                 type: policy.data.distanceProperties_properties
                 required: true
 data_types:
+  -
     policy.data.distanceProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml
index 63f0d8ada..be4110517 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.HpaPolicy.yaml
@@ -36,8 +36,9 @@ policy_types:
                 type: list
                 required: true
                 entry_schema:
-                    type:policy.data.flavorFeatures_properties
+                    type: policy.data.flavorFeatures_properties
 data_types:
+  -
     policy.data.flavorFeatures_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -57,6 +58,7 @@ data_types:
                 required: true
                 entry_schema:
                     type: policy.data.flavorProperties_properties
+  -
     policy.data.directives_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -66,6 +68,7 @@ data_types:
                 type: list
                 entry_schema:
                     type: policy.data.directives_attributes_properties
+  -
     policy.data.directives_attributes_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -73,6 +76,7 @@ data_types:
                 type: string
             attribute_value:
                 type: string
+  -
     policy.data.flavorProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
@@ -101,6 +105,7 @@ data_types:
                 required: true
                 entry_schema:
                     type: policy.data.hpa-feature-attributes_properties
+  -
     policy.data.hpa-feature-attributes_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml
index edfac1496..267109861 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.OptimizationPolicy.yaml
@@ -40,6 +40,7 @@ policy_types:
                 type: policy.data.objectiveParameter_properties
                 required: true
 data_types:
+  -
     policy.data.objectiveParameter_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml
index 1355eb031..ba4dbec37 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.PciPolicy.yaml
@@ -36,8 +36,9 @@ policy_types:
                 type: list
                 required: false
                 entry_schema:
-                -   type:policy.data.pciProperties_properties
+                  type: policy.data.pciProperties_properties
 data_types:
+  -
     policy.data.pciProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml
index f7036dc80..82cd60a46 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.QueryPolicy.yaml
@@ -31,8 +31,9 @@ policy_types:
                 type: list
                 required: true
                 entry_schema:
-                    type:policy.data.queryProperties_properties
+                    type: policy.data.queryProperties_properties
 data_types:
+  -
     policy.data.queryProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml
index 3c2c2b2c3..ab006fa57 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.SubscriberPolicy.yaml
@@ -31,6 +31,7 @@ policy_types:
                 type: policy.data.properties_properties
                 required: true
 data_types:
+  -
     policy.data.properties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml
index 860c37fa0..9c12f7d63 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.Vim_fit.yaml
@@ -45,6 +45,7 @@ policy_types:
                 type: policy.data.capacityProperties_properties
                 required: true
 data_types:
+  -
     policy.data.capacityProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml b/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml
index 13d4f137c..4bfc24216 100644
--- a/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml
+++ b/models-examples/src/main/resources/policytypes/onap.policies.optimization.VnfPolicy.yaml
@@ -45,8 +45,9 @@ policy_types:
                 type: list
                 required: true
                 entry_schema:
-                    type:policy.data.vnfProperties_properties
+                    type: policy.data.vnfProperties_properties
 data_types:
+  -
     policy.data.vnfProperties_properties:
         derived_from: tosca.nodes.Root
         properties:
diff --git a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java
index a59d1af5a..9665fd472 100644
--- a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java
+++ b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroup.java
@@ -40,7 +40,7 @@ import org.onap.policy.models.pdp.enums.PdpState;
  */
 @Data
 @NoArgsConstructor
-public class PdpGroup implements PfNameVersion {
+public class PdpGroup implements PfNameVersion, Comparable<PdpGroup> {
     private String name;
     private String version;
     private String description;
@@ -67,4 +67,9 @@ public class PdpGroup implements PfNameVersion {
         this.properties = (source.properties == null ? null : new LinkedHashMap<>(source.properties));
         this.pdpSubgroups = PfUtils.mapList(source.pdpSubgroups, PdpSubGroup::new);
     }
+
+    @Override
+    public int compareTo(final PdpGroup other) {
+        return compareNameVersion(this, other);
+    }
 }
diff --git a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java
index b49bedefe..c5c0bc541 100644
--- a/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java
+++ b/models-pdp/src/main/java/org/onap/policy/models/pdp/concepts/PdpGroupFilter.java
@@ -27,20 +27,20 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NonNull;
 
-import org.onap.policy.models.base.PfObjectFiler;
+import org.apache.commons.lang3.ObjectUtils;
+import org.onap.policy.models.base.PfObjectFilter;
 import org.onap.policy.models.pdp.enums.PdpState;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyIdentifier;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeIdentifier;
 
 /**
- * Filter class for searches for {@link PdpGroup} instances.
- * If any fields are null, they are ignored.
+ * Filter class for searches for {@link PdpGroup} instances. If any fields are null, they are ignored.
  *
  * @author Liam Fallon (liam.fallon@est.tech)
  */
 @Builder
 @Data
-public class PdpGroupFilter implements PfObjectFiler<PdpGroup> {
+public class PdpGroupFilter implements PfObjectFilter<PdpGroup> {
     public static final String LATEST_VERSION = "LATEST";
 
     // Regular expression
@@ -60,15 +60,95 @@ public class PdpGroupFilter implements PfObjectFiler<PdpGroup> {
     // Set regular expressions on fields to match policy names and versions
     private ToscaPolicyIdentifier policy;
 
+    private PdpState pdpState;
+
     @Override
     public List<PdpGroup> filter(@NonNull final List<PdpGroup> originalList) {
 
         // @formatter:off
-        return originalList.stream()
-                .filter(p -> name       != null && p.getName()   .matches(name))
-                .filter(p -> version    != null && p.getVersion().matches(version))
-                .filter(p -> groupState != null && p.getPdpGroupState().equals(groupState))
+        List<PdpGroup> returnList = originalList.stream()
+                .filter(p -> filterOnRegexp(p.getName(),    name))
+                .filter(p -> filterOnRegexp(p.getVersion(), version))
+                .filter(p -> ObjectUtils.compare(p.getPdpGroupState(), groupState) == 0)
+                .filter(p -> filterOnPdpType(p, pdpType))
+                .filter(p -> filterOnPolicyType(p, policyType))
+                .filter(p -> filterOnPolicy(p, policy))
                 .collect(Collectors.toList());
         // @formatter:off
+
+        if (LATEST_VERSION.equals(version)) {
+            returnList = this.latestVersionFilter(returnList);
+        }
+
+        return returnList;
+    }
+
+    /**
+     * Filter PDP groups on PDP type.
+     *
+     * @param pdpGroup the PDP group to check
+     * @param pdpType the PDP type to check for
+     * @return true if the filter should let this PDP group through
+     */
+    private boolean filterOnPdpType(final PdpGroup pdpGroup, final String pdpType) {
+        if (pdpType == null) {
+            return true;
+        }
+
+        for (PdpSubGroup pdpSubGroup: pdpGroup.getPdpSubgroups()) {
+            if (pdpSubGroup.getPdpType().equals(pdpType)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Filter PDP groups on policy type.
+     *
+     * @param pdpGroup the PDP group to check
+     * @param policyTypeFilter the policy type regular expressions to check for
+     * @return true if the filter should let this PDP group through
+     */
+    private boolean filterOnPolicyType(final PdpGroup pdpGroup, final ToscaPolicyTypeIdentifier policyTypeFiler) {
+        if (policyTypeFiler == null) {
+            return true;
+        }
+
+        for (PdpSubGroup pdpSubGroup: pdpGroup.getPdpSubgroups()) {
+            for (ToscaPolicyTypeIdentifier foundPolicyType : pdpSubGroup.getSupportedPolicyTypes()) {
+                if (foundPolicyType.getName().matches(policyTypeFiler.getName())
+                        && foundPolicyType.getVersion().matches(policyTypeFiler.getVersion())) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Filter PDP groups on policy.
+     *
+     * @param pdpGroup the PDP group to check
+     * @param policyFilter the policy regular expressions to check for
+     * @return true if the filter should let this PDP group through
+     */
+    private boolean filterOnPolicy(final PdpGroup pdpGroup, final ToscaPolicyIdentifier policyFiler) {
+        if (policyFiler == null) {
+            return true;
+        }
+
+        for (PdpSubGroup pdpSubGroup: pdpGroup.getPdpSubgroups()) {
+            for (ToscaPolicyIdentifier foundPolicy : pdpSubGroup.getPolicies()) {
+                if (foundPolicy.getName().matches(policyFiler.getName())
+                        && foundPolicy.getVersion().matches(policyFiler.getVersion())) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
     }
 }
diff --git a/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java b/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java
index cac506acc..4de1f2ee3 100644
--- a/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java
+++ b/models-pdp/src/test/java/org/onap/policy/models/pdp/concepts/TestPdpGroup.java
@@ -49,7 +49,7 @@ public class TestPdpGroup {
         // verify with all values
         orig.setDescription("my-descript");
         orig.setName("my-name");
-        orig.setVersion("my-version");
+        orig.setVersion("1.2.3");
         orig.setDescription("my-description");
         orig.setPdpGroupState(PdpState.SAFE);
 
@@ -64,7 +64,7 @@ public class TestPdpGroup {
         props.put("key-B", "value-B");
         orig.setProperties(props);
 
-        assertEquals("PdpGroup(name=my-name, version=my-version, description=my-description, "
+        assertEquals("PdpGroup(name=my-name, version=1.2.3, description=my-description, "
                 + "pdpGroupState=SAFE, properties={key-A=value-A, key-B=value-B}, "
                 + "pdpSubgroups=[PdpSubGroup(pdpType=null, supportedPolicyTypes=[], policies=[], "
                 + "currentInstanceCount=10, desiredInstanceCount=0, properties=null, pdpInstances=[]), "
diff --git a/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyTypePersistenceTest.java b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyTypePersistenceTest.java
new file mode 100644
index 000000000..7f90a0b58
--- /dev/null
+++ b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyTypePersistenceTest.java
@@ -0,0 +1,152 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2019 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.models.provider.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import com.google.gson.GsonBuilder;
+
+import java.util.Base64;
+import java.util.List;
+
+import lombok.NonNull;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.onap.policy.common.utils.coder.StandardCoder;
+import org.onap.policy.common.utils.resources.ResourceUtils;
+import org.onap.policy.models.base.PfModelException;
+import org.onap.policy.models.provider.PolicyModelsProvider;
+import org.onap.policy.models.provider.PolicyModelsProviderFactory;
+import org.onap.policy.models.provider.PolicyModelsProviderParameters;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyType;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * Test persistence of monitoring policies to and from the database.
+ *
+ * @author Liam Fallon (liam.fallon@est.tech)
+ */
+public class PolicyTypePersistenceTest {
+    // Logger for this class
+    private static final Logger LOGGER = LoggerFactory.getLogger(PolicyTypePersistenceTest.class);
+
+    private StandardCoder standardCoder;
+
+    private PolicyModelsProvider databaseProvider;
+
+    // @formatter:off
+    private String[] policyTypeResourceNames = {
+        "policytypes/onap.policies.monitoring.dcaegen2.collectors.datafile.datafile-app-server.yaml",
+        "policytypes/onap.policies.optimization.AffinityPolicy.yaml",
+        "policytypes/onap.policies.optimization.DistancePolicy.yaml",
+        "policytypes/onap.policies.optimization.HpaPolicy.yaml",
+        "policytypes/onap.policies.optimization.OptimizationPolicy.yaml",
+        "policytypes/onap.policies.optimization.PciPolicy.yaml",
+        "policytypes/onap.policies.optimization.QueryPolicy.yaml",
+        "policytypes/onap.policies.optimization.SubscriberPolicy.yaml",
+        "policytypes/onap.policies.optimization.Vim_fit.yaml",
+        "policytypes/onap.policies.optimization.VnfPolicy.yaml",
+        "policytypes/onap.policy.monitoring.cdap.tca.hi.lo.app.yaml"
+    };
+    // @formatter:on
+
+    /**
+     * Initialize provider.
+     *
+     * @throws PfModelException on exceptions in the tests
+     */
+    @Before
+    public void setupParameters() throws PfModelException {
+        PolicyModelsProviderParameters parameters = new PolicyModelsProviderParameters();
+        parameters.setDatabaseUrl("jdbc:h2:mem:testdb");
+        parameters.setDatabaseUser("policy");
+        parameters.setDatabasePassword(Base64.getEncoder().encodeToString("P01icY".getBytes()));
+        parameters.setPersistenceUnit("ToscaConceptTest");
+
+        databaseProvider = new PolicyModelsProviderFactory().createPolicyModelsProvider(parameters);
+    }
+
+    /**
+     * Set up GSON.
+     */
+    @Before
+    public void setupGson() {
+        standardCoder = new StandardCoder();
+    }
+
+    @After
+    public void teardown() throws Exception {
+        databaseProvider.close();
+    }
+
+    @Test
+    public void testPolicyPersistence() {
+        try {
+            for (String policyTypeResourceName : policyTypeResourceNames) {
+                String policyTypeString = ResourceUtils.getResourceAsString(policyTypeResourceName);
+
+                if (policyTypeResourceName.endsWith("yaml")) {
+                    testYamlStringPolicyTypePersistence(policyTypeString);
+                } else {
+                    testJsonStringPolicyTypePersistence(policyTypeString);
+                }
+            }
+        } catch (Exception exc) {
+            LOGGER.warn("error processing policies", exc);
+            fail("test should not throw an exception");
+        }
+    }
+
+    private void testYamlStringPolicyTypePersistence(final String policyTypeString) throws Exception {
+        Object yamlObject = new Yaml().load(policyTypeString);
+        String yamlAsJsonString = new GsonBuilder().setPrettyPrinting().create().toJson(yamlObject);
+
+        testJsonStringPolicyTypePersistence(yamlAsJsonString);
+    }
+
+    /**
+     * Check persistence of a policy.
+     *
+     * @param policyTypeString the policy as a string
+     * @throws Exception any exception thrown
+     */
+    public void testJsonStringPolicyTypePersistence(@NonNull final String policyTypeString) throws Exception {
+        ToscaServiceTemplate serviceTemplate = standardCoder.decode(policyTypeString, ToscaServiceTemplate.class);
+
+        assertNotNull(serviceTemplate);
+        ToscaPolicyType inPolicyType = serviceTemplate.getPolicyTypes().get(0).values().iterator().next();
+
+        databaseProvider.createPolicyTypes(serviceTemplate);
+
+        List<ToscaPolicyType> policyTypeList =
+                databaseProvider.getPolicyTypeList(inPolicyType.getName(), inPolicyType.getVersion());
+
+        assertEquals(1, policyTypeList.size());
+        assertEquals(inPolicyType.getName(), policyTypeList.get(0).getName());
+    }
+}
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java
index 9d327a2ca..e89b31635 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaEntity.java
@@ -30,6 +30,7 @@ import java.util.Map.Entry;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.NonNull;
+
 import org.onap.policy.models.base.PfNameVersion;
 
 /**
@@ -70,5 +71,4 @@ public class ToscaEntity implements PfNameVersion {
             }
         }
     }
-
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java
index 9c6a375de..284e39c9b 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicy.java
@@ -41,7 +41,7 @@ import lombok.ToString;
 @EqualsAndHashCode(callSuper = true)
 @NoArgsConstructor
 @ToString(callSuper = true)
-public class ToscaPolicy extends ToscaEntity {
+public class ToscaPolicy extends ToscaEntity implements Comparable<ToscaPolicy> {
     private String type;
 
     private String typeVersion;
@@ -84,4 +84,9 @@ public class ToscaPolicy extends ToscaEntity {
     public ToscaPolicyTypeIdentifier getTypeIdentifier() {
         return new ToscaPolicyTypeIdentifier(getType(), getTypeVersion());
     }
+
+    @Override
+    public int compareTo(final ToscaPolicy other) {
+        return compareNameVersion(this, other);
+    }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java
index 7781af236..d29f303f2 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyFilter.java
@@ -27,7 +27,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NonNull;
 
-import org.onap.policy.models.base.PfObjectFiler;
+import org.onap.policy.models.base.PfObjectFilter;
 
 /**
  * Filter class for searches for {@link ToscaPolicy} instances.
@@ -37,7 +37,7 @@ import org.onap.policy.models.base.PfObjectFiler;
  */
 @Builder
 @Data
-public class ToscaPolicyFilter implements PfObjectFiler<ToscaPolicy> {
+public class ToscaPolicyFilter implements PfObjectFilter<ToscaPolicy> {
     public static final String LATEST_VERSION = "LATEST";
 
     // Regular expression
@@ -47,21 +47,28 @@ public class ToscaPolicyFilter implements PfObjectFiler<ToscaPolicy> {
     private String version;
 
     // Regular expression
-    private String policyTypeName;
+    private String type;
 
     // Regular Expression, set to LATEST_VERRSION to get the latest version
-    private String policyTypeVersion;
+    private String typeVersion;
 
     @Override
     public List<ToscaPolicy> filter(@NonNull final List<ToscaPolicy> originalList) {
 
         // @formatter:off
-        return originalList.stream()
-                .filter(p -> name              != null && p.getName()       .matches(name))
-                .filter(p -> version           != null && p.getVersion()    .matches(version))
-                .filter(p -> policyTypeName    != null && p.getType()       .matches(policyTypeName))
-                .filter(p -> policyTypeVersion != null && p.getTypeVersion().matches(policyTypeVersion))
+        List<ToscaPolicy> returnList = originalList.stream()
+                .filter(p -> filterOnRegexp(p.getName(),        name))
+                .filter(p -> filterOnRegexp(p.getVersion(),     version))
+                .filter(p -> filterOnRegexp(p.getType(),        type))
+                .filter(p -> filterOnRegexp(p.getTypeVersion(), typeVersion))
                 .collect(Collectors.toList());
         // @formatter:off
+
+        if (LATEST_VERSION.equals(version)) {
+            return this.latestVersionFilter(returnList);
+        }
+        else {
+            return returnList;
+        }
     }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java
index d64a5facd..75f17ea5b 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyType.java
@@ -36,6 +36,11 @@ import lombok.NoArgsConstructor;
 @Data
 @EqualsAndHashCode(callSuper = true)
 @NoArgsConstructor
-public class ToscaPolicyType extends ToscaEntity {
+public class ToscaPolicyType extends ToscaEntity implements Comparable<ToscaPolicyType> {
     private Map<String, ToscaProperty> properties;
+
+    @Override
+    public int compareTo(final ToscaPolicyType other) {
+        return compareNameVersion(this, other);
+    }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java
index baa95045c..097fb6139 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/concepts/ToscaPolicyTypeFilter.java
@@ -27,7 +27,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NonNull;
 
-import org.onap.policy.models.base.PfObjectFiler;
+import org.onap.policy.models.base.PfObjectFilter;
 
 /**
  * Filter class for searches for {@link ToscaPolicyType} instances.
@@ -37,7 +37,7 @@ import org.onap.policy.models.base.PfObjectFiler;
  */
 @Builder
 @Data
-public class ToscaPolicyTypeFilter implements PfObjectFiler<ToscaPolicyType> {
+public class ToscaPolicyTypeFilter implements PfObjectFilter<ToscaPolicyType> {
     public static final String LATEST_VERSION = "LATEST";
 
     // Regular expression
@@ -50,10 +50,17 @@ public class ToscaPolicyTypeFilter implements PfObjectFiler<ToscaPolicyType> {
     public List<ToscaPolicyType> filter(@NonNull final List<ToscaPolicyType> originalList) {
 
         // @formatter:off
-        return originalList.stream()
-                .filter(p -> name    != null && p.getName()   .matches(name))
-                .filter(p -> version != null && p.getVersion().matches(version))
+        List<ToscaPolicyType> returnList = originalList.stream()
+                .filter(p -> filterOnRegexp(p.getName(),    name))
+                .filter(p -> filterOnRegexp(p.getVersion(), version))
                 .collect(Collectors.toList());
         // @formatter:off
+
+        if (LATEST_VERSION.equals(version)) {
+            return this.latestVersionFilter(returnList);
+        }
+        else {
+            return returnList;
+        }
     }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
index 274130a71..a843711ee 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
@@ -157,8 +157,8 @@ public class AuthorativeToscaProvider {
      * @return the policies found
      * @throws PfModelException on errors getting policies
      */
-    public ToscaServiceTemplate getPolicies(@NonNull final PfDao dao, @NonNull final String name,
-            @NonNull final String version) throws PfModelException {
+    public ToscaServiceTemplate getPolicies(@NonNull final PfDao dao, final String name, final String version)
+            throws PfModelException {
 
         return new SimpleToscaProvider().getPolicies(dao, name, version).toAuthorative();
     }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
index f2ab2460f..a39515b49 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
@@ -143,7 +143,7 @@ public class JpaToscaEntityType<T extends ToscaEntity> extends PfConcept impleme
 
     @Override
     public void fromAuthorative(T toscaEntity) {
-        key =  new PfConceptKey();
+        key = new PfConceptKey();
 
         if (toscaEntity.getName() != null) {
             key.setName(toscaEntity.getName());
@@ -158,8 +158,7 @@ public class JpaToscaEntityType<T extends ToscaEntity> extends PfConcept impleme
             // CHeck if the derived from field contains a name-version ID
             if (toscaEntity.getDerivedFrom().contains(":")) {
                 derivedFrom = new PfConceptKey(toscaEntity.getDerivedFrom());
-            }
-            else {
+            } else {
                 derivedFrom = new PfConceptKey(toscaEntity.getDerivedFrom(), PfKey.NULL_KEY_VERSION);
             }
         }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
index 67a833c94..671b5ccac 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
@@ -36,6 +36,7 @@ import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
+import javax.persistence.Lob;
 import javax.persistence.Table;
 
 import lombok.Data;
@@ -79,7 +80,7 @@ public class JpaToscaPolicy extends JpaToscaEntityType<ToscaPolicy> implements P
     private PfConceptKey type;
 
     @ElementCollection
-    @Column(length = 10000)
+    @Lob
     private Map<String, String> properties;
 
     @ElementCollection
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
index fc982965c..a6bc835b4 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
@@ -33,6 +33,7 @@ import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
+import javax.persistence.Lob;
 import javax.persistence.Table;
 
 import lombok.Data;
@@ -68,6 +69,7 @@ public class JpaToscaPolicyType extends JpaToscaEntityType<ToscaPolicyType> impl
     private static final long serialVersionUID = -563659852901842616L;
 
     @ElementCollection
+    @Lob
     private Map<String, JpaToscaProperty> properties;
 
     @ElementCollection
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java
index 38d5c0938..e2b6e6f93 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaProperty.java
@@ -83,7 +83,6 @@ public class JpaToscaProperty extends PfConcept implements PfAuthorative<ToscaPr
     private String defaultValue;
 
     @Column
-    @NonNull
     private Status status = Status.SUPPORTED;
 
     @ElementCollection
diff --git a/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java b/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java
index 83100cc3c..b8766601b 100644
--- a/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java
+++ b/models-tosca/src/test/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPropertyTest.java
@@ -200,13 +200,6 @@ public class JpaToscaPropertyTest {
         assertTrue(tp.validate(new PfValidationResult()).isValid());
 
         try {
-            tp.setStatus(null);
-            fail("test should throw an exception");
-        } catch (Exception exc) {
-            assertEquals("status is marked @NonNull but is null", exc.getMessage());
-        }
-
-        try {
             tp.validate(null);
             fail("test should throw an exception");
         } catch (Exception exc) {