Implement validation and hierarchical get

NOTE: This review will require changes to the persistence.xml file in
other policy framework components.

This change brings in two changes:

1) Validation of the references betwen policies, policy types, and data
types

This change mans that additions to and changes to policies, policy
types, and data types must be structurally correct in the change coming
in and with whatever is already in the database

2) Hierarchical return of referenced entitites on policy types,
policies and data types.

When a policy, policy type, or data type is read from the database, the
entities referenced byt the returned policy, policy type, or data type
are returned as well.

Tests are amended for the stricter validation and structure.

Examples are corrected where they are incorrect.

Issue-ID: POLICY-1402
Change-Id: Ie6a4cb7ed336562338924079114df405f0ab889f
Signed-off-by: liamfallon <liam.fallon@est.tech>

10 files changed, 308 insertions, 151 deletions

diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
index 7999f620b..6e60303a0 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
@@ -22,15 +22,19 @@
 package org.onap.policy.models.tosca.authorative.provider;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
+import javax.ws.rs.core.Response.Status;
+
 import lombok.NonNull;
 
 import org.onap.policy.models.base.PfConceptKey;
 import org.onap.policy.models.base.PfModelException;
+import org.onap.policy.models.base.PfModelRuntimeException;
 import org.onap.policy.models.dao.PfDao;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaEntity;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy;
@@ -87,8 +91,14 @@ public class AuthorativeToscaProvider {
 
         LOGGER.debug("->getPolicyTypeList: name={}, version={}", name, version);
 
-        List<ToscaPolicyType> policyTypeList = new ArrayList<>(
-                new SimpleToscaProvider().getPolicyTypes(dao, name, version).toAuthorative().getPolicyTypes().values());
+        List<ToscaPolicyType> policyTypeList;
+
+        try {
+            policyTypeList = new ArrayList<>(new SimpleToscaProvider().getPolicyTypes(dao, name, version)
+                    .toAuthorative().getPolicyTypes().values());
+        } catch (PfModelRuntimeException pfme) {
+            return handlePfModelRuntimeException(pfme);
+        }
 
         LOGGER.debug("<-getPolicyTypeList: name={}, version={}, policyTypeList={}", name, version, policyTypeList);
         return policyTypeList;
@@ -136,6 +146,7 @@ public class AuthorativeToscaProvider {
 
         LOGGER.debug("<-getFilteredPolicyTypeList: filter={}, filteredPolicyTypeList={}", filter,
                 filteredPolicyTypeList);
+
         return filteredPolicyTypeList;
     }
 
@@ -234,8 +245,14 @@ public class AuthorativeToscaProvider {
             throws PfModelException {
         LOGGER.debug("->getPolicyList: name={}, version={}", name, version);
 
-        List<ToscaPolicy> policyList = asConceptList(new SimpleToscaProvider().getPolicies(dao, name, version)
-                .toAuthorative().getToscaTopologyTemplate().getPolicies());
+        List<ToscaPolicy> policyList;
+
+        try {
+            policyList = asConceptList(new SimpleToscaProvider().getPolicies(dao, name, version).toAuthorative()
+                    .getToscaTopologyTemplate().getPolicies());
+        } catch (PfModelRuntimeException pfme) {
+            return handlePfModelRuntimeException(pfme);
+        }
 
         LOGGER.debug("<-getPolicyList: name={}, version={}, policyTypeList={}", name, version, policyList);
         return policyList;
@@ -397,4 +414,18 @@ public class AuthorativeToscaProvider {
 
         return conceptMap;
     }
+
+    /**
+     * Handle a PfModelRuntimeException on a list call.
+     *
+     * @param pfme the model exception
+     * @return an empty list on 404
+     */
+    private <T extends ToscaEntity> List<T> handlePfModelRuntimeException(final PfModelRuntimeException pfme) {
+        if (Status.NOT_FOUND.equals(pfme.getErrorResponse().getResponseCode())) {
+            return Collections.emptyList();
+        } else {
+            throw pfme;
+        }
+    }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaDataType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaDataType.java
index 86d67e4d8..a44d7654c 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaDataType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaDataType.java
@@ -3,7 +3,7 @@
  * ONAP Policy Model
  * ================================================================================
  * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
- * Modifications Copyright (C) 2019 Nordix Foundation.
+ * Modifications Copyright (C) 2019-2020 Nordix Foundation.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,10 +24,13 @@
 package org.onap.policy.models.tosca.simple.concepts;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
@@ -40,6 +43,7 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NonNull;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.onap.policy.models.base.PfAuthorative;
 import org.onap.policy.models.base.PfConcept;
 import org.onap.policy.models.base.PfConceptKey;
@@ -52,6 +56,7 @@ import org.onap.policy.models.base.PfValidationResult.ValidationResult;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaConstraint;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaDataType;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaProperty;
+import org.onap.policy.models.tosca.utils.ToscaUtils;
 
 /**
  * Class to represent custom data type in TOSCA definition.
@@ -68,11 +73,11 @@ public class JpaToscaDataType extends JpaToscaEntityType<ToscaDataType> implemen
     private static final long serialVersionUID = -3922690413436539164L;
 
     @ElementCollection
-    private List<JpaToscaConstraint> constraints;
+    private List<JpaToscaConstraint> constraints = new ArrayList<>();
 
     @ElementCollection
     @Lob
-    private Map<String, JpaToscaProperty> properties;
+    private Map<String, JpaToscaProperty> properties = new LinkedHashMap<>();
 
     /**
      * The Default Constructor creates a {@link JpaToscaDataType} object with a null key.
@@ -268,4 +273,29 @@ public class JpaToscaDataType extends JpaToscaEntityType<ToscaDataType> implemen
 
         return 0;
     }
+
+    /**
+     * Get the data types referenced in a data type.
+     *
+     * @return the data types referenced in a data type
+     */
+    public Collection<PfConceptKey> getReferencedDataTypes() {
+        if (properties == null) {
+            return CollectionUtils.emptyCollection();
+        }
+
+        Set<PfConceptKey> referencedDataTypes = new LinkedHashSet<>();
+
+        for (JpaToscaProperty property : properties.values()) {
+            referencedDataTypes.add(property.getType());
+
+            if (property.getEntrySchema() != null) {
+                referencedDataTypes.add(property.getEntrySchema().getType());
+            }
+        }
+
+        referencedDataTypes.removeAll(ToscaUtils.getPredefinedDataTypes());
+
+        return referencedDataTypes;
+    }
 }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
index 4823efc01..6544e7221 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaEntityType.java
@@ -1,6 +1,6 @@
 /*-
  * ============LICENSE_START=======================================================
- *  Copyright (C) 2019 Nordix Foundation.
+ *  Copyright (C) 2019-2020 Nordix Foundation.
  *  Modifications Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,15 +26,18 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
+
 import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.ElementCollection;
 import javax.persistence.EmbeddedId;
 import javax.persistence.MappedSuperclass;
+
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NonNull;
+
 import org.apache.commons.lang3.ObjectUtils;
 import org.onap.policy.common.utils.validation.ParameterValidationUtils;
 import org.onap.policy.models.base.PfAuthorative;
@@ -70,7 +73,7 @@ public class JpaToscaEntityType<T extends ToscaEntity> extends PfConcept impleme
     private PfConceptKey derivedFrom;
 
     @ElementCollection
-    private Map<String, String> metadata;
+    private Map<String, String> metadata = new TreeMap<>();
 
     @Column
     private String description;
@@ -154,7 +157,6 @@ public class JpaToscaEntityType<T extends ToscaEntity> extends PfConcept impleme
             key.setVersion(toscaEntity.getVersion());
         }
 
-
         if (toscaEntity.getDerivedFrom() != null) {
             // CHeck if the derived from field contains a name-version ID
             if (toscaEntity.getDerivedFrom().contains(":")) {
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
index 36acec018..2816df004 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicy.java
@@ -3,7 +3,7 @@
  * ONAP Policy Model
  * ================================================================================
  * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
- * Modifications Copyright (C) 2019 Nordix Foundation.
+ * Modifications Copyright (C) 2019-2020 Nordix Foundation.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,10 +23,12 @@
 
 package org.onap.policy.models.tosca.simple.concepts;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+
 import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
@@ -37,9 +39,11 @@ import javax.persistence.InheritanceType;
 import javax.persistence.Lob;
 import javax.persistence.Table;
 import javax.ws.rs.core.Response;
+
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NonNull;
+
 import org.onap.policy.common.utils.coder.CoderException;
 import org.onap.policy.common.utils.coder.StandardCoder;
 import org.onap.policy.common.utils.validation.ParameterValidationUtils;
@@ -87,7 +91,7 @@ public class JpaToscaPolicy extends JpaToscaEntityType<ToscaPolicy> implements P
     private Map<String, String> properties;
 
     @ElementCollection
-    private List<PfConceptKey> targets;
+    private List<PfConceptKey> targets = new ArrayList<>();
     // @formatter:on
 
     /**
@@ -186,15 +190,13 @@ public class JpaToscaPolicy extends JpaToscaEntityType<ToscaPolicy> implements P
 
         if (toscaPolicy.getType() != null) {
             type.setName(toscaPolicy.getType());
-        }
-        else {
+        } else {
             type.setName(PfKey.NULL_KEY_NAME);
         }
 
         if (toscaPolicy.getTypeVersion() != null) {
             type.setVersion(toscaPolicy.getTypeVersion());
-        }
-        else {
+        } else {
             type.setVersion(PfKey.NULL_KEY_VERSION);
         }
 
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
index 43d7ad633..b068beaa0 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyType.java
@@ -23,6 +23,7 @@
 
 package org.onap.policy.models.tosca.simple.concepts;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -73,13 +74,13 @@ public class JpaToscaPolicyType extends JpaToscaEntityType<ToscaPolicyType> impl
 
     @ElementCollection
     @Lob
-    private Map<String, JpaToscaProperty> properties;
+    private Map<String, JpaToscaProperty> properties = new LinkedHashMap<>();
 
     @ElementCollection
-    private List<PfConceptKey> targets;
+    private List<PfConceptKey> targets = new ArrayList<>();
 
     @ElementCollection
-    private List<JpaToscaTrigger> triggers;
+    private List<JpaToscaTrigger> triggers = new ArrayList<>();
 
     /**
      * The Default Constructor creates a {@link JpaToscaPolicyType} object with a null key.
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyTypes.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyTypes.java
index 9c059b483..00a6d6b50 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyTypes.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaPolicyTypes.java
@@ -109,6 +109,7 @@ public class JpaToscaPolicyTypes extends PfConceptContainer<JpaToscaPolicyType,
     public PfValidationResult validate(@NonNull final PfValidationResult resultIn) {
         PfValidationResult result = super.validate(resultIn);
 
+        // Check that all ancestors of this policy type exist
         for (JpaToscaPolicyType policyType : this.getConceptMap().values()) {
             ToscaUtils.getEntityTypeAncestors(this, policyType, result);
         }
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaServiceTemplate.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaServiceTemplate.java
index 786784c09..aa4f231c0 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaServiceTemplate.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaServiceTemplate.java
@@ -23,6 +23,7 @@ package org.onap.policy.models.tosca.simple.concepts;
 
 import com.google.gson.annotations.SerializedName;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -101,7 +102,15 @@ public class JpaToscaServiceTemplate extends JpaToscaEntityType<ToscaServiceTemp
     @SerializedName("policy_types")
     private JpaToscaPolicyTypes policyTypes;
 
-    @Column
+    @OneToOne(fetch = FetchType.EAGER, cascade = CascadeType.ALL, orphanRemoval = true)
+    @JoinColumns(
+            {
+                @JoinColumn(name = "topologyTemplateParentKeyName",    referencedColumnName = "parentKeyName"),
+                @JoinColumn(name = "topologyTemplateParentKeyVersion", referencedColumnName = "parentKeyVersion"),
+                @JoinColumn(name = "topologyTemplateParentLocalName",  referencedColumnName = "parentLocalName"),
+                @JoinColumn(name = "topologyTemplateLocalName",        referencedColumnName = "localName")
+            }
+        )
     @SerializedName("topology_template")
     private JpaToscaTopologyTemplate topologyTemplate;
     // @formatter:on
@@ -281,7 +290,7 @@ public class JpaToscaServiceTemplate extends JpaToscaEntityType<ToscaServiceTemp
             return result;
         }
 
-        validateDatatypesInPolicyTypes(result);
+        validateReferencedDataTypes(result);
 
         return validatePolicyTypesInPolicies(result);
     }
@@ -339,36 +348,54 @@ public class JpaToscaServiceTemplate extends JpaToscaEntityType<ToscaServiceTemp
      * @param result the validation result object to use for the validation result
      * @return the validation result object
      */
-    private PfValidationResult validateDatatypesInPolicyTypes(final PfValidationResult result) {
+    private PfValidationResult validateReferencedDataTypes(final PfValidationResult result) {
         if (policyTypes == null) {
             return result;
         }
 
-        for (JpaToscaPolicyType policyType : policyTypes.getAll(null)) {
-            for (PfConceptKey datatypeKey : policyType.getReferencedDataTypes()) {
-                if (dataTypes == null || dataTypes.get(datatypeKey) == null) {
-                    result.addValidationMessage(
-                            new PfValidationMessage(policyType.getKey(), this.getClass(), ValidationResult.INVALID,
-                                    "data type " + datatypeKey + " referenced in policy type not found"));
-                }
+        if (dataTypes != null) {
+            for (JpaToscaDataType dataType : dataTypes.getAll(null)) {
+                validateReferencedDataTypesExists(dataType.getKey(), dataType.getReferencedDataTypes(), result);
             }
         }
 
+        for (JpaToscaPolicyType policyType : policyTypes.getAll(null)) {
+            validateReferencedDataTypesExists(policyType.getKey(), policyType.getReferencedDataTypes(), result);
+        }
+
         return result;
     }
 
     /**
+     * Validate that the referenced data types exist for a collection of data type keys.
+     *
+     * @param referencingEntityKey the key of the referencing entity
+     * @param dataTypeKeyCollection the data type key collection
+     * @param result the result of the validation
+     */
+    private void validateReferencedDataTypesExists(final PfConceptKey referencingEntityKey,
+            final Collection<PfConceptKey> dataTypeKeyCollection, final PfValidationResult result) {
+        for (PfConceptKey dataTypeKey : dataTypeKeyCollection) {
+            if (dataTypes == null || dataTypes.get(dataTypeKey) == null) {
+                result.addValidationMessage(new PfValidationMessage(referencingEntityKey, this.getClass(),
+                        ValidationResult.INVALID, "referenced data type " + dataTypeKey.getId() + " not found"));
+            }
+        }
+    }
+
+    /**
      * Validate that all policy types referenced in policies exist.
      *
      * @param result the validation result object to use for the validation result
      * @return the validation result object
      */
     private PfValidationResult validatePolicyTypesInPolicies(PfValidationResult result) {
-        if (topologyTemplate == null || topologyTemplate.getPolicies() == null) {
+        if (topologyTemplate == null || topologyTemplate.getPolicies() == null
+                || topologyTemplate.getPolicies().getConceptMap().isEmpty()) {
             return result;
         }
 
-        if (policyTypes == null) {
+        if (policyTypes == null || policyTypes.getConceptMap().isEmpty()) {
             result.addValidationMessage(new PfValidationMessage(this.getKey(), this.getClass(),
                     ValidationResult.INVALID,
                     "no policy types are defined on the service template for the policies in the topology template"));
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaTopologyTemplate.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaTopologyTemplate.java
index c8dfa5a36..176608709 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaTopologyTemplate.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/concepts/JpaToscaTopologyTemplate.java
@@ -30,6 +30,8 @@ import javax.persistence.Entity;
 import javax.persistence.FetchType;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
+import javax.persistence.JoinColumn;
+import javax.persistence.JoinColumns;
 import javax.persistence.OneToOne;
 import javax.persistence.Table;
 
@@ -69,7 +71,15 @@ public class JpaToscaTopologyTemplate extends PfConcept implements PfAuthorative
     @Column(name = "description")
     private String description;
 
+    // @formatter:off
     @OneToOne(fetch = FetchType.EAGER, cascade = CascadeType.ALL, orphanRemoval = true)
+    @JoinColumns(
+            {
+                @JoinColumn(name = "policyName",    referencedColumnName = "name"),
+                @JoinColumn(name = "policyVersion", referencedColumnName = "version")
+            }
+        )
+    // @formatter:on
     private JpaToscaPolicies policies;
 
     /**
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java
index 508b47060..9c7d6d305 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java
@@ -21,9 +21,8 @@
 package org.onap.policy.models.tosca.simple.provider;
 
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
+import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 
 import javax.ws.rs.core.Response;
 
@@ -36,6 +35,7 @@ import org.onap.policy.models.base.PfConceptKey;
 import org.onap.policy.models.base.PfKey;
 import org.onap.policy.models.base.PfModelException;
 import org.onap.policy.models.base.PfModelRuntimeException;
+import org.onap.policy.models.base.PfValidationResult;
 import org.onap.policy.models.dao.PfDao;
 import org.onap.policy.models.tosca.simple.concepts.JpaToscaDataType;
 import org.onap.policy.models.tosca.simple.concepts.JpaToscaDataTypes;
@@ -44,7 +44,7 @@ import org.onap.policy.models.tosca.simple.concepts.JpaToscaPolicy;
 import org.onap.policy.models.tosca.simple.concepts.JpaToscaPolicyType;
 import org.onap.policy.models.tosca.simple.concepts.JpaToscaPolicyTypes;
 import org.onap.policy.models.tosca.simple.concepts.JpaToscaServiceTemplate;
-import org.onap.policy.models.tosca.simple.concepts.JpaToscaTopologyTemplate;
+import org.onap.policy.models.tosca.utils.ToscaServiceTemplateUtils;
 import org.onap.policy.models.tosca.utils.ToscaUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -57,6 +57,64 @@ import org.slf4j.LoggerFactory;
 public class SimpleToscaProvider {
     private static final Logger LOGGER = LoggerFactory.getLogger(SimpleToscaProvider.class);
 
+    // Recurring string constants
+    private static final String SERVICE_TEMPLATE_NOT_FOUND_IN_DATABASE = "service template not found in database";
+    private static final String DO_NOT_EXIST = " do not exist";
+
+    /**
+     * Get Service Template.
+     *
+     * @param dao the DAO to use to access the database
+     * @return the service template
+     * @throws PfModelException on errors getting the service template
+     */
+    public JpaToscaServiceTemplate getServiceTemplate(@NonNull final PfDao dao) throws PfModelException {
+        LOGGER.debug("->getServiceTemplate");
+
+        JpaToscaServiceTemplate serviceTemplate = new SimpleToscaServiceTemplateProvider().read(dao);
+        if (serviceTemplate == null) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND, SERVICE_TEMPLATE_NOT_FOUND_IN_DATABASE);
+        }
+
+        LOGGER.debug("<-getServiceTemplate: serviceTemplate={}", serviceTemplate);
+        return serviceTemplate;
+    }
+
+    /**
+     * Append a service template fragment to the service template in the database.
+     *
+     * @param dao the DAO to use to access the database
+     * @param incomingServiceTemplateFragment the service template containing the definition of the entities to be
+     *        created
+     * @return the TOSCA service template in the database after the operation
+     * @throws PfModelException on errors appending a service template to the template in the database
+     */
+    public JpaToscaServiceTemplate appendToServiceTemplate(@NonNull final PfDao dao,
+            @NonNull final JpaToscaServiceTemplate incomingServiceTemplateFragment) throws PfModelException {
+        LOGGER.debug("->appendServiceTemplateFragment: incomingServiceTemplateFragment={}",
+                incomingServiceTemplateFragment);
+
+        JpaToscaServiceTemplate dbServiceTemplate = new SimpleToscaServiceTemplateProvider().read(dao);
+
+        JpaToscaServiceTemplate serviceTemplateToWrite;
+        if (dbServiceTemplate == null) {
+            serviceTemplateToWrite = incomingServiceTemplateFragment;
+        } else {
+            serviceTemplateToWrite =
+                    ToscaServiceTemplateUtils.addFragment(dbServiceTemplate, incomingServiceTemplateFragment);
+        }
+
+        PfValidationResult result = serviceTemplateToWrite.validate(new PfValidationResult());
+        if (!result.isValid()) {
+            throw new PfModelRuntimeException(Response.Status.BAD_REQUEST, result.toString());
+        }
+
+        new SimpleToscaServiceTemplateProvider().write(dao, serviceTemplateToWrite);
+
+        LOGGER.debug("<-appendServiceTemplateFragment: returnServiceTempalate={}", serviceTemplateToWrite);
+        return serviceTemplateToWrite;
+    }
+
     /**
      * Get data types.
      *
@@ -70,13 +128,34 @@ public class SimpleToscaProvider {
             throws PfModelException {
         LOGGER.debug("->getDataTypes: name={}, version={}", name, version);
 
-        // Create the structure of the TOSCA service template to contain the data type
-        JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate();
-        serviceTemplate.setDataTypes(new JpaToscaDataTypes());
+        JpaToscaServiceTemplate serviceTemplate = getServiceTemplate(dao);
+
+        if (!ToscaUtils.doDataTypesExist(serviceTemplate)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "data types for " + name + ":" + version + DO_NOT_EXIST);
+        }
 
-        // Add the data type to the TOSCA service template
-        List<JpaToscaDataType> jpaDataTypeList = dao.getFiltered(JpaToscaDataType.class, name, version);
-        serviceTemplate.getDataTypes().getConceptMap().putAll(asConceptMap(jpaDataTypeList));
+        serviceTemplate.setPolicyTypes(null);
+        serviceTemplate.setTopologyTemplate(null);
+
+        ToscaUtils.getEntityTree(serviceTemplate.getDataTypes(), name, version);
+
+        if (!ToscaUtils.doDataTypesExist(serviceTemplate)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "data types for " + name + ":" + version + DO_NOT_EXIST);
+        }
+
+        for (JpaToscaDataType dataType : serviceTemplate.getDataTypes().getConceptMap().values()) {
+            Collection<PfConceptKey> referencedDataTypeKeys = dataType.getReferencedDataTypes();
+
+            for (PfConceptKey referencedDataTypeKey : referencedDataTypeKeys) {
+                JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate =
+                        getDataTypes(dao, referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion());
+
+                serviceTemplate =
+                        ToscaServiceTemplateUtils.addFragment(serviceTemplate, dataTypeEntityTreeServiceTemplate);
+            }
+        }
 
         LOGGER.debug("<-getDataTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate);
         return serviceTemplate;
@@ -86,32 +165,20 @@ public class SimpleToscaProvider {
      * Create data types.
      *
      * @param dao the DAO to use to access the database
-     * @param serviceTemplate the service template containing the definition of the data types to be created
+     * @param incomingServiceTemplate the service template containing the definition of the data types to be created
      * @return the TOSCA service template containing the created data types
      * @throws PfModelException on errors creating data types
      */
     public JpaToscaServiceTemplate createDataTypes(@NonNull final PfDao dao,
-            @NonNull final JpaToscaServiceTemplate serviceTemplate) throws PfModelException {
-        LOGGER.debug("->createDataTypes: serviceTempalate={}", serviceTemplate);
-
-        ToscaUtils.assertDataTypesExist(serviceTemplate);
-
-        for (JpaToscaDataType dataType : serviceTemplate.getDataTypes().getAll(null)) {
-            dao.create(dataType);
-        }
+            @NonNull final JpaToscaServiceTemplate incomingServiceTemplate) throws PfModelException {
+        LOGGER.debug("->createDataTypes: incomingServiceTemplate={}", incomingServiceTemplate);
 
-        // Return the created Data types
-        JpaToscaDataTypes returnDataTypes = new JpaToscaDataTypes();
+        ToscaUtils.assertDataTypesExist(incomingServiceTemplate);
 
-        for (PfConceptKey dataTypeKey : serviceTemplate.getDataTypes().getConceptMap().keySet()) {
-            returnDataTypes.getConceptMap().put(dataTypeKey, dao.get(JpaToscaDataType.class, dataTypeKey));
-        }
+        JpaToscaServiceTemplate writtenServiceTemplate = appendToServiceTemplate(dao, incomingServiceTemplate);
 
-        JpaToscaServiceTemplate returnServiceTemplate = new JpaToscaServiceTemplate();
-        returnServiceTemplate.setDataTypes(returnDataTypes);
-
-        LOGGER.debug("<-createDataTypes: returnServiceTempalate={}", returnServiceTemplate);
-        return returnServiceTemplate;
+        LOGGER.debug("<-createDataTypes: returnServiceTempalate={}", writtenServiceTemplate);
+        return writtenServiceTemplate;
     }
 
     /**
@@ -180,20 +247,33 @@ public class SimpleToscaProvider {
             throws PfModelException {
         LOGGER.debug("->getPolicyTypes: name={}, version={}", name, version);
 
-        // Create the structure of the TOSCA service template to contain the policy type
-        JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate();
-        serviceTemplate.setPolicyTypes(new JpaToscaPolicyTypes());
+        JpaToscaServiceTemplate serviceTemplate = getServiceTemplate(dao);
+
+        serviceTemplate.setDataTypes(null);
+        serviceTemplate.setTopologyTemplate(null);
+
+        if (!ToscaUtils.doPolicyTypesExist(serviceTemplate)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "policy types for " + name + ":" + version + DO_NOT_EXIST);
+        }
+
+        ToscaUtils.getEntityTree(serviceTemplate.getPolicyTypes(), name, version);
 
-        // Add the policy type to the TOSCA service template
-        List<JpaToscaPolicyType> jpaPolicyTypeList = dao.getFiltered(JpaToscaPolicyType.class, name, version);
-        serviceTemplate.getPolicyTypes().getConceptMap().putAll(asConceptMap(jpaPolicyTypeList));
+        if (!ToscaUtils.doPolicyTypesExist(serviceTemplate)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "policy types for " + name + ":" + version + DO_NOT_EXIST);
+        }
+
+        for (JpaToscaPolicyType policyType : serviceTemplate.getPolicyTypes().getConceptMap().values()) {
+            Collection<PfConceptKey> referencedDataTypeKeys = policyType.getReferencedDataTypes();
 
-        // Return all data types
-        // TODO: In an upcoming review, return just the data types used by the policy types on the policy type list
-        List<JpaToscaDataType> jpaDataTypeList = dao.getFiltered(JpaToscaDataType.class, null, null);
-        if (!CollectionUtils.isEmpty(jpaDataTypeList)) {
-            serviceTemplate.setDataTypes(new JpaToscaDataTypes());
-            serviceTemplate.getDataTypes().getConceptMap().putAll(asConceptMap(jpaDataTypeList));
+            for (PfConceptKey referencedDataTypeKey : referencedDataTypeKeys) {
+                JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate =
+                        getDataTypes(dao, referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion());
+
+                serviceTemplate =
+                        ToscaServiceTemplateUtils.addFragment(serviceTemplate, dataTypeEntityTreeServiceTemplate);
+            }
         }
 
         LOGGER.debug("<-getPolicyTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate);
@@ -204,37 +284,20 @@ public class SimpleToscaProvider {
      * Create policy types.
      *
      * @param dao the DAO to use to access the database
-     * @param serviceTemplate the service template containing the definition of the policy types to be created
+     * @param incomingServiceTemplate the service template containing the definition of the policy types to be created
      * @return the TOSCA service template containing the created policy types
      * @throws PfModelException on errors creating policy types
      */
     public JpaToscaServiceTemplate createPolicyTypes(@NonNull final PfDao dao,
-            @NonNull final JpaToscaServiceTemplate serviceTemplate) throws PfModelException {
-        LOGGER.debug("->createPolicyTypes: serviceTempalate={}", serviceTemplate);
-
-        ToscaUtils.assertPolicyTypesExist(serviceTemplate);
+            @NonNull final JpaToscaServiceTemplate incomingServiceTemplate) throws PfModelException {
+        LOGGER.debug("->createPolicyTypes: serviceTempalate={}", incomingServiceTemplate);
 
-        // Create the data types on the policy type
-        if (ToscaUtils.doDataTypesExist(serviceTemplate)) {
-            createDataTypes(dao, serviceTemplate);
-        }
+        ToscaUtils.assertPolicyTypesExist(incomingServiceTemplate);
 
-        for (JpaToscaPolicyType policyType : serviceTemplate.getPolicyTypes().getAll(null)) {
-            dao.create(policyType);
-        }
-
-        // Return the created policy types
-        JpaToscaPolicyTypes returnPolicyTypes = new JpaToscaPolicyTypes();
+        JpaToscaServiceTemplate writtenServiceTemplate = appendToServiceTemplate(dao, incomingServiceTemplate);
 
-        for (PfConceptKey policyTypeKey : serviceTemplate.getPolicyTypes().getConceptMap().keySet()) {
-            returnPolicyTypes.getConceptMap().put(policyTypeKey, dao.get(JpaToscaPolicyType.class, policyTypeKey));
-        }
-
-        JpaToscaServiceTemplate returnServiceTemplate = new JpaToscaServiceTemplate();
-        returnServiceTemplate.setPolicyTypes(returnPolicyTypes);
-
-        LOGGER.debug("<-createPolicyTypes: returnServiceTempalate={}", returnServiceTemplate);
-        return returnServiceTemplate;
+        LOGGER.debug("<-createPolicyTypes: returnServiceTempalate={}", writtenServiceTemplate);
+        return writtenServiceTemplate;
     }
 
     /**
@@ -309,14 +372,29 @@ public class SimpleToscaProvider {
             throws PfModelException {
         LOGGER.debug("->getPolicies: name={}, version={}", name, version);
 
-        // Create the structure of the TOSCA service template to contain the policy type
-        JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate();
-        serviceTemplate.setTopologyTemplate(new JpaToscaTopologyTemplate());
-        serviceTemplate.getTopologyTemplate().setPolicies(new JpaToscaPolicies());
+        JpaToscaServiceTemplate serviceTemplate = getServiceTemplate(dao);
+
+        if (!ToscaUtils.doPoliciesExist(serviceTemplate)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "policies for " + name + ":" + version + DO_NOT_EXIST);
+        }
+
+        ToscaUtils.getEntityTree(serviceTemplate.getTopologyTemplate().getPolicies(), name, version);
+
+        if (!ToscaUtils.doPoliciesExist(serviceTemplate)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "policies for " + name + ":" + version + DO_NOT_EXIST);
+        }
 
-        // Add the policy type to the TOSCA service template
-        List<JpaToscaPolicy> jpaPolicyList = dao.getFiltered(JpaToscaPolicy.class, name, version);
-        serviceTemplate.getTopologyTemplate().getPolicies().getConceptMap().putAll(asConceptMap(jpaPolicyList));
+        for (JpaToscaPolicy policy : serviceTemplate.getTopologyTemplate().getPolicies().getConceptMap().values()) {
+            if (policy.getDerivedFrom() != null) {
+                JpaToscaServiceTemplate referencedEntitiesServiceTemplate =
+                        getPolicyTypes(dao, policy.getDerivedFrom().getName(), policy.getDerivedFrom().getVersion());
+
+                serviceTemplate =
+                        ToscaServiceTemplateUtils.addFragment(serviceTemplate, referencedEntitiesServiceTemplate);
+            }
+        }
 
         LOGGER.debug("<-getPolicies: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate);
         return serviceTemplate;
@@ -326,33 +404,20 @@ public class SimpleToscaProvider {
      * Create policies.
      *
      * @param dao the DAO to use to access the database
-     * @param serviceTemplate the service template containing the definitions of the new policies to be created.
+     * @param incomingServiceTemplate the service template containing the definitions of the new policies to be created.
      * @return the TOSCA service template containing the policy types that were created
      * @throws PfModelException on errors creating policies
      */
     public JpaToscaServiceTemplate createPolicies(@NonNull final PfDao dao,
-            @NonNull final JpaToscaServiceTemplate serviceTemplate) throws PfModelException {
-        LOGGER.debug("->createPolicies: serviceTempalate={}", serviceTemplate);
-
-        ToscaUtils.assertPoliciesExist(serviceTemplate);
-
-        for (JpaToscaPolicy policy : serviceTemplate.getTopologyTemplate().getPolicies().getAll(null)) {
-            verifyPolicyTypeForPolicy(dao, policy);
-            dao.create(policy);
-        }
-
-        // Return the created policy types
-        JpaToscaPolicies returnPolicies = new JpaToscaPolicies();
-        returnPolicies.setKey(serviceTemplate.getTopologyTemplate().getPolicies().getKey());
+            @NonNull final JpaToscaServiceTemplate incomingServiceTemplate) throws PfModelException {
+        LOGGER.debug("->createPolicies: incomingServiceTemplate={}", incomingServiceTemplate);
 
-        for (PfConceptKey policyKey : serviceTemplate.getTopologyTemplate().getPolicies().getConceptMap().keySet()) {
-            returnPolicies.getConceptMap().put(policyKey, dao.get(JpaToscaPolicy.class, policyKey));
-        }
+        ToscaUtils.assertPoliciesExist(incomingServiceTemplate);
 
-        serviceTemplate.getTopologyTemplate().setPolicies(returnPolicies);
+        JpaToscaServiceTemplate writtenServiceTemplate = appendToServiceTemplate(dao, incomingServiceTemplate);
 
-        LOGGER.debug("<-createPolicies: serviceTemplate={}", serviceTemplate);
-        return serviceTemplate;
+        LOGGER.debug("<-createPolicies: serviceTemplate={}", writtenServiceTemplate);
+        return writtenServiceTemplate;
     }
 
     /**
@@ -409,21 +474,6 @@ public class SimpleToscaProvider {
     }
 
     /**
-     * Convert a list of concepts to a map of concepts.
-     *
-     * @param conceptList the concept list
-     * @return the concept map
-     */
-    private <T extends PfConcept> Map<PfConceptKey, T> asConceptMap(List<T> conceptList) {
-        Map<PfConceptKey, T> conceptMap = new LinkedHashMap<>();
-        for (T concept : conceptList) {
-            conceptMap.put((PfConceptKey) concept.getKey(), concept);
-        }
-
-        return conceptMap;
-    }
-
-    /**
      * Verify the policy type for a policy exists.
      *
      * @param dao the DAO to use to access policy types in the database
@@ -447,7 +497,6 @@ public class SimpleToscaProvider {
         if (policyType == null) {
             String errorMessage =
                     "policy type " + policyTypeKey.getId() + " for policy " + policy.getId() + " does not exist";
-            LOGGER.warn(errorMessage);
             throw new PfModelRuntimeException(Response.Status.BAD_REQUEST, errorMessage);
         }
     }
@@ -476,8 +525,7 @@ public class SimpleToscaProvider {
 
         // We should have one and only one returned entry
         if (filterdPolicyTypeList.size() != 1) {
-            String errorMessage = "search for lates policy type " + policyTypeName + " returned more than one entry";
-            LOGGER.warn(errorMessage);
+            String errorMessage = "search for latest policy type " + policyTypeName + " returned more than one entry";
             throw new PfModelRuntimeException(Response.Status.BAD_REQUEST, errorMessage);
         }
 
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/utils/ToscaUtils.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/utils/ToscaUtils.java
index 390692277..cc0431946 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/utils/ToscaUtils.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/utils/ToscaUtils.java
@@ -52,14 +52,18 @@ public final class ToscaUtils {
 
     // @formatter:off
     private static final Set<PfConceptKey> PREDEFINED_TOSCA_DATA_TYPES = Set.of(
-            new PfConceptKey("string",    PfKey.NULL_KEY_VERSION),
-            new PfConceptKey("integer",   PfKey.NULL_KEY_VERSION),
-            new PfConceptKey("float",     PfKey.NULL_KEY_VERSION),
-            new PfConceptKey("boolean",   PfKey.NULL_KEY_VERSION),
-            new PfConceptKey("timestamp", PfKey.NULL_KEY_VERSION),
-            new PfConceptKey("null",      PfKey.NULL_KEY_VERSION),
-            new PfConceptKey("list",      PfKey.NULL_KEY_VERSION),
-            new PfConceptKey("map",       PfKey.NULL_KEY_VERSION)
+            new PfConceptKey("string",                       PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("integer",                      PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("float",                        PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("boolean",                      PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("timestamp",                    PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("null",                         PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("list",                         PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("map",                          PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("scalar-unit.size",             PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("scalar-unit.time",             PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("scalar-unit.frequency",        PfKey.NULL_KEY_VERSION),
+            new PfConceptKey("tosca.datatypes.TimeInterval", PfKey.NULL_KEY_VERSION)
         );
     // @formatter:on
 
@@ -242,17 +246,18 @@ public final class ToscaUtils {
      * Get the entity tree from a concept container for a given entity key.
      *
      * @param entityTypes the concept container containing entity types
-     * @param searchKey the key to search for
+     * @param entityName the name of the entity
+     * @param entityVersion the version of the entity
      */
     public static void getEntityTree(
             @NonNull final PfConceptContainer<? extends PfConcept, ? extends PfNameVersion> entityTypes,
-            @NonNull final PfConceptKey searchKey) {
+            final String entityName, final String entityVersion) {
 
         PfValidationResult result = new PfValidationResult();
 
         @SuppressWarnings("unchecked")
         Set<JpaToscaEntityType<?>> filteredEntitySet =
-                (Set<JpaToscaEntityType<?>>) entityTypes.getAll(searchKey.getName(), searchKey.getVersion());
+                (Set<JpaToscaEntityType<?>>) entityTypes.getAll(entityName, entityVersion);
         for (JpaToscaEntityType<?> filteredEntityType : filteredEntitySet) {
             filteredEntitySet.addAll(ToscaUtils.getEntityTypeAncestors(entityTypes, filteredEntityType, result));
         }