Read Performance Improvement - Using Native Query

- Native query for FragmentExtracts
- Convert FragmentExtracts to tree of FragmentEntity
- Native Query now used for all Gets with descendants (orignal hibernate option only used when descendanst ommited)
- Added error handling for not-found on native query
- Ommit descendants by default on many udpate use-cases (this might have a signifcant perf. improvemnt impact too)
- Improved legacy tests for delete use-cases
- Corrected performace test expectation
- Fix TTL test realizing TTL resolution is whole seconds!

Issue-ID: CPS-1301
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>
Change-Id: I658ac1b7b7036f01050f30bdf9e5bd175725ef1d

5 files changed, 166 insertions, 19 deletions

diff --git a/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentEntityArranger.java b/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentEntityArranger.java
new file mode 100644
index 0000000000..50187a487b
--- /dev/null
+++ b/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentEntityArranger.java
@@ -0,0 +1,79 @@
+/*
+ *  ============LICENSE_START=======================================================
+ *  Copyright (C) 2022 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.spi.entities;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class FragmentEntityArranger {
+
+    /**
+     * Convert a collection of (related) FragmentExtracts into a FragmentEntity (tree) with descendants.
+     * Multiple top level nodes not yet support. If found only the first top level element is returned
+     *
+     * @param anchorEntity the anchor(entity) all the fragments belong to
+     * @param fragmentExtracts FragmentExtracts to convert
+     * @return a FragmentEntity (tree) with descendants, null if none found.
+     */
+    public static FragmentEntity toFragmentEntityTree(final AnchorEntity anchorEntity,
+                                                      final Collection<FragmentExtract> fragmentExtracts) {
+        final Map<Long, FragmentEntity> fragmentEntityPerId = new HashMap<>();
+        for (final FragmentExtract fragmentExtract : fragmentExtracts) {
+            final FragmentEntity fragmentEntity = toFragmentEntity(anchorEntity, fragmentExtract);
+            fragmentEntityPerId.put(fragmentEntity.getId(), fragmentEntity);
+        }
+        return reuniteChildrenWithTheirParents(fragmentEntityPerId);
+    }
+
+    private static FragmentEntity toFragmentEntity(final AnchorEntity anchorEntity,
+                                                   final FragmentExtract fragmentExtract) {
+        final FragmentEntity fragmentEntity = new FragmentEntity();
+        fragmentEntity.setAnchor(anchorEntity);
+        fragmentEntity.setId(fragmentExtract.getId());
+        fragmentEntity.setXpath(fragmentExtract.getXpath());
+        fragmentEntity.setAttributes(fragmentExtract.getAttributes());
+        fragmentEntity.setParentId(fragmentExtract.getParentId());
+        fragmentEntity.setChildFragments(new HashSet<>());
+        return fragmentEntity;
+    }
+
+    private static FragmentEntity reuniteChildrenWithTheirParents(final Map<Long, FragmentEntity> fragmentEntityPerId) {
+        final Collection<FragmentEntity> fragmentEntitiesWithoutParentInResultSet = new HashSet<>();
+        for (final FragmentEntity fragmentEntity : fragmentEntityPerId.values()) {
+            final FragmentEntity parentFragmentEntity = fragmentEntityPerId.get(fragmentEntity.getParentId());
+            if (parentFragmentEntity == null) {
+                fragmentEntitiesWithoutParentInResultSet.add(fragmentEntity);
+            } else {
+                parentFragmentEntity.getChildFragments().add(fragmentEntity);
+            }
+        }
+        if (fragmentEntitiesWithoutParentInResultSet.iterator().hasNext()) {
+            return fragmentEntitiesWithoutParentInResultSet.iterator().next();
+        }
+        return null;
+    }
+
+}
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentExtract.java b/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentExtract.java
new file mode 100644
index 0000000000..52c1a603bb
--- /dev/null
+++ b/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentExtract.java
@@ -0,0 +1,34 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2022 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.spi.entities;
+
+public interface FragmentExtract {
+
+    Long getId();
+
+    Long getAnchorId();
+
+    String getXpath();
+
+    Long getParentId();
+
+    String getAttributes();
+}
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java
index 06c12a89d2..2a4a192260 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java
@@ -51,6 +51,8 @@ import org.onap.cps.spi.cache.AnchorDataCacheEntry;
 import org.onap.cps.spi.entities.AnchorEntity;
 import org.onap.cps.spi.entities.DataspaceEntity;
 import org.onap.cps.spi.entities.FragmentEntity;
+import org.onap.cps.spi.entities.FragmentEntityArranger;
+import org.onap.cps.spi.entities.FragmentExtract;
 import org.onap.cps.spi.entities.SchemaSetEntity;
 import org.onap.cps.spi.entities.YangResourceEntity;
 import org.onap.cps.spi.exceptions.AlreadyDefinedException;
@@ -120,7 +122,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
 
     private void addNewChildDataNode(final String dataspaceName, final String anchorName,
                                      final String parentNodeXpath, final DataNode newChild) {
-        final FragmentEntity parentFragmentEntity = getFragmentByXpath(dataspaceName, anchorName, parentNodeXpath);
+        final FragmentEntity parentFragmentEntity =
+            getFragmentWithoutDescendantsByXpath(dataspaceName, anchorName, parentNodeXpath);
         final FragmentEntity newChildAsFragmentEntity =
                 convertToFragmentWithAllDescendants(parentFragmentEntity.getDataspace(),
                         parentFragmentEntity.getAnchor(), newChild);
@@ -135,7 +138,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
 
     private void addChildrenDataNodes(final String dataspaceName, final String anchorName, final String parentNodeXpath,
                                       final Collection<DataNode> newChildren) {
-        final FragmentEntity parentFragmentEntity = getFragmentByXpath(dataspaceName, anchorName, parentNodeXpath);
+        final FragmentEntity parentFragmentEntity =
+            getFragmentWithoutDescendantsByXpath(dataspaceName, anchorName, parentNodeXpath);
         final List<FragmentEntity> fragmentEntities = new ArrayList<>(newChildren.size());
         try {
             newChildren.forEach(newChildAsDataNode -> {
@@ -219,19 +223,25 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
     @Override
     public DataNode getDataNode(final String dataspaceName, final String anchorName, final String xpath,
                                 final FetchDescendantsOption fetchDescendantsOption) {
-        final FragmentEntity fragmentEntity = getFragmentByXpath(dataspaceName, anchorName, xpath);
+        final FragmentEntity fragmentEntity = getFragmentByXpath(dataspaceName, anchorName, xpath,
+            fetchDescendantsOption);
         final DataNode dataNode = toDataNode(fragmentEntity, fetchDescendantsOption);
         dataNode.setModuleNamePrefix(getRootModuleNamePrefix(fragmentEntity.getAnchor()));
         return dataNode;
     }
 
+    private FragmentEntity getFragmentWithoutDescendantsByXpath(final String dataspaceName,
+                                                                final String anchorName,
+                                                                final String xpath) {
+        return getFragmentByXpath(dataspaceName, anchorName, xpath, FetchDescendantsOption.OMIT_DESCENDANTS);
+    }
+
     private FragmentEntity getFragmentByXpath(final String dataspaceName, final String anchorName,
-                                              final String xpath) {
+                                              final String xpath, final FetchDescendantsOption fetchDescendantsOption) {
         final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
         final AnchorEntity anchorEntity = anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName);
         if (isRootXpath(xpath)) {
-            return fragmentRepository.findFirstRootByDataspaceAndAnchor(
-                    dataspaceEntity, anchorEntity);
+            return fragmentRepository.findFirstRootByDataspaceAndAnchor(dataspaceEntity, anchorEntity);
         } else {
             final String normalizedXpath;
             try {
@@ -239,9 +249,21 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
             } catch (final PathParsingException e) {
                 throw new CpsPathException(e.getMessage());
             }
-
-            return fragmentRepository.getByDataspaceAndAnchorAndXpath(
-                    dataspaceEntity, anchorEntity, normalizedXpath);
+            final FragmentEntity fragmentEntity;
+            if (FetchDescendantsOption.OMIT_DESCENDANTS.equals(fetchDescendantsOption)) {
+                fragmentEntity =
+                    fragmentRepository.getByDataspaceAndAnchorAndXpath(dataspaceEntity, anchorEntity, normalizedXpath);
+            } else {
+                final List<FragmentExtract> fragmentExtracts =
+                    fragmentRepository.findByAnchorIdAndParentXpath(anchorEntity.getId(), normalizedXpath);
+                log.debug("Fetched {} fragment entities by anchor {} and cps path {}.",
+                    fragmentExtracts.size(), anchorName, xpath);
+                fragmentEntity = FragmentEntityArranger.toFragmentEntityTree(anchorEntity, fragmentExtracts);
+            }
+            if (fragmentEntity == null) {
+                throw new DataNodeNotFoundException(dataspaceEntity.getName(), anchorEntity.getName(), xpath);
+            }
+            return fragmentEntity;
         }
     }
 
@@ -316,7 +338,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
     }
 
     private String getRootModuleNamePrefix(final AnchorEntity anchorEntity) {
-        final String cachedModuleNamePrefix = getModuleNamePrefixFromCache(anchorEntity);
+        final String cachedModuleNamePrefix = getModuleNamePrefixFromCache(anchorEntity.getName());
         if (cachedModuleNamePrefix != null) {
             return cachedModuleNamePrefix;
         }
@@ -344,9 +366,9 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
         }
     }
 
-    private String getModuleNamePrefixFromCache(final AnchorEntity anchorEntity) {
-        if (anchorDataCache.containsKey(anchorEntity.getName())) {
-            final AnchorDataCacheEntry anchorDataCacheEntry = anchorDataCache.get(anchorEntity.getName());
+    private String getModuleNamePrefixFromCache(final String anchorName) {
+        if (anchorDataCache.containsKey(anchorName)) {
+            final AnchorDataCacheEntry anchorDataCacheEntry = anchorDataCache.get(anchorName);
             return anchorDataCacheEntry.hasProperty(TOP_LEVEL_MODULE_PREFIX_PROPERTY_NAME)
                     ? anchorDataCacheEntry.getProperty(TOP_LEVEL_MODULE_PREFIX_PROPERTY_NAME).toString() : null;
         }
@@ -366,7 +388,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
     @Override
     public void updateDataLeaves(final String dataspaceName, final String anchorName, final String xpath,
                                  final Map<String, Object> leaves) {
-        final FragmentEntity fragmentEntity = getFragmentByXpath(dataspaceName, anchorName, xpath);
+        final FragmentEntity fragmentEntity = getFragmentWithoutDescendantsByXpath(dataspaceName, anchorName, xpath);
         fragmentEntity.setAttributes(jsonObjectMapper.asJsonString(leaves));
         fragmentRepository.save(fragmentEntity);
     }
@@ -374,7 +396,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
     @Override
     public void updateDataNodeAndDescendants(final String dataspaceName, final String anchorName,
                                              final DataNode dataNode) {
-        final FragmentEntity fragmentEntity = getFragmentByXpath(dataspaceName, anchorName, dataNode.getXpath());
+        final FragmentEntity fragmentEntity =
+            getFragmentWithoutDescendantsByXpath(dataspaceName, anchorName, dataNode.getXpath());
         updateFragmentEntityAndDescendantsWithDataNode(fragmentEntity, dataNode);
         try {
             fragmentRepository.save(fragmentEntity);
@@ -393,7 +416,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
         final Map<DataNode, FragmentEntity> dataNodeFragmentEntityMap = dataNodes.stream()
                 .collect(Collectors.toMap(
                         dataNode -> dataNode,
-                        dataNode -> getFragmentByXpath(dataspaceName, anchorName, dataNode.getXpath())));
+                        dataNode ->
+                            getFragmentWithoutDescendantsByXpath(dataspaceName, anchorName, dataNode.getXpath())));
         dataNodeFragmentEntityMap.forEach(
                 (dataNode, fragmentEntity) -> updateFragmentEntityAndDescendantsWithDataNode(fragmentEntity, dataNode));
         try {
@@ -453,7 +477,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
     @Transactional
     public void replaceListContent(final String dataspaceName, final String anchorName, final String parentNodeXpath,
                                    final Collection<DataNode> newListElements) {
-        final FragmentEntity parentEntity = getFragmentByXpath(dataspaceName, anchorName, parentNodeXpath);
+        final FragmentEntity parentEntity =
+            getFragmentWithoutDescendantsByXpath(dataspaceName, anchorName, parentNodeXpath);
         final String listElementXpathPrefix = getListElementXpathPrefix(newListElements);
         final Map<String, FragmentEntity> existingListElementFragmentEntitiesByXPath =
                 extractListElementFragmentEntitiesByXPath(parentEntity.getChildFragments(), listElementXpathPrefix);
@@ -507,7 +532,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
             } else {
                 parentNodeXpath = targetXpath.substring(0, targetXpath.lastIndexOf('/'));
             }
-            parentFragmentEntity = getFragmentByXpath(dataspaceName, anchorName, parentNodeXpath);
+            parentFragmentEntity = getFragmentWithoutDescendantsByXpath(dataspaceName, anchorName, parentNodeXpath);
             final String lastXpathElement = targetXpath.substring(targetXpath.lastIndexOf('/'));
             final boolean isListElement = REG_EX_PATTERN_FOR_LIST_ELEMENT_KEY_PREDICATE
                     .matcher(lastXpathElement).find();
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java
index c48c79ef65..112ebfda70 100755
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java
@@ -30,6 +30,7 @@ import org.checkerframework.checker.nullness.qual.NonNull;
 import org.onap.cps.spi.entities.AnchorEntity;

 import org.onap.cps.spi.entities.DataspaceEntity;

 import org.onap.cps.spi.entities.FragmentEntity;

+import org.onap.cps.spi.entities.FragmentExtract;

 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;

 import org.springframework.data.jpa.repository.JpaRepository;

 import org.springframework.data.jpa.repository.Modifying;

@@ -69,4 +70,12 @@ public interface FragmentRepository extends JpaRepository<FragmentEntity, Long>,
     @Modifying

     @Query("DELETE FROM FragmentEntity fe WHERE fe.anchor IN (:anchors)")

     void deleteByAnchorIn(@NotNull @Param("anchors") Collection<AnchorEntity> anchorEntities);

+

+    @Query(value = "SELECT id, anchor_id AS anchorId, xpath, parent_id AS parentId,"

+        + " CAST(attributes AS TEXT) AS attributes"

+        + " FROM FRAGMENT WHERE anchor_id = :anchorId"

+        + " AND ( xpath = :parentXpath OR xpath LIKE CONCAT(:parentXpath,'/%') )",

+           nativeQuery = true)

+    List<FragmentExtract> findByAnchorIdAndParentXpath(@Param("anchorId") int anchorId,

+                                                       @Param("parentXpath") String parentXpath);

 }

diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java
index 654c1c0854..1d61416cfd 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java
@@ -57,7 +57,7 @@ public class FragmentRepositoryCpsPathQueryImpl implements FragmentRepositoryCps
         if (cpsPathQuery.hasLeafConditions()) {
             sqlStringBuilder.append(" AND attributes @> :leafDataAsJson\\:\\:jsonb");
             queryParameters.put("leafDataAsJson", jsonObjectMapper.asJsonString(
-                    cpsPathQuery.getLeavesData()));
+                cpsPathQuery.getLeavesData()));
         }
 
         addTextFunctionCondition(cpsPathQuery, sqlStringBuilder, queryParameters);