Add gremlin-based pagination to aai-common

- enhance query building to support gremlin-based pagination
- pagination is supported in two variants: with and without the total count of elements [1]
- enhance query building to support gremlin-based sorting
- add query logging that is currently disabled

[1] due to the design of gremlin, including the total count results in a full graph scan.
As such there is the option to not include it, which should make it (much) faster for the first pages that are returned.
Issue-ID: AAI-3893
Change-Id: I6bc0c9b9f398556cc41a0a8f82e24e50c85e5690
Signed-off-by: Fiete Ostkamp <Fiete.Ostkamp@telekom.de>

2 files changed, 207 insertions, 37 deletions

diff --git a/aai-core/src/test/java/org/onap/aai/parsers/query/GraphTraversalTest.java b/aai-core/src/test/java/org/onap/aai/parsers/query/GraphTraversalTest.java
index 34b29ceb..04a29916 100644
--- a/aai-core/src/test/java/org/onap/aai/parsers/query/GraphTraversalTest.java
+++ b/aai-core/src/test/java/org/onap/aai/parsers/query/GraphTraversalTest.java
@@ -52,6 +52,8 @@ import org.onap.aai.TinkerpopUpgrade;
 import org.onap.aai.db.props.AAIProperties;
 import org.onap.aai.exceptions.AAIException;
 import org.onap.aai.introspection.ModelType;
+import org.onap.aai.query.builder.Pageable;
+import org.onap.aai.query.builder.QueryBuilder;
 import org.onap.aai.rest.RestTokens;
 import org.onap.aai.serialization.engines.JanusGraphDBEngine;
 import org.onap.aai.serialization.engines.QueryStyle;
@@ -79,7 +81,7 @@ public class GraphTraversalTest extends DataLinkSetup {
 
     /**
      * Configure.
-     * 
+     *
      * @throws Exception
      * @throws SecurityException
      * @throws NoSuchFieldException
@@ -579,8 +581,7 @@ public class GraphTraversalTest extends DataLinkSetup {
 
         thrown.expect(AAIException.class);
         thrown.expectMessage(containsString(RestTokens.COUSIN.toString()));
-        QueryParser query = dbEngineDepthVersion.getQueryBuilder().createQueryFromURI(uri);
-
+        dbEngineDepthVersion.getQueryBuilder().createQueryFromURI(uri);
     }
 
     @Test
@@ -589,7 +590,6 @@ public class GraphTraversalTest extends DataLinkSetup {
 
         thrown.expect(AAIException.class);
         thrown.expectMessage(containsString("chain plurals"));
-        QueryParser query = dbEngineDepthVersion.getQueryBuilder().createQueryFromURI(uri);
-
+        dbEngineDepthVersion.getQueryBuilder().createQueryFromURI(uri);
     }
 }
diff --git a/aai-core/src/test/java/org/onap/aai/rest/db/HttpEntryTest.java b/aai-core/src/test/java/org/onap/aai/rest/db/HttpEntryTest.java
index bdb02b99..0dcec24d 100644
--- a/aai-core/src/test/java/org/onap/aai/rest/db/HttpEntryTest.java
+++ b/aai-core/src/test/java/org/onap/aai/rest/db/HttpEntryTest.java
@@ -27,6 +27,7 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -35,6 +36,7 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.util.Collections;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -85,6 +87,10 @@ import org.onap.aai.introspection.Loader;
 import org.onap.aai.introspection.ModelType;
 import org.onap.aai.parsers.query.QueryParser;
 import org.onap.aai.prevalidation.ValidationService;
+import org.onap.aai.query.builder.Pageable;
+import org.onap.aai.query.builder.QueryOptions;
+import org.onap.aai.query.builder.Sort;
+import org.onap.aai.query.builder.Sort.Direction;
 import org.onap.aai.rest.db.responses.ErrorResponse;
 import org.onap.aai.rest.db.responses.Relationship;
 import org.onap.aai.rest.db.responses.RelationshipWrapper;
@@ -96,6 +102,7 @@ import org.onap.aai.serialization.engines.TransactionalGraphEngine;
 import org.onap.aai.util.AAIConfig;
 import org.skyscreamer.jsonassert.JSONAssert;
 import org.skyscreamer.jsonassert.JSONCompareMode;
+import org.skyscreamer.jsonassert.comparator.JSONComparator;
 import org.springframework.boot.test.mock.mockito.MockBean;
 
 @RunWith(value = Parameterized.class)
@@ -189,7 +196,7 @@ public class HttpEntryTest extends AAISetup {
     }
 
     @Test
-    public void thatObjectsCanBeRetrieved() throws UnsupportedEncodingException, AAIException {
+    public void thatObjectCanBeRetrieved() throws UnsupportedEncodingException, AAIException {
         String uri = "/cloud-infrastructure/pservers/pserver/theHostname";
         traversal.addV()
                 .property("aai-node-type", "pserver")
@@ -197,15 +204,11 @@ public class HttpEntryTest extends AAISetup {
                 .property("equip-type", "theEquipType")
                 .property(AAIProperties.AAI_URI, uri)
                 .next();
-        String requestBody = new JSONObject()
-                .put("hostname", "theHostname")
-                .put("equip-type", "theEquipType")
-                .toString();
 
         JSONObject expectedResponseBody = new JSONObject()
                 .put("hostname", "theHostname")
                 .put("equip-type", "theEquipType");
-        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, requestBody);
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri);
         JSONObject actualResponseBody = new JSONObject(response.getEntity().toString());
 
         JSONAssert.assertEquals(expectedResponseBody, actualResponseBody, JSONCompareMode.NON_EXTENSIBLE);
@@ -214,11 +217,168 @@ public class HttpEntryTest extends AAISetup {
     }
 
     @Test
+    public void thatObjectsCanBeRetrieved() throws UnsupportedEncodingException, AAIException {
+        String uri = "/cloud-infrastructure/pservers/pserver/theHostname";
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname")
+                .property("equip-type", "theEquipType")
+                .property(AAIProperties.AAI_URI, uri)
+                .next();
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname2")
+                .property("equip-type", "theEquipType2")
+                .property(AAIProperties.AAI_URI, uri + "2")
+                .next();
+
+        JSONObject expectedResponseBody = new JSONObject();
+        JSONObject pserver1 = new JSONObject()
+                .put("hostname", "theHostname")
+                .put("equip-type", "theEquipType");
+        JSONObject pserver2 = new JSONObject()
+                .put("hostname", "theHostname2")
+                .put("equip-type", "theEquipType2");
+        expectedResponseBody.put("pserver", new JSONArray()
+                .put(pserver1)
+                .put(pserver2));
+
+        uri = "/cloud-infrastructure/pservers";
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri);
+        JSONObject actualResponseBody = new JSONObject(response.getEntity().toString());
+
+        JSONAssert.assertEquals(expectedResponseBody, actualResponseBody, JSONCompareMode.NON_EXTENSIBLE);
+        assertEquals("Expected the pservers to be returned", 200, response.getStatus());
+        verify(validationService, times(1)).validate(any());
+    }
+
+    @Test
+    public void thatPaginatedObjectsCanBeRetrieved() throws UnsupportedEncodingException, AAIException {
+        String uri = "/cloud-infrastructure/pservers/pserver/theHostname";
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname")
+                .property("equip-type", "theEquipType")
+                .property(AAIProperties.AAI_URI, uri)
+                .next();
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname2")
+                .property("equip-type", "theEquipType2")
+                .property(AAIProperties.AAI_URI, uri + "2")
+                .next();
+
+        JSONObject expectedResponseBody = new JSONObject();
+        JSONObject pserver1 = new JSONObject()
+                .put("hostname", "theHostname")
+                .put("equip-type", "theEquipType");
+        expectedResponseBody.put("pserver", new JSONArray()
+                .put(pserver1));
+
+        uri = "/cloud-infrastructure/pservers";
+        QueryOptions queryOptions = QueryOptions.builder().pageable(new Pageable(1, 1)).build();
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, queryOptions);
+        JSONObject actualResponseBody = new JSONObject(response.getEntity().toString());
+
+        assertNull(response.getHeaderString("total-results"));
+        assertEquals(1, actualResponseBody.getJSONArray("pserver").length());
+        assertEquals("Expected the pservers to be returned", 200, response.getStatus());
+        verify(validationService, times(1)).validate(any());
+
+        queryOptions = QueryOptions.builder().pageable(new Pageable(0,5).includeTotalCount()).build();
+        response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, queryOptions);
+        actualResponseBody = new JSONObject(response.getEntity().toString());
+        assertEquals(2, actualResponseBody.getJSONArray("pserver").length());
+    }
+
+    @Test
+    public void thatPagationResultWithTotalCountCanBeRetrieved() throws UnsupportedEncodingException, AAIException {
+        String uri = "/cloud-infrastructure/pservers/pserver/theHostname";
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname")
+                .property("equip-type", "theEquipType")
+                .property(AAIProperties.AAI_URI, uri)
+                .next();
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname2")
+                .property("equip-type", "theEquipType2")
+                .property(AAIProperties.AAI_URI, uri + "2")
+                .next();
+
+        JSONObject expectedResponseBody = new JSONObject();
+        JSONObject pserver1 = new JSONObject()
+                .put("hostname", "theHostname")
+                .put("equip-type", "theEquipType");
+        expectedResponseBody.put("pserver", new JSONArray()
+                .put(pserver1));
+
+        uri = "/cloud-infrastructure/pservers";
+        QueryOptions queryOptions = QueryOptions.builder().pageable(new Pageable(1, 1).includeTotalCount()).build();
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, queryOptions);
+        JSONObject actualResponseBody = new JSONObject(response.getEntity().toString());
+        String totalCount = response.getHeaderString("total-results");
+
+        assertEquals(2, Integer.parseInt(totalCount));
+        assertEquals(1, actualResponseBody.getJSONArray("pserver").length());
+        assertEquals("Expected the pservers to be returned", 200, response.getStatus());
+        verify(validationService, times(1)).validate(any());
+
+        queryOptions = QueryOptions.builder().pageable(new Pageable(0, 2)).build();
+        response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, queryOptions);
+        actualResponseBody = new JSONObject(response.getEntity().toString());
+        assertEquals(2, actualResponseBody.getJSONArray("pserver").length());
+    }
+
+    @Test
+    public void thatSortedObjectsCanBeRetrieved() throws UnsupportedEncodingException, AAIException {
+        String uri = "/cloud-infrastructure/pservers/pserver/theHostname";
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname")
+                .property("equip-type", "theEquipType")
+                .property(AAIProperties.AAI_URI, uri)
+                .next();
+        traversal.addV()
+                .property("aai-node-type", "pserver")
+                .property("hostname", "theHostname2")
+                .property("equip-type", "theEquipType2")
+                .property(AAIProperties.AAI_URI, uri + "2")
+                .next();
+
+        JSONObject expectedResponseBody = new JSONObject();
+        JSONObject pserver1 = new JSONObject()
+                .put("hostname", "theHostname")
+                .put("equip-type", "theEquipType");
+        JSONObject pserver2 = new JSONObject()
+                .put("hostname", "theHostname2")
+                .put("equip-type", "theEquipType2");
+        expectedResponseBody.put("pserver", new JSONArray()
+                .put(pserver1).put(pserver2));
+
+        // ascending
+        uri = "/cloud-infrastructure/pservers";
+        QueryOptions queryOptions = QueryOptions.builder().sort(Sort.builder().property("equip-type").direction(Direction.ASC).build()).build();
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, queryOptions);
+        JSONObject actualResponseBody = new JSONObject(response.getEntity().toString());
+        assertEquals("theEquipType", actualResponseBody.getJSONArray("pserver").getJSONObject(0).getString("equip-type"));
+        assertEquals("theEquipType2", actualResponseBody.getJSONArray("pserver").getJSONObject(1).getString("equip-type"));
+
+        // descending
+        queryOptions = QueryOptions.builder().sort(Sort.builder().property("equip-type").direction(Direction.DESC).build()).build();
+        response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, queryOptions);
+        actualResponseBody = new JSONObject(response.getEntity().toString());
+        assertEquals("theEquipType2", actualResponseBody.getJSONArray("pserver").getJSONObject(0).getString("equip-type"));
+        assertEquals("theEquipType", actualResponseBody.getJSONArray("pserver").getJSONObject(1).getString("equip-type"));
+
+    }
+
+    @Test
     public void thatObjectsCanNotBeFound() throws UnsupportedEncodingException, AAIException {
         String uri = "/cloud-infrastructure/pservers/pserver/junit-test2";
-        String requestBody = "";
 
-        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, requestBody);
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri);
         assertEquals("The pserver is not found", 404, response.getStatus());
     }
 
@@ -536,7 +696,7 @@ public class HttpEntryTest extends AAISetup {
         queryParameters.add("format", "pathed");
         String requestBody = "";
         Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET,
-                "/cloud-infrastructure/pservers", requestBody);
+                "/cloud-infrastructure/pservers");
         queryParameters.remove("format");
 
         String responseEntity = response.getEntity().toString();
@@ -573,8 +733,7 @@ public class HttpEntryTest extends AAISetup {
                 .next();
 
         uri = "/cloud-infrastructure/complexes/complex/related-to-complex/related-to/pservers";
-        String responseBody = "";
-        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, responseBody);
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri);
 
         assertEquals("Expected the response to be successful", 200, response.getStatus());
         assertThat("Related pserver is returned", response.getEntity().toString(),
@@ -604,9 +763,8 @@ public class HttpEntryTest extends AAISetup {
                 .addE("tosca.relationships.HostedOn").from("v2").to("v1")
                 .next();
 
-        String requestBody = "";
         uri = "/network/generic-vnfs/generic-vnf/abstract-generic-vnf/related-to/pservers";
-        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, requestBody);
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri);
         assertThat("Related to pserver is returned.", response.getEntity().toString(),
                 containsString("\"hostname\":\"abstract-pserver\""));
         verify(validationService, times(1)).validate(any());
@@ -644,9 +802,7 @@ public class HttpEntryTest extends AAISetup {
 
         // Get Relationship
         uri = "/cloud-infrastructure/pservers/pserver/related-to-pserver";
-        String requestBody = "";
-        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET_RELATIONSHIP, uri,
-                requestBody);
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET_RELATIONSHIP, uri);
         Relationship[] relationships = mapper.readValue(response.getEntity().toString(), RelationshipWrapper.class)
                 .getRelationships();
 
@@ -691,9 +847,7 @@ public class HttpEntryTest extends AAISetup {
         // Get Relationship
         uri = "/cloud-infrastructure/pservers/pserver/related-to-pserver";
         queryParameters.add("format", "resource");
-        String requestBody = "";
-        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET_RELATIONSHIP, uri,
-                requestBody);
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET_RELATIONSHIP, uri);
 
         JSONObject actualResponseBody = new JSONObject(response.getEntity().toString());
 
@@ -766,10 +920,29 @@ public class HttpEntryTest extends AAISetup {
     }
 
     private Response doRequest(HttpEntry httpEntry, Loader loader, TransactionalGraphEngine dbEngine, HttpMethod method,
-            String uri, String requestBody) throws UnsupportedEncodingException, AAIException {
+    String uri) throws UnsupportedEncodingException, AAIException {
+        return doRequest(httpEntry, loader, dbEngine, method, uri, null, null);
+    }
+
+    private Response doRequest(HttpEntry httpEntry, Loader loader, TransactionalGraphEngine dbEngine, HttpMethod method,
+    String uri, String requestBody) throws UnsupportedEncodingException, AAIException {
+        return doRequest(httpEntry, loader, dbEngine, method, uri, requestBody, null);
+    }
+    private Response doRequest(HttpEntry httpEntry, Loader loader, TransactionalGraphEngine dbEngine, HttpMethod method,
+    String uri, QueryOptions queryOptions) throws UnsupportedEncodingException, AAIException {
+        return doRequest(httpEntry, loader, dbEngine, method, uri, null, queryOptions);
+    }
+
+    private Response doRequest(HttpEntry httpEntry, Loader loader, TransactionalGraphEngine dbEngine, HttpMethod method,
+            String uri, String requestBody, QueryOptions queryOptions) throws UnsupportedEncodingException, AAIException {
         URI uriObject = UriBuilder.fromPath(uri).build();
         QueryParser uriQuery = dbEngine.getQueryBuilder().createQueryFromURI(uriObject);
-        String objType = uriQuery.getResultType();
+        String objType;
+        if (!uriQuery.getContainerType().equals("")) {
+                objType = uriQuery.getContainerType();
+        } else {
+                objType = uriQuery.getResultType();
+        }
         if (uri.endsWith("relationship")) {
             objType = "relationship";
         }
@@ -780,14 +953,15 @@ public class HttpEntryTest extends AAISetup {
             obj = loader.unmarshal(objType, requestBody, org.onap.aai.restcore.MediaType.getEnum("application/json"));
         }
 
-        DBRequest dbRequest = new DBRequest.Builder(method, uriObject, uriQuery, obj, httpHeaders, uriInfo,
-                "JUNIT-TRANSACTION")
-                .rawRequestContent(requestBody).build();
+        DBRequest.Builder builder = new DBRequest.Builder(method, uriObject, uriQuery, obj, httpHeaders, uriInfo,
+                "JUNIT-TRANSACTION");
+        DBRequest dbRequest = requestBody != null
+                ? builder.rawRequestContent(requestBody).build()
+                : builder.build();
 
-        List<DBRequest> dbRequestList = new ArrayList<>();
-        dbRequestList.add(dbRequest);
+        List<DBRequest> dbRequestList = Collections.singletonList(dbRequest);
 
-        Pair<Boolean, List<Pair<URI, Response>>> responsesTuple = httpEntry.process(dbRequestList, "JUNIT");
+        Pair<Boolean, List<Pair<URI, Response>>> responsesTuple = httpEntry.process(dbRequestList, "JUNIT", Collections.emptySet(), true, queryOptions);
         return responsesTuple.getValue1().get(0).getValue1();
     }
 
@@ -844,15 +1018,11 @@ public class HttpEntryTest extends AAISetup {
                 .property("equip-type", "theEquipType")
                 .property(AAIProperties.AAI_URI, uri)
                 .next();
-        String requestBody = new JSONObject()
-                .put("hostname", "theHostname")
-                .put("equip-type", "theEquipType")
-                .toString();
 
         JSONObject expectedResponseBody = new JSONObject()
                 .put("hostname", "theHostname")
                 .put("equip-type", "theEquipType");
-        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri, requestBody);
+        Response response = doRequest(traversalHttpEntry, loader, dbEngine, HttpMethod.GET, uri);
         JSONObject actualResponseBody = new JSONObject(response.getEntity().toString());
 
         JSONAssert.assertEquals(expectedResponseBody, actualResponseBody, JSONCompareMode.NON_EXTENSIBLE);