From 796c4dcf56a1aafc58773bedc82ddef5242a108c Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Thu, 13 Apr 2023 21:01:58 +0100 Subject: Remove 32K limit from queries with collection parameters SQL queries taking collection parameters currently create a seperate query parameter for each collection element. There is a limit of around 2^15 (32,768) query parameters. Postgres DB natively supports array types, in which the whole array is transmitted in binary as a single parameter. Changing queries to use arrays removes the 32K limit on queries. - Add support for Postgres arrays to queries - Change repository methods to use arrays, and provide overloaded versions taking collection parameters - Update tests to reflect 32K limit being lifted Issue-ID: CPS-1573 Signed-off-by: danielhanrahan Change-Id: I64f2aeaedbe54bfe12e3079cba0f2216759142c3 --- .../performance/cps/CpsAdminServiceLimits.groovy | 13 ++++++------- .../performance/cps/CpsDataServiceLimits.groovy | 21 ++++++++++----------- 2 files changed, 16 insertions(+), 18 deletions(-) (limited to 'integration-test/src/test/groovy/org') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsAdminServiceLimits.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsAdminServiceLimits.groovy index 7875caec35..0034af453b 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsAdminServiceLimits.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsAdminServiceLimits.groovy @@ -22,7 +22,6 @@ package org.onap.cps.integration.performance.cps import org.onap.cps.api.CpsAdminService import org.onap.cps.integration.performance.base.CpsPerfTestBase -import org.springframework.dao.DataAccessResourceFailureException class CpsAdminServiceLimits extends CpsPerfTestBase { @@ -32,20 +31,20 @@ class CpsAdminServiceLimits extends CpsPerfTestBase { def 'Get anchors from multiple schema set names limit exceeded: 32,766 (~ 2^15) schema set names.'() { given: 'more than 32,766 schema set names' - def schemaSetNames = (0..32_766).collect { "size-of-this-name-does-not-matter-for-limit-" + it } + def schemaSetNames = (0..40_000).collect { "size-of-this-name-does-not-matter-for-limit-" + it } when: 'single get is executed to get all the anchors' objectUnderTest.getAnchors(CPS_PERFORMANCE_TEST_DATASPACE, schemaSetNames) - then: 'a database exception is thrown' - thrown(DataAccessResourceFailureException.class) + then: 'a database exception is not thrown' + noExceptionThrown() } def 'Querying anchor names limit exceeded: 32,766 (~ 2^15) modules.'() { given: 'more than 32,766 module names' - def moduleNames = (0..32_766).collect { "size-of-this-name-does-not-matter-for-limit-" + it } + def moduleNames = (0..40_000).collect { "size-of-this-name-does-not-matter-for-limit-" + it } when: 'single query is executed to get all the anchors' objectUnderTest.queryAnchorNames(CPS_PERFORMANCE_TEST_DATASPACE, moduleNames) - then: 'a database exception is thrown' - thrown(DataAccessResourceFailureException.class) + then: 'a database exception is not thrown' + noExceptionThrown() } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimits.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimits.groovy index 2df910194d..1579470eab 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimits.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimits.groovy @@ -23,8 +23,7 @@ package org.onap.cps.integration.performance.cps import java.time.OffsetDateTime import org.onap.cps.api.CpsDataService import org.onap.cps.integration.performance.base.CpsPerfTestBase -import org.springframework.dao.DataAccessResourceFailureException -import org.springframework.transaction.TransactionSystemException +import org.onap.cps.spi.exceptions.DataNodeNotFoundException import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS @@ -36,29 +35,29 @@ class CpsDataServiceLimits extends CpsPerfTestBase { def 'Multiple get limit exceeded: 32,764 (~ 2^15) xpaths.'() { given: 'more than 32,764 xpaths' - def xpaths = (0..32_764).collect { "/size/of/this/path/does/not/matter/for/limit[@id='" + it + "']" } + def xpaths = (0..40_000).collect { "/size/of/this/path/does/not/matter/for/limit[@id='" + it + "']" } when: 'single operation is executed to get all datanodes with given xpaths' objectUnderTest.getDataNodesForMultipleXpaths(CPS_PERFORMANCE_TEST_DATASPACE, 'bookstore1', xpaths, INCLUDE_ALL_DESCENDANTS) - then: 'a database exception is thrown' - thrown(DataAccessResourceFailureException.class) + then: 'a database exception is not thrown' + noExceptionThrown() } def 'Delete multiple datanodes limit exceeded: 32,767 (~ 2^15) xpaths.'() { given: 'more than 32,767 xpaths' - def xpaths = (0..32_767).collect { "/size/of/this/path/does/not/matter/for/limit[@id='" + it + "']" } + def xpaths = (0..40_000).collect { "/size/of/this/path/does/not/matter/for/limit[@id='" + it + "']" } when: 'single operation is executed to delete all datanodes with given xpaths' objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'bookstore1', xpaths, OffsetDateTime.now()) - then: 'a database exception is thrown' - thrown(TransactionSystemException.class) + then: 'a database exception is not thrown (but a CPS DataNodeNotFoundException is thrown)' + thrown(DataNodeNotFoundException.class) } def 'Delete datanodes from multiple anchors limit exceeded: 32,766 (~ 2^15) anchors.'() { given: 'more than 32,766 anchor names' - def anchorNames = (0..32_766).collect { "size-of-this-name-does-not-matter-for-limit-" + it } + def anchorNames = (0..40_000).collect { "size-of-this-name-does-not-matter-for-limit-" + it } when: 'single operation is executed to delete all datanodes in given anchors' objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, anchorNames, OffsetDateTime.now()) - then: 'a database exception is thrown' - thrown(DataAccessResourceFailureException.class) + then: 'a database exception is not thrown' + noExceptionThrown() } } -- cgit 1.2.3-korg