From 36b81b27703d298f016180a71c85113d5c90c6b1 Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Thu, 17 Aug 2023 14:58:30 +0100 Subject: Enable hibernate write batching - Implement fragment ID sequence generator with allocation size of 100 - Enable write batching for insert and update, with batch size of 100 - Write performance is over 2x faster Issue-ID: CPS-1795 Signed-off-by: danielhanrahan Change-Id: I7ebd0f834abb57c593603e2cd3fafc7e3425ab1e --- .../performance/base/CpsPerfTestBase.groovy | 6 ++--- .../cps/CpsDataServiceLimitsPerfTest.groovy | 6 ++--- .../performance/cps/DeletePerfTest.groovy | 4 ++-- .../performance/cps/UpdatePerfTest.groovy | 26 +++++++++++----------- .../performance/cps/WritePerfTest.groovy | 12 +++++----- .../src/test/resources/application.yml | 14 +++++++----- .../test/resources/liquibase/test-changelog.yaml | 9 ++++++-- 7 files changed, 42 insertions(+), 35 deletions(-) (limited to 'integration-test') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy index 816aeff71..920f40749 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2024 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -67,7 +67,7 @@ class CpsPerfTestBase extends PerfTestBase { addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data) resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 100, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) } def generateOpenRoadData(numberOfNodes) { @@ -87,7 +87,7 @@ class CpsPerfTestBase extends PerfTestBase { then: 'memory used is within #peakMemoryUsage' assert resourceMeter.getTotalMemoryUsageInMB() <= 30 and: 'all data is read within expected time' - recordAndAssertResourceUsage("Warming database", 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Warming database", 100, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy index 4d4d6125c..0c1e1f5d8 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2024 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -46,8 +46,8 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { } resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'the operation completes within 25 seconds' - recordAndAssertResourceUsage("Creating 33,000 books", 25, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) + then: 'the operation completes within 12 seconds' + recordAndAssertResourceUsage("Creating 33,000 books", 12, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) } def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy index e7bfabefb..2efbf7d3f 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2024 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'setup duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete test setup', 200, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete test setup', 100, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 container nodes'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index b3030b1c6..69f64773f 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -79,12 +79,12 @@ class UpdatePerfTest extends CpsPerfTestBase { memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: scenario | totalNodes | startId | changeLeaves || timeLimit | memoryLimit - 'Replace 0 nodes with 100' | 100 | 1 | false || 7 | 250 - 'Replace 100 using same data' | 100 | 1 | false || 3 | 250 - 'Replace 100 with new leaf values' | 100 | 1 | true || 3 | 250 - 'Replace 100 with 100 new nodes' | 100 | 101 | false || 12 | 300 - 'Replace 50 existing and 50 new' | 100 | 151 | true || 8 | 250 - 'Replace 100 nodes with 0' | 0 | 1 | false || 5 | 250 + 'Replace 0 nodes with 100' | 100 | 1 | false || 2.5 | 200 + 'Replace 100 using same data' | 100 | 1 | false || 3.0 | 200 + 'Replace 100 with new leaf values' | 100 | 1 | true || 3.0 | 200 + 'Replace 100 with 100 new nodes' | 100 | 101 | false || 6.0 | 200 + 'Replace 50 existing and 50 new' | 100 | 151 | true || 4.5 | 200 + 'Replace 100 nodes with 0' | 0 | 1 | false || 3.0 | 200 } def 'Replace list content: #scenario.'() { @@ -105,12 +105,12 @@ class UpdatePerfTest extends CpsPerfTestBase { memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: scenario | totalNodes | startId | changeLeaves || timeLimit | memoryLimit - 'Replace list of 0 with 100' | 100 | 1 | false || 7 | 250 - 'Replace list of 100 using same data' | 100 | 1 | false || 3 | 250 - 'Replace list of 100 with new leaf values' | 100 | 1 | true || 3 | 250 - 'Replace list with 100 new nodes' | 100 | 101 | false || 12 | 300 - 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 8 | 250 - 'Replace list of 100 nodes with 1' | 1 | 1 | false || 5 | 250 + 'Replace list of 0 with 100' | 100 | 1 | false || 2.5 | 200 + 'Replace list of 100 using same data' | 100 | 1 | false || 3.0 | 200 + 'Replace list of 100 with new leaf values' | 100 | 1 | true || 3.0 | 200 + 'Replace list with 100 new nodes' | 100 | 101 | false || 6.0 | 200 + 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 4.5 | 200 + 'Replace list of 100 nodes with 1' | 1 | 1 | false || 3.0 | 200 } def 'Update leaves for 100 data nodes.'() { @@ -127,7 +127,7 @@ class UpdatePerfTest extends CpsPerfTestBase { assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]') and: 'update completes within expected time and memory used is within limit' recordAndAssertResourceUsage('Update leaves for 100 data nodes', - 0.5, resourceMeter.getTotalTimeInSeconds(), + 0.4, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB()) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index a5669baf6..10329a558 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -45,10 +45,10 @@ class WritePerfTest extends CpsPerfTestBase { cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalNodes || expectedDuration | memoryLimit - 50 || 4 | 100 - 100 || 7 | 200 - 200 || 14 | 400 - 400 || 28 | 500 + 50 || 2 | 100 + 100 || 4 | 200 + 200 || 7 | 400 + 400 || 14 | 500 } def 'Writing bookstore data has exponential time.'() { @@ -71,8 +71,8 @@ class WritePerfTest extends CpsPerfTestBase { cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalBooks || expectedDuration | memoryLimit - 800 || 1 | 50 - 1600 || 2 | 100 + 800 || 0.5 | 50 + 1600 || 1.5 | 100 3200 || 6 | 150 6400 || 18 | 200 } diff --git a/integration-test/src/test/resources/application.yml b/integration-test/src/test/resources/application.yml index 0aefac83f..55560aa62 100644 --- a/integration-test/src/test/resources/application.yml +++ b/integration-test/src/test/resources/application.yml @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (C) 2023 Nordix Foundation. +# Copyright (C) 2023-2024 Nordix Foundation. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,11 +20,13 @@ spring: ddl-auto: create show-sql: false properties: - hibernate: - enable_lazy_load_no_trans: true - dialect: org.hibernate.dialect.PostgreSQLDialect - format_sql: true - show_sql: false + hibernate.enable_lazy_load_no_trans: true + hibernate.dialect: org.hibernate.dialect.PostgreSQLDialect + hibernate.format_sql: true + hibernate.show_sql: false + # Please ensure these values match those used in cps-application/src/main/resources/application.yml + hibernate.id.new_generator_mappings: true + hibernate.jdbc.batch_size: 100 datasource: url: ${DB_URL} diff --git a/integration-test/src/test/resources/liquibase/test-changelog.yaml b/integration-test/src/test/resources/liquibase/test-changelog.yaml index 3f1233145..ba639ede0 100644 --- a/integration-test/src/test/resources/liquibase/test-changelog.yaml +++ b/integration-test/src/test/resources/liquibase/test-changelog.yaml @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2023 Nordix Foundation. +# Copyright (c) 2023-2024 Nordix Foundation. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -100,7 +100,7 @@ databaseChangeLog: - createTable: columns: - column: - autoIncrement: true + autoIncrement: false constraints: nullable: false primaryKey: true @@ -127,6 +127,11 @@ databaseChangeLog: name: schema_node_id type: INTEGER tableName: fragment + - createSequence: + dataType: BIGINT + incrementBy: 100 + sequenceName: fragment_id_seq + startValue: 100 - changeSet: id: 1-5 author: cps -- cgit 1.2.3-korg