From 7a697663603a0d3a955076b1a36ce674e62d3ab7 Mon Sep 17 00:00:00 2001 From: Jozsef Csongvai Date: Wed, 6 Oct 2021 11:47:23 -0400 Subject: Add missing code for Metrics and PV/PVC elimination Earlier patch introducing MeterRegistry to AbstractComponentFunction was not initializing the lateinit property and causing NPE. Also add additional code to handle compatibility issues after the introduction of PV/PVC elminiation for CommandExecutor. This allows blueprintsprocessor to communicate with earlier versions of command- executor which still use the shared pvc. Issue-ID: CCSDK-3471 Change-Id: I84a04601c4fe09c5f3a06664ce877800a30531f1 Signed-off-by: Jozsef Csongvai --- .../python/executor/ComponentRemotePythonExecutor.kt | 3 --- .../configs/api/ResourceConfigSnapshotControllerTest.kt | 5 +++++ .../resource/api/TemplateControllerTest.kt | 5 +++++ .../services/workflow/NodeTemplateExecutionService.kt | 4 +++- .../workflow/BluePrintWorkflowExecutionServiceImplTest.kt | 4 ++++ .../services/workflow/BlueprintServiceLogicTest.kt | 4 ++++ .../services/workflow/DGWorkflowExecutionServiceTest.kt | 4 ++++ .../workflow/ImperativeWorkflowExecutionServiceTest.kt | 2 +- .../services/workflow/NodeTemplateExecutionServiceTest.kt | 7 ++++++- .../src/main/python/command_executor_handler.py | 15 ++++++++++++--- ms/command-executor/src/main/python/utils.py | 11 ++++++----- 11 files changed, 50 insertions(+), 14 deletions(-) diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt index a15e2f1ab..00f8e5a19 100644 --- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt +++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt @@ -138,9 +138,6 @@ open class ComponentRemotePythonExecutor( // component level timeout should be => env_prep_timeout + execution_timeout val timeout = implementation.timeout - // NOTE: this was reverted back to absolute path for SR7 compatibility. - // CMD-EXEC SR10 onwards will look for absence of blueprint UUID in the absolute path. - // If such request is found, UUID will be appended. var scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath) if (args != null && args.isNotEmpty()) { scriptCommand = scriptCommand.plus(" ").plus(args) diff --git a/ms/blueprintsprocessor/modules/inbounds/configs-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/configs/api/ResourceConfigSnapshotControllerTest.kt b/ms/blueprintsprocessor/modules/inbounds/configs-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/configs/api/ResourceConfigSnapshotControllerTest.kt index 2fde05504..d119f835a 100644 --- a/ms/blueprintsprocessor/modules/inbounds/configs-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/configs/api/ResourceConfigSnapshotControllerTest.kt +++ b/ms/blueprintsprocessor/modules/inbounds/configs-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/configs/api/ResourceConfigSnapshotControllerTest.kt @@ -16,6 +16,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.configs.api +import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.runBlocking import org.junit.Test import org.junit.runner.RunWith @@ -24,6 +25,7 @@ import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintCatalogS import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest +import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.context.annotation.ComponentScan import org.springframework.http.MediaType import org.springframework.test.context.ContextConfiguration @@ -46,6 +48,9 @@ class ResourceConfigSnapshotControllerTest { @Autowired lateinit var webTestClient: WebTestClient + @MockBean + lateinit var meterRegistry: MeterRegistry + val resourceId = "fcaa6ac3ff08" val resourceType = "PNF" val snapshotData = "PAYLOAD DATA" diff --git a/ms/blueprintsprocessor/modules/inbounds/resource-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/resource/api/TemplateControllerTest.kt b/ms/blueprintsprocessor/modules/inbounds/resource-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/resource/api/TemplateControllerTest.kt index 945a30c64..7586032a0 100644 --- a/ms/blueprintsprocessor/modules/inbounds/resource-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/resource/api/TemplateControllerTest.kt +++ b/ms/blueprintsprocessor/modules/inbounds/resource-api/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/resource/api/TemplateControllerTest.kt @@ -16,6 +16,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.resource.api +import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.runBlocking import org.junit.Test import org.junit.runner.RunWith @@ -25,6 +26,7 @@ import org.onap.ccsdk.cds.controllerblueprints.core.interfaces.BluePrintCatalogS import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest +import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.context.annotation.ComponentScan import org.springframework.http.MediaType import org.springframework.test.context.ContextConfiguration @@ -51,6 +53,9 @@ class TemplateControllerTest { @Autowired lateinit var webTestClient: WebTestClient + @MockBean + lateinit var meterRegistry: MeterRegistry + var resolutionKey = "7cafa9f3-bbc8-49ec-8f25-fcaa6ac3ff08" val blueprintName = "baseconfiguration" val blueprintVersion = "1.0.0" diff --git a/ms/blueprintsprocessor/modules/services/workflow-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionService.kt b/ms/blueprintsprocessor/modules/services/workflow-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionService.kt index 88799d102..53c327a5b 100644 --- a/ms/blueprintsprocessor/modules/services/workflow-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionService.kt +++ b/ms/blueprintsprocessor/modules/services/workflow-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionService.kt @@ -17,6 +17,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.services.workflow import com.fasterxml.jackson.databind.JsonNode +import io.micrometer.core.instrument.MeterRegistry import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceOutput import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.StepData @@ -31,7 +32,7 @@ import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @Service -open class NodeTemplateExecutionService(private val bluePrintClusterService: BluePrintClusterService) { +open class NodeTemplateExecutionService(private val bluePrintClusterService: BluePrintClusterService, private val meterRegistry: MeterRegistry) { private val log = LoggerFactory.getLogger(NodeTemplateExecutionService::class.java)!! @@ -69,6 +70,7 @@ open class NodeTemplateExecutionService(private val bluePrintClusterService: Blu plugin.bluePrintClusterService = bluePrintClusterService plugin.stepName = stepName plugin.nodeTemplateName = nodeTemplateName + plugin.meterRegistry = meterRegistry // Parent request shouldn't tamper, so need to clone the request and send to the actual component. val clonedExecutionServiceInput = ExecutionServiceInput().apply { diff --git a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BluePrintWorkflowExecutionServiceImplTest.kt b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BluePrintWorkflowExecutionServiceImplTest.kt index c28e36acc..0a58a080f 100644 --- a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BluePrintWorkflowExecutionServiceImplTest.kt +++ b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BluePrintWorkflowExecutionServiceImplTest.kt @@ -18,6 +18,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.services.workflow import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode +import io.micrometer.core.instrument.MeterRegistry import io.mockk.every import io.mockk.mockk import io.mockk.mockkObject @@ -63,6 +64,9 @@ class BluePrintWorkflowExecutionServiceImplTest { @MockBean lateinit var bluePrintClusterService: BluePrintClusterService + @MockBean + lateinit var meterRegistry: MeterRegistry + @Before fun init() { mockkObject(BluePrintDependencyService) diff --git a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BlueprintServiceLogicTest.kt b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BlueprintServiceLogicTest.kt index d391050a6..ce6fcf0ca 100644 --- a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BlueprintServiceLogicTest.kt +++ b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/BlueprintServiceLogicTest.kt @@ -17,6 +17,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.services.workflow +import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.runBlocking import org.junit.Before import org.junit.Test @@ -51,6 +52,9 @@ class BlueprintServiceLogicTest { @MockBean lateinit var bluePrintClusterService: BluePrintClusterService + @MockBean + lateinit var meterRegistry: MeterRegistry + @Before fun init() { BluePrintDependencyService.inject(applicationContext) diff --git a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/DGWorkflowExecutionServiceTest.kt b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/DGWorkflowExecutionServiceTest.kt index cc1bfee7f..887671384 100644 --- a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/DGWorkflowExecutionServiceTest.kt +++ b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/DGWorkflowExecutionServiceTest.kt @@ -17,6 +17,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.services.workflow +import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.runBlocking import org.junit.Before import org.junit.Test @@ -49,6 +50,9 @@ class DGWorkflowExecutionServiceTest { @MockBean lateinit var bluePrintClusterService: BluePrintClusterService + @MockBean + lateinit var meterRegistry: MeterRegistry + @Before fun init() { BluePrintDependencyService.inject(applicationContext) diff --git a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/ImperativeWorkflowExecutionServiceTest.kt b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/ImperativeWorkflowExecutionServiceTest.kt index 2367422a7..d63194153 100644 --- a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/ImperativeWorkflowExecutionServiceTest.kt +++ b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/ImperativeWorkflowExecutionServiceTest.kt @@ -120,7 +120,7 @@ class ImperativeWorkflowExecutionServiceTest { ExecutionServiceInput::class.java )!! - val imperativeWorkflowExecutionService = ImperativeWorkflowExecutionService(NodeTemplateExecutionService(mockk())) + val imperativeWorkflowExecutionService = ImperativeWorkflowExecutionService(NodeTemplateExecutionService(mockk(), mockk())) val output = imperativeWorkflowExecutionService .executeBluePrintWorkflow(bluePrintRuntimeService, executionServiceInput, hashMapOf()) assertNotNull(output, "failed to get imperative workflow output") diff --git a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionServiceTest.kt b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionServiceTest.kt index 783939880..14eb24e10 100644 --- a/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionServiceTest.kt +++ b/ms/blueprintsprocessor/modules/services/workflow-service/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/workflow/NodeTemplateExecutionServiceTest.kt @@ -15,7 +15,9 @@ */ package org.onap.ccsdk.cds.blueprintsprocessor.services.workflow +import io.micrometer.core.instrument.MeterRegistry import io.mockk.every +import io.mockk.mockk import io.mockk.mockkObject import io.mockk.unmockkAll import kotlinx.coroutines.runBlocking @@ -44,6 +46,9 @@ class NodeTemplateExecutionServiceTest { @MockBean lateinit var bluePrintClusterService: BluePrintClusterService + @MockBean + lateinit var meterRegistry: MeterRegistry + @Before fun init() { mockkObject(BluePrintDependencyService) @@ -75,7 +80,7 @@ class NodeTemplateExecutionServiceTest { val stepName = bluePrintRuntimeService.bluePrintContext() .workflowSteps("resource-assignment").keys.first() val nodeTemplate = "resource-assignment" - val nodeTemplateExecutionService = NodeTemplateExecutionService(bluePrintClusterService) + val nodeTemplateExecutionService = NodeTemplateExecutionService(bluePrintClusterService, mockk()) val executionServiceOutput = nodeTemplateExecutionService .executeNodeTemplate(bluePrintRuntimeService, stepName, nodeTemplate, executionServiceInput) diff --git a/ms/command-executor/src/main/python/command_executor_handler.py b/ms/command-executor/src/main/python/command_executor_handler.py index a862559cf..6a301574b 100644 --- a/ms/command-executor/src/main/python/command_executor_handler.py +++ b/ms/command-executor/src/main/python/command_executor_handler.py @@ -49,9 +49,12 @@ class CommandExecutorHandler(): self.uuid = utils.get_blueprint_uuid(request) self.request_id = utils.get_blueprint_requestid(request) self.sub_request_id = utils.get_blueprint_subRequestId(request) + self.blueprint_name_version = utils.blueprint_name_version(request) # for legacy support self.blueprint_name_version_uuid = utils.blueprint_name_version_uuid(request) self.execution_timeout = utils.get_blueprint_timeout(request) # onap/blueprints/deploy will be ephemeral now + # if the command matches “/opt/app/onap/blueprints/deploy/$cba_name/$cba_version/stuff_that_is_not_cba_uuid/” + # then prepend the $cba_uuid before “stuff_that_is_not_cba_uuid” self.blueprint_dir = self.BLUEPRINTS_DEPLOY_DIR + self.blueprint_name_version_uuid self.blueprint_tosca_meta_file = self.blueprint_dir + '/' + self.TOSCA_META_FILE self.extra = utils.getExtraLogData(request) @@ -243,11 +246,17 @@ class CommandExecutorHandler(): if request.properties is not None and len(request.properties) > 0: properties = " " + re.escape(MessageToJson(request.properties)) + # compatibility hack + # check if the path for the request.command does not contain UUID, then add it after cba_name/cba_version path. + updated_request_command = request.command + if self.blueprint_name_version in updated_request_command and self.blueprint_name_version_uuid not in updated_request_command: + updated_request_command = updated_request_command.replace(self.blueprint_name_version, self.blueprint_name_version_uuid) + ### TODO: replace with os.environ['VIRTUAL_ENV']? - if "ansible-playbook" in request.command: - cmd = cmd + "; " + request.command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'" + if "ansible-playbook" in updated_request_command: + cmd = cmd + "; " + updated_request_command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'" else: - cmd = cmd + "; " + request.command + properties + cmd = cmd + "; " + updated_request_command + properties ### extract the original header request into sys-env variables ### OriginatorID diff --git a/ms/command-executor/src/main/python/utils.py b/ms/command-executor/src/main/python/utils.py index e7924a936..d59f8cc47 100644 --- a/ms/command-executor/src/main/python/utils.py +++ b/ms/command-executor/src/main/python/utils.py @@ -26,12 +26,13 @@ RESULTS_LOG_KEY = "results_log" REUPLOAD_CBA_KEY = "reupload_cba" RESPONSE_MAX_SIZE = 4 * 1024 * 1024 # 4Mb - +# part of cba_name/version/uuid path def blueprint_name_version_uuid(request): - blueprint_name = request.identifiers.blueprintName - blueprint_version = request.identifiers.blueprintVersion - blueprint_uuid = request.identifiers.blueprintUUID - return blueprint_name + '/' + blueprint_version + '/' + blueprint_uuid + return get_blueprint_name(request) + '/' + get_blueprint_version(request) + '/' + get_blueprint_uuid(request) + +# return blueprint_name and version part of the path (needed for legacy cmd-exec support +def blueprint_name_version(request): + return get_blueprint_name(request) + '/' + get_blueprint_version(request) def get_blueprint_name(request): return request.identifiers.blueprintName -- cgit 1.2.3-korg