summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSerge Simard <serge@agilitae.com>2019-09-16 16:55:59 -0400
committerSerge Simard <serge@agilitae.com>2019-09-17 14:15:23 -0400
commit828311059c00f50dc76e3fc370968ff377dfd2de (patch)
treed18e910b81952ddde545d9c1c5d73852d3c18bcd
parent799f04c129b09d29ebd1b87092dea699a4174d11 (diff)
Updated Remote Ansible component to retrieve all artifacts and logs produced by job/workflow.
Issue-ID: CCSDK-1741 Signed-off-by: Serge Simard <serge@agilitae.com> Change-Id: I83e8bac79977a08d16d5356c0d21af1d7897cfc0 Signed-off-by: Serge Simard <serge@agilitae.com>
-rw-r--r--components/model-catalog/definition-type/starter-type/node_type/component-remote-ansible-executor.json4
-rw-r--r--ms/blueprintsprocessor/functions/ansible-awx-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutor.kt97
-rw-r--r--ms/blueprintsprocessor/functions/ansible-awx-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutorTest.kt2
3 files changed, 70 insertions, 33 deletions
diff --git a/components/model-catalog/definition-type/starter-type/node_type/component-remote-ansible-executor.json b/components/model-catalog/definition-type/starter-type/node_type/component-remote-ansible-executor.json
index f5d9d3f7a..b14d5b28d 100644
--- a/components/model-catalog/definition-type/starter-type/node_type/component-remote-ansible-executor.json
+++ b/components/model-catalog/definition-type/starter-type/node_type/component-remote-ansible-executor.json
@@ -10,7 +10,7 @@
"required": true,
"type": "string"
},
- "response-data": {
+ "ansible-artifacts": {
"required": false,
"type": "json"
}
@@ -27,7 +27,7 @@
"inputs": {
"job-template-name": {
"description": "Primary key or name of the job template to launch new job.",
- "required": true,
+ "required": false,
"type": "string"
},
"workflow-job-template-id": {
diff --git a/ms/blueprintsprocessor/functions/ansible-awx-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutor.kt b/ms/blueprintsprocessor/functions/ansible-awx-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutor.kt
index 743aa714b..25bb3c938 100644
--- a/ms/blueprintsprocessor/functions/ansible-awx-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutor.kt
+++ b/ms/blueprintsprocessor/functions/ansible-awx-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutor.kt
@@ -56,8 +56,9 @@ open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertySe
private val HTTP_SUCCESS = 200..202
private val GET = HttpMethod.GET.name
private val POST = HttpMethod.POST.name
+ private val plainTextHeaders = mapOf("Accept" to "text/plain")
- var checkDelay: Long = 1_000
+ var checkDelay: Long = 15_000
companion object {
private val log = LoggerFactory.getLogger(ComponentRemoteAnsibleExecutor::class.java)
@@ -73,6 +74,7 @@ open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertySe
const val INPUT_SKIP_TAGS = "skip-tags"
// output fields names (and values) populated by this executor; aligned with job details status field values.
+ const val ATTRIBUTE_EXEC_CMD_ARTIFACTS = "ansible-artifacts"
const val ATTRIBUTE_EXEC_CMD_STATUS = "ansible-command-status"
const val ATTRIBUTE_EXEC_CMD_LOG = "ansible-command-logs"
const val ATTRIBUTE_EXEC_CMD_STATUS_ERROR = "error"
@@ -161,13 +163,14 @@ open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertySe
*/
private fun runJobTemplateOnAWX(awxClient: BlueprintWebClientService, job_template_name: String?, jtId: String,
workflowPrefix : String) {
- setNodeOutputProperties("preparing".asJsonPrimitive(), "".asJsonPrimitive())
+ setNodeOutputProperties("preparing".asJsonPrimitive(), "".asJsonPrimitive(), "".asJsonPrimitive())
// Get Job Template requirements
var response = awxClient.exchangeResource(GET, "/api/v2/${workflowPrefix}job_templates/${jtId}/launch/", "")
// FIXME: handle non-successful SC
val jtLaunchReqs: JsonNode = mapper.readTree(response.body)
- val payload = prepareLaunchPayload(awxClient, jtLaunchReqs, workflowPrefix.isBlank())
+ val payload = prepareLaunchPayload(awxClient, jtLaunchReqs, workflowPrefix.isNotBlank())
+
log.info("Running job with $payload, for requestId $processId.")
// Launch the job for the targeted template
@@ -197,10 +200,8 @@ open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertySe
log.info("Execution of job template $job_template_name in job #$jobId finished with status ($jobStatus) for requestId $processId")
- // Get workflow/job execution results
- val collectedOutput = extractJobRunResponse(awxClient, jobId, workflowPrefix)
+ populateJobRunResponse(awxClient, jobId, workflowPrefix, jobStatus)
- setNodeOutputProperties(jobStatus.asJsonPrimitive(), collectedOutput.asJsonPrimitive())
} else {
// The job template requirements were not fulfilled with the values passed in. The message below will
// provide more information via the response, like the ignored_fields, or variables_needed_to_start,
@@ -213,33 +214,67 @@ open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertySe
}
/**
- * Extracts the response from either a job stdout call OR collects the workflow run output
+ * Extracts the output from either a job stdout call OR collects the workflow run output, as well as the artifacts
+ * and populate the component corresponding output properties
+ */
+ private fun populateJobRunResponse(awxClient: BlueprintWebClientService, jobId: String, workflowPrefix: String,
+ jobStatus: String) {
+
+ val collectedResponses = StringBuilder(4096)
+ val artifacts: MutableMap<String, JsonNode> = mutableMapOf()
+
+ collectJobIdsRelatedToJobRun(awxClient, jobId, workflowPrefix).forEach { aJobId ->
+
+ // Collect the response text from the corresponding jobIds
+ var response = awxClient.exchangeResource(GET, "/api/v2/jobs/${aJobId}/stdout/?format=txt", "", plainTextHeaders)
+ if (response.status in HTTP_SUCCESS) {
+ val jobOutput = response.body
+ collectedResponses
+ .append("Output for Job $aJobId :" + System.lineSeparator())
+ .append(jobOutput)
+ .append(System.lineSeparator())
+ log.info("Response for job ${aJobId}: \n ${jobOutput} \n")
+ } else {
+ log.warn("Could not gather response for job ${aJobId}. Status=${response.status}")
+ }
+
+ // Collect artifacts variables from each job and gather them up in one json node
+ response = awxClient.exchangeResource(GET, "/api/v2/jobs/${aJobId}/", "")
+ if (response.status in HTTP_SUCCESS) {
+ val jobArtifacts = mapper.readTree(response.body).at("/artifacts")
+ if (jobArtifacts != null) {
+ artifacts.putAll(jobArtifacts.rootFieldsToMap())
+ }
+ }
+ }
+
+ log.info("Artifacts for job ${jobId}: \n $artifacts \n")
+
+ setNodeOutputProperties(jobStatus.asJsonPrimitive(), collectedResponses.toString().asJsonPrimitive(), artifacts.asJsonNode())
+ }
+
+ /**
+ * List all the job Ids for a give workflow, i.e. sub jobs, or the jobId if not a workflow instance
*/
- private fun extractJobRunResponse(awxClient: BlueprintWebClientService, jobId: String, workflowPrefix: String): String {
+ private fun collectJobIdsRelatedToJobRun(awxClient: BlueprintWebClientService, jobId: String, workflowPrefix: String): Array<String> {
+
+ var jobIds: Array<String>
- // First, collect all job ID from either the job template run or the workflow nodes that ran
- var jobIds : Array<String>
- var collectedResponses = StringBuilder()
if (workflowPrefix.isNotEmpty()) {
var response = awxClient.exchangeResource(GET, "/api/v2/${workflowPrefix}jobs/${jobId}/workflow_nodes/", "")
val jobDetails = mapper.readTree(response.body).at("/results")
+
+ // gather up job Id of all actual job nodes that ran during the workflow
jobIds = emptyArray()
for (jobDetail in jobDetails.elements()) {
- jobIds = jobIds.plus( jobDetail.at("/summary_fields/job/id").asText() )
+ if (jobDetail.at("/do_not_run").asText() == "false") {
+ jobIds = jobIds.plus(jobDetail.at("/summary_fields/job/id").asText())
+ }
}
} else {
jobIds = arrayOf(jobId)
}
-
- // Then collect the response text from the corresponding jobIds
- val plainTextHeaders = mutableMapOf<String, String>()
- plainTextHeaders["Content-Type"] = "text/plain ;utf-8"
- for (aJobId in jobIds) {
- var response = awxClient.exchangeResource(GET, "/api/v2/jobs/${aJobId}/stdout/?format=txt", "", plainTextHeaders)
- collectedResponses.append("Output for job ${aJobId}:")
- collectedResponses.append(response.body)
- }
- return collectedResponses.toString()
+ return jobIds
}
/**
@@ -282,10 +317,9 @@ open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertySe
}
payload.set(INPUT_INVENTORY, inventoryKeyId)
}
- val askVariablesOnLaunch = jtLaunchReqs.at("/ask_variables_on_launch").asBoolean()
- if (askVariablesOnLaunch) {
+
payload.set("extra_vars", extraArgs)
- }
+
return payload.asJsonString(false)
}
@@ -317,20 +351,23 @@ open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertySe
/**
* Utility function to set the output properties of the executor node
*/
- private fun setNodeOutputProperties(status: JsonNode, message: JsonNode) {
+ private fun setNodeOutputProperties(status: JsonNode, message: JsonNode, artifacts: JsonNode) {
setAttribute(ATTRIBUTE_EXEC_CMD_STATUS, status)
- log.info("Executor status: $status")
+ log.info("Executor status : $status")
+ setAttribute(ATTRIBUTE_EXEC_CMD_ARTIFACTS, artifacts)
+ log.info("Executor artifacts: $artifacts")
setAttribute(ATTRIBUTE_EXEC_CMD_LOG, message)
- log.info("Executor message: $message")
+ log.info("Executor message : $message")
}
/**
* Utility function to set the output properties and errors of the executor node, in cas of errors
*/
- private fun setNodeOutputErrors(status: String, message: String) {
+ private fun setNodeOutputErrors(status: String, message: String, artifacts: JsonNode = "".asJsonPrimitive() ) {
setAttribute(ATTRIBUTE_EXEC_CMD_STATUS, status.asJsonPrimitive())
setAttribute(ATTRIBUTE_EXEC_CMD_LOG, message.asJsonPrimitive())
+ setAttribute(ATTRIBUTE_EXEC_CMD_ARTIFACTS, artifacts)
addError(status, ATTRIBUTE_EXEC_CMD_LOG, message)
}
-} \ No newline at end of file
+}
diff --git a/ms/blueprintsprocessor/functions/ansible-awx-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutorTest.kt b/ms/blueprintsprocessor/functions/ansible-awx-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutorTest.kt
index b60290268..5e0905dec 100644
--- a/ms/blueprintsprocessor/functions/ansible-awx-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutorTest.kt
+++ b/ms/blueprintsprocessor/functions/ansible-awx-executor/src/test/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/ansible/executor/ComponentRemoteAnsibleExecutorTest.kt
@@ -86,7 +86,7 @@ class ComponentRemoteAnsibleExecutorTest {
)
every {
webClientService.exchangeResource("GET", "/api/v2/jobs/$jobId/stdout/?format=txt", "",
- mapOf("Content-Type" to "text/plain ;utf-8"))
+ mapOf("Accept" to "text/plain"))
} returns WebClientResponse(200, getReport())
val selector = mapper.readTree(endpointSelector)
val bluePrintRestLibPropertyService = mockk<BluePrintRestLibPropertyService>()