[vFW_CNF_CDS] Add workflow health-check and K8sHealthCheck.kt script

Issue-ID: INT-1899
Signed-off-by: j.blixt <j.blixt@partner.samsung.com>
Signed-off-by: Grzegorz Wielgosinski <g.wielgosins@samsung.com>
Signed-off-by: Konrad Bańka <k.banka@samsung.com>
Change-Id: I7d52c49bd0e40d30a560b2012362d38488392be6
(cherry picked from commit 532cfff8eaad73781cbb410064cd3b4bda5e3095)

2 files changed, 123 insertions, 5 deletions

diff --git a/heat/vFW_CNF_CDS/templates/cba/Scripts/kotlin/K8sHealthCheck.kt b/heat/vFW_CNF_CDS/templates/cba/Scripts/kotlin/K8sHealthCheck.kt
new file mode 100644
index 00000000..6c45b4e9
--- /dev/null
+++ b/heat/vFW_CNF_CDS/templates/cba/Scripts/kotlin/K8sHealthCheck.kt
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2021 Samsung Electronics
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.onap.ccsdk.cds.blueprintsprocessor.services.execution.scripts
+
+import com.fasterxml.jackson.databind.node.ObjectNode
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.joinAll
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import org.onap.ccsdk.cds.blueprintsprocessor.core.BlueprintPropertiesService
+import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
+import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.K8sConnectionPluginConfiguration
+import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.instance.K8sPluginInstanceApi
+import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.instance.healthcheck.K8sRbInstanceHealthCheck
+import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.instance.healthcheck.K8sRbInstanceHealthCheckSimple
+import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractScriptComponentFunction
+import org.slf4j.LoggerFactory
+
+open class K8sHealthCheck : AbstractScriptComponentFunction() {
+
+    private val log = LoggerFactory.getLogger(K8sHealthCheck::class.java)!!
+
+    override fun getName(): String {
+        return "K8sHealthCheck"
+    }
+
+    private fun initPluginApi(): K8sPluginInstanceApi {
+        val bluePrintPropertiesService: BlueprintPropertiesService = this.functionDependencyInstanceAsType("blueprintPropertiesService")!!
+        val k8sConfiguration = K8sConnectionPluginConfiguration(bluePrintPropertiesService)
+
+        return K8sPluginInstanceApi(k8sConfiguration)
+    }
+
+    override suspend fun processNB(executionRequest: ExecutionServiceInput) {
+        val instanceApi = initPluginApi()
+
+        log.info("Health check script execution - START")
+        val configValueSetup: ObjectNode = getDynamicProperties("config-deploy-setup") as ObjectNode
+        log.info("Config Value Setup: $configValueSetup")
+
+        val instanceHealthCheckList = startInstanceHealthCheck(configValueSetup, instanceApi)
+        val statuses = getStatuses(instanceHealthCheckList, instanceApi)
+        log.info("Health check script execution - END")
+    }
+
+    private fun startInstanceHealthCheck(configValueSetup: ObjectNode, instanceApi: K8sPluginInstanceApi): List<HealthCheckInstance> {
+        val healthCheckInstanceList = arrayListOf<HealthCheckInstance>()
+
+        configValueSetup.fields().forEach {
+            val instanceName = it.value.get("k8s-instance-id").asText()
+            val response: K8sRbInstanceHealthCheckSimple? = instanceApi.startInstanceHealthCheck(instanceName)
+            log.debug("K8sRbInstanceHealthCheckSimple response: $$response")
+            healthCheckInstanceList.add(HealthCheckInstance(instanceName, response?.id))
+        }
+        log.info("healthCheckInstanceList: $healthCheckInstanceList")
+
+        return healthCheckInstanceList
+    }
+
+    private fun getStatuses(instanceHealthCheckList: List<HealthCheckInstance>, instanceApi: K8sPluginInstanceApi): Map<String, String> {
+        val statuses = hashMapOf<String, String>()
+        runBlocking {
+            val jobs: List<Job> = instanceHealthCheckList.map {
+                launch {
+                    log.info("Thread started: ${Thread.currentThread().name} for $it")
+                    // WAIT APPROX 5 MINUTES
+                    repeat(30) { _ ->
+                        val response: K8sRbInstanceHealthCheck = instanceApi.getInstanceHealthCheck(it.heatStackId, it.healthCheckInstance!!)!!
+                        log.debug("Response for $it: $response")
+                        val status = response.status!!
+                        if (!"RUNNING".equals(status, true)) {
+                            statuses[it.heatStackId] = status
+                            log.info("Poll status: $status for $it")
+                            instanceApi.deleteInstanceHealthCheck(it.heatStackId, it.healthCheckInstance)
+                            cancel()
+                        }
+                        delay(10_000L)
+                    }
+                    statuses[it.heatStackId] = "Timeout"
+                    log.warn("Send delete hc request")
+                    instanceApi.deleteInstanceHealthCheck(it.heatStackId, it.healthCheckInstance!!)
+                }
+            }
+            jobs.joinAll()
+        }
+        log.info("Get statuses finished:")
+        log.info("$statuses")
+        return statuses
+    }
+
+    data class HealthCheckInstance(val heatStackId: String, val healthCheckInstance: String?) {
+        override fun toString(): String {
+            return "HealthCheckInstance(heatStackId='$heatStackId', healthCheckInstance='$healthCheckInstance')"
+        }
+    }
+
+    override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
+        log.info("Executing Recovery")
+        bluePrintRuntimeService.getBlueprintError().addError("${runtimeException.message}", getName())
+    }
+}
diff --git a/heat/vFW_CNF_CDS/templates/cba/Scripts/kotlin/SimpleStatusCheck.kt b/heat/vFW_CNF_CDS/templates/cba/Scripts/kotlin/SimpleStatusCheck.kt
index 90330644..e2d10534 100644
--- a/heat/vFW_CNF_CDS/templates/cba/Scripts/kotlin/SimpleStatusCheck.kt
+++ b/heat/vFW_CNF_CDS/templates/cba/Scripts/kotlin/SimpleStatusCheck.kt
@@ -40,7 +40,7 @@ open class SimpleStatusCheck : AbstractScriptComponentFunction() {
         val configValueSetup: ObjectNode = getDynamicProperties("config-deploy-setup") as ObjectNode
 
         val bluePrintPropertiesService: BlueprintPropertiesService =
-                this.functionDependencyInstanceAsType("blueprintPropertiesService")
+            this.functionDependencyInstanceAsType("blueprintPropertiesService")
 
         val k8sConfiguration = K8sConnectionPluginConfiguration(bluePrintPropertiesService)
 
@@ -54,7 +54,10 @@ open class SimpleStatusCheck : AbstractScriptComponentFunction() {
                 val instanceName = it.value.get("k8s-instance-id").asText()
 
                 val instanceStatus: K8sRbInstanceStatus? = instanceApi.getInstanceStatus(instanceName)
+                log.debug("Get status for $instanceName")
+                var status = ""
                 instanceStatus?.resourcesStatus?.forEach {
+                    log.debug("Resource: name=$it.name kind=$it.gvk.kind group=$it.gvk.group version=$it.gvk.version")
                     if (it.gvk?.kind == "Pod") {
                         var version = it.gvk?.version!!
                         if (it.gvk?.group!! != "")
@@ -62,10 +65,12 @@ open class SimpleStatusCheck : AbstractScriptComponentFunction() {
                         // val podStatus = instanceApi.queryInstanceStatus(instanceName, it.gvk?.kind!!, version, it.name, null)
                         // log.info(podStatus.toString())
                         val podState = it.status?.get("status") as Map<String, Object>
-
-                        if ((podState["phase"] as String) != "Running") {
+                        status = podState["phase"] as String
+                        if (status != "Running") {
                             continueCheck = true
-                            log.info("Pod ${it.name} [$vfModuleName] has invalid state ${(podState["phase"])}")
+                            log.info("Pod ${it.name} [$vfModuleName] has INVALID state ${(podState["phase"])}")
+                        } else {
+                            log.info("Pod ${it.name} [$vfModuleName] has VALID state ${(podState["phase"])}")
                         }
                     }
                 }
@@ -79,7 +84,7 @@ open class SimpleStatusCheck : AbstractScriptComponentFunction() {
                 checkCount = 0
         }
 
-        log.info("SIMPLE STATUS CHECK - END")
+        log.info("SIMPLE STATUS CHECK - END SUCCESS")
     }
 
     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {