Merge "Add k8s-upload-profile templating and packaging"

4 files changed, 204 insertions, 26 deletions

diff --git a/ms/blueprintsprocessor/functions/k8s-profile-upload/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/k8s/profile/upload/K8sProfileUploadComponent.kt b/ms/blueprintsprocessor/functions/k8s-profile-upload/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/k8s/profile/upload/K8sProfileUploadComponent.kt
index aa726a5e9..0ae76ea7e 100644
--- a/ms/blueprintsprocessor/functions/k8s-profile-upload/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/k8s/profile/upload/K8sProfileUploadComponent.kt
+++ b/ms/blueprintsprocessor/functions/k8s-profile-upload/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/k8s/profile/upload/K8sProfileUploadComponent.kt
@@ -19,26 +19,40 @@
 
 package org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.profile.upload
 
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import org.apache.commons.io.FileUtils
+import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintPropertiesService
+import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
+import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionConstants
+import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionService
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
+import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
+import org.onap.ccsdk.cds.controllerblueprints.core.data.ArtifactDefinition
+import org.onap.ccsdk.cds.controllerblueprints.core.returnNullIfMissing
+import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintVelocityTemplateService
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.ArchiveType
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintArchiveUtils
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.config.ConfigurableBeanFactory
 import org.springframework.context.annotation.Scope
 import org.springframework.stereotype.Component
-import java.util.ArrayList
-import java.nio.file.Paths
+import org.yaml.snakeyaml.Yaml
 import java.io.File
-import com.fasterxml.jackson.databind.JsonNode
-import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
-import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
-import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintPropertiesService
-import org.slf4j.LoggerFactory
+import java.nio.file.Files
 import java.nio.file.Path
-import org.onap.ccsdk.cds.controllerblueprints.core.returnNullIfMissing
-import com.fasterxml.jackson.databind.node.ObjectNode
-import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
+import java.nio.file.Paths
+import kotlin.collections.ArrayList
 
 @Component("component-k8s-profile-upload")
 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-open class K8sProfileUploadComponent(private var bluePrintPropertiesService: BluePrintPropertiesService) :
+open class K8sProfileUploadComponent(
+    private var bluePrintPropertiesService: BluePrintPropertiesService,
+    private val resourceResolutionService: ResourceResolutionService
+) :
 
     AbstractComponentFunction() {
 
@@ -98,9 +112,9 @@ open class K8sProfileUploadComponent(private var bluePrintPropertiesService: Blu
             }
 
             // For clarity we pull out the required fields
-            val profileName = prefixInputParamsMap[INPUT_K8S_PROFILE_NAME]?.returnNullIfMissing()?.textValue()
-            val definitionName = prefixInputParamsMap[INPUT_K8S_DEFINITION_NAME]?.returnNullIfMissing()?.textValue()
-            val definitionVersion = prefixInputParamsMap[INPUT_K8S_DEFINITION_VERSION]?.returnNullIfMissing()?.textValue()
+            val profileName: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAME]?.returnNullIfMissing()?.asText()
+            val definitionName: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_NAME]?.returnNullIfMissing()?.asText()
+            val definitionVersion: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_VERSION]?.returnNullIfMissing()?.asText()
 
             val k8sProfileUploadConfiguration = K8sProfileUploadConfiguration(bluePrintPropertiesService)
 
@@ -124,14 +138,26 @@ open class K8sProfileUploadComponent(private var bluePrintPropertiesService: Blu
             } else {
                 log.info("Uploading K8s Profile..")
                 outputPrefixStatuses.put(prefix, OUTPUT_ERROR)
-
+                val profileNamespace: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAMESPACE]?.returnNullIfMissing()?.asText()
+                var profileSource: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_SOURCE]?.returnNullIfMissing()?.asText()
+                if (profileNamespace == null)
+                    throw BluePrintProcessorException("Profile $profileName namespace is missing")
+                if (profileSource == null) {
+                    profileSource = profileName
+                    log.info("Profile name used instead of profile source")
+                }
+                val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
+                val artifact: ArtifactDefinition = bluePrintContext.nodeTemplateArtifact(nodeTemplateName, profileSource)
+                if (artifact.type != BluePrintConstants.MODEL_TYPE_ARTIFACT_K8S_PROFILE)
+                    throw BluePrintProcessorException("Unexpected profile artifact type for profile source " +
+                            "$profileSource. Expecting: $artifact.type")
                 var profile = K8sProfile()
                 profile.profileName = profileName
                 profile.rbName = definitionName
                 profile.rbVersion = definitionVersion
-                profile.namespace = prefixInputParamsMap[INPUT_K8S_PROFILE_NAMESPACE]?.textValue()
+                profile.namespace = profileNamespace
+                val profileFilePath: Path = prepareProfileFile(profileName, profileSource, artifact.file)
                 api.createProfile(profile)
-                val profileFilePath: Path = prepareProfileFile(profileName)
                 api.uploadProfileContent(profile, profileFilePath)
 
                 log.info("K8s Profile Upload Completed")
@@ -156,16 +182,150 @@ open class K8sProfileUploadComponent(private var bluePrintPropertiesService: Blu
         return result
     }
 
-    fun prepareProfileFile(k8sRbProfileName: String): Path {
+    private suspend fun prepareProfileFile(k8sRbProfileName: String, ks8ProfileSource: String, ks8ProfileLocation: String): Path {
         val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
         val bluePrintBasePath: String = bluePrintContext.rootPath
-        return Paths.get(
-            bluePrintBasePath.plus(File.separator)
-                .plus("Templates")
-                .plus(File.separator)
-                .plus("k8s-profiles")
-                .plus(File.separator)
-                .plus("$k8sRbProfileName.tar.gz")
-        )
+        val profileSourceFileFolderPath: String = bluePrintBasePath.plus(File.separator)
+                .plus(ks8ProfileLocation)
+        val profileFilePathTarGz: String = profileSourceFileFolderPath.plus(".tar.gz")
+        val profileFilePathTgz: String = profileSourceFileFolderPath.plus(".tgz")
+
+        if (Paths.get(profileFilePathTarGz).toFile().exists())
+            return Paths.get(profileFilePathTarGz)
+        else if (Paths.get(profileFilePathTgz).toFile().exists())
+            return Paths.get(profileFilePathTgz)
+        else if (Paths.get(profileSourceFileFolderPath).toFile().exists()) {
+            log.info("Profile building started from source $ks8ProfileSource")
+            val properties: MutableMap<String, Any> = mutableMapOf()
+            properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT] = false
+            properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_KEY] = ""
+            properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_ID] = ""
+            properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE] = ""
+            properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE] = 1
+            properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_SUMMARY] = false
+            val resolutionResult: Pair<String, JsonNode> = resourceResolutionService.resolveResources(
+                    bluePrintRuntimeService,
+                    nodeTemplateName,
+                    ks8ProfileSource,
+                    properties)
+            val tempMainPath: File = createTempDir("k8s-profile-", "")
+            val tempProfilePath: File = createTempDir("content-", "", tempMainPath)
+
+            try {
+                val manifestFiles: ArrayList<File>? = readManifestFiles(Paths.get(profileSourceFileFolderPath).toFile(),
+                        tempProfilePath)
+                if (manifestFiles != null) {
+                    templateLocation(Paths.get(profileSourceFileFolderPath).toFile(), resolutionResult.second,
+                            tempProfilePath, manifestFiles)
+                } else
+                    throw BluePrintProcessorException("Manifest file is missing")
+                // Preparation of the final profile content
+                val finalProfileFilePath = Paths.get(tempMainPath.toString().plus(File.separator).plus(
+                        "$k8sRbProfileName.tar.gz"))
+                if (!BluePrintArchiveUtils.compress(tempProfilePath, finalProfileFilePath.toFile(),
+                                ArchiveType.TarGz)) {
+                    throw BluePrintProcessorException("Profile compression has failed")
+                }
+                FileUtils.deleteDirectory(tempProfilePath)
+
+                return finalProfileFilePath
+            } catch (t: Throwable) {
+                FileUtils.deleteDirectory(tempMainPath)
+                throw t
+            }
+        } else
+            throw BluePrintProcessorException("Profile source $ks8ProfileSource is missing in CBA folder")
+    }
+
+    private fun readManifestFiles(profileSource: File, destinationFolder: File): ArrayList<File>? {
+        val directoryListing: Array<File>? = profileSource.listFiles()
+        var result: ArrayList<File>? = null
+        if (directoryListing != null) {
+            for (child in directoryListing) {
+                if (!child.isDirectory && child.name.toLowerCase() == "manifest.yaml") {
+                    child.bufferedReader().use { inr ->
+                        val manifestYaml = Yaml()
+                        val manifestObject: Map<String, Any> = manifestYaml.load(inr)
+                        val typeObject: MutableMap<String, Any>? = manifestObject["type"] as MutableMap<String, Any>?
+                        if (typeObject != null) {
+                            result = ArrayList<File>()
+                            val valuesObject = typeObject["values"]
+                            if (valuesObject != null) {
+                                result!!.add(File(destinationFolder.toString().plus(File.separator).plus(valuesObject)))
+                                result!!.add(File(destinationFolder.toString().plus(File.separator).plus(child.name)))
+                            }
+                            (typeObject["configresource"] as ArrayList<*>?)?.forEach { item ->
+                                val fileInfo: Map<String, Any> = item as Map<String, Any>
+                                val filePath = fileInfo["filepath"]
+                                val chartPath = fileInfo["chartpath"]
+                                if (filePath == null || chartPath == null)
+                                    log.error("One configresource in manifest was skipped because of the wrong format")
+                                else {
+                                    result!!.add(File(destinationFolder.toString().plus(File.separator).plus(filePath)))
+                                }
+                            }
+                        }
+                    }
+                    break
+                }
+            }
+        }
+        return result
+    }
+
+    private fun templateLocation(
+        location: File,
+        params: JsonNode,
+        destinationFolder: File,
+        manifestFiles: ArrayList<File>
+    ) {
+        val directoryListing: Array<File>? = location.listFiles()
+        if (directoryListing != null) {
+            for (child in directoryListing) {
+                var newDestinationFolder = destinationFolder.toPath()
+                if (child.isDirectory)
+                    newDestinationFolder = Paths.get(destinationFolder.toString().plus(File.separator).plus(child.name))
+
+                templateLocation(child, params, newDestinationFolder.toFile(), manifestFiles)
+            }
+        } else if (!location.isDirectory) {
+            if (location.extension.toLowerCase() == "vtl") {
+                templateFile(location, params, destinationFolder, manifestFiles)
+            } else {
+                val finalFilePath = Paths.get(destinationFolder.path.plus(File.separator)
+                        .plus(location.name)).toFile()
+                if (isFileInTheManifestFiles(finalFilePath, manifestFiles)) {
+                    if (!destinationFolder.exists())
+                        Files.createDirectories(destinationFolder.toPath())
+                    FileUtils.copyFile(location, finalFilePath)
+                }
+            }
+        }
+    }
+
+    private fun isFileInTheManifestFiles(file: File, manifestFiles: ArrayList<File>): Boolean {
+        manifestFiles.forEach { fileFromManifest ->
+            if (fileFromManifest.toString().toLowerCase() == file.toString().toLowerCase())
+                return true
+        }
+        return false
+    }
+
+    private fun templateFile(
+        templatedFile: File,
+        params: JsonNode,
+        destinationFolder: File,
+        manifestFiles: ArrayList<File>
+    ) {
+        val finalFile = File(destinationFolder.path.plus(File.separator)
+                .plus(templatedFile.nameWithoutExtension))
+        if (!isFileInTheManifestFiles(finalFile, manifestFiles))
+            return
+        val fileContent = templatedFile.bufferedReader().readText()
+        val finalFileContent = BluePrintVelocityTemplateService.generateContent(fileContent,
+                params, true)
+        if (!destinationFolder.exists())
+            Files.createDirectories(destinationFolder.toPath())
+        finalFile.bufferedWriter().use { out -> out.write(finalFileContent) }
     }
 }
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
index 8290167cd..a0ef565cc 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
@@ -228,6 +228,7 @@ object BluePrintConstants {
     const val MODEL_TYPE_ARTIFACT_SCRIPT_KOTLIN = "artifact-script-kotlin"
     const val MODEL_TYPE_ARTIFACT_DIRECTED_GRAPH = "artifact-directed-graph"
     const val MODEL_TYPE_ARTIFACT_COMPONENT_JAR = "artifact-component-jar"
+    const val MODEL_TYPE_ARTIFACT_K8S_PROFILE = "artifact-k8sprofile-content"
 
     const val TOSCA_SPEC = "TOSCA"
 
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSL.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSL.kt
index 3c5561994..64553230a 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSL.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSL.kt
@@ -311,6 +311,22 @@ fun BluePrintTypes.artifactTypeScriptKotlin(): ArtifactType {
     }
 }
 
+fun ServiceTemplateBuilder.artifactTypeK8sProfileFolder() {
+    val artifactType = BluePrintTypes.artifactTypeK8sProfileFolder()
+    if (this.artifactTypes == null) this.artifactTypes = hashMapOf()
+    this.artifactTypes!![artifactType.id!!] = artifactType
+}
+
+fun BluePrintTypes.artifactTypeK8sProfileFolder(): ArtifactType {
+    return artifactType(
+            id = BluePrintConstants.MODEL_TYPE_ARTIFACT_K8S_PROFILE,
+            version = BluePrintConstants.DEFAULT_VERSION_NUMBER,
+            derivedFrom = BluePrintConstants.MODEL_TYPE_ARTIFACT_TYPE_IMPLEMENTATION,
+            description = "K8s Profile Folder Artifact"
+    ) {
+    }
+}
+
 @Deprecated("CDS won't support, use implerative workflow definitions.")
 fun BluePrintTypes.artifactTypeDirectedGraph(): ArtifactType {
     return artifactType(
diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/test/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSLTest.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/test/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSLTest.kt
index bf16ea93a..824d7ddec 100644
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/test/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSLTest.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/test/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/dsl/BluePrintDSLTest.kt
@@ -110,6 +110,7 @@ class BluePrintDSLTest {
             artifactTypeScriptKotlin()
             artifactTypeMappingResource()
             artifactTypeComponentJar()
+            artifactTypeK8sProfileFolder()
 
             relationshipTypeConnectsTo()
             relationshipTypeDependsOn()