Resolve the helm templates

Resolving the helm templates for each of the apps
in the compositeApp by utilizing the helm libraries
and thus getting the sorted templates with all values
rendered.
This is a subtask of adding instantiation APIs.

Issue-ID: MULTICLOUD-1041
Signed-off-by: Rajamohan Raj <rajamohan.raj@intel.com>
Change-Id: Id4fddda69b5276b2409d835723b11367400ab6ea

1 files changed, 320 insertions, 0 deletions

diff --git a/src/orchestrator/utils/helm/helm.go b/src/orchestrator/utils/helm/helm.go
new file mode 100644
index 00000000..f0d15fbf
--- /dev/null
+++ b/src/orchestrator/utils/helm/helm.go
@@ -0,0 +1,320 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helm
+
+import (
+	"bytes"
+	utils "github.com/onap/multicloud-k8s/src/orchestrator/utils"
+	"github.com/onap/multicloud-k8s/src/orchestrator/utils/types"
+
+	pkgerrors "github.com/pkg/errors"
+	"log"
+
+	"fmt"
+	"io/ioutil"
+	"k8s.io/helm/pkg/strvals"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/apimachinery/pkg/util/validation"
+	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/helm/pkg/chartutil"
+	"k8s.io/helm/pkg/manifest"
+	"k8s.io/helm/pkg/proto/hapi/chart"
+	"k8s.io/helm/pkg/releaseutil"
+	"k8s.io/helm/pkg/renderutil"
+	"k8s.io/helm/pkg/tiller"
+	"k8s.io/helm/pkg/timeconv"
+)
+
+// Template is the interface for all helm templating commands
+// Any backend implementation will implement this interface and will
+// access the functionality via this.
+type Template interface {
+	GenerateKubernetesArtifacts(
+		chartPath string,
+		valueFiles []string,
+		values []string) (map[string][]string, error)
+}
+
+// TemplateClient implements the Template interface
+// It will also be used to maintain any localized state
+type TemplateClient struct {
+	whitespaceRegex *regexp.Regexp
+	kubeVersion     string
+	kubeNameSpace   string
+	releaseName     string
+	manifestName    string
+}
+
+// NewTemplateClient returns a new instance of TemplateClient
+func NewTemplateClient(k8sversion, namespace, releasename, manifestFileName string) *TemplateClient {
+	return &TemplateClient{
+		whitespaceRegex: regexp.MustCompile(`^\s*$`),
+		// defaultKubeVersion is the default value of --kube-version flag
+		kubeVersion:   k8sversion,
+		kubeNameSpace: namespace,
+		releaseName:   releasename,
+		manifestName:  manifestFileName,
+	}
+}
+
+// Combines valueFiles and values into a single values stream.
+// values takes precedence over valueFiles
+func (h *TemplateClient) processValues(valueFiles []string, values []string) ([]byte, error) {
+	base := map[string]interface{}{}
+
+	//Values files that are used for overriding the chart
+	for _, filePath := range valueFiles {
+		currentMap := map[string]interface{}{}
+
+		var bytes []byte
+		var err error
+		if strings.TrimSpace(filePath) == "-" {
+			bytes, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			bytes, err = ioutil.ReadFile(filePath)
+		}
+
+		if err != nil {
+			return []byte{}, err
+		}
+
+		if err := yaml.Unmarshal(bytes, &currentMap); err != nil {
+			return []byte{}, fmt.Errorf("failed to parse %s: %s", filePath, err)
+		}
+		// Merge with the previous map
+		base = h.mergeValues(base, currentMap)
+	}
+
+	//User specified value. Similar to ones provided by -x
+	for _, value := range values {
+		if err := strvals.ParseInto(value, base); err != nil {
+			return []byte{}, fmt.Errorf("failed parsing --set data: %s", err)
+		}
+	}
+
+	return yaml.Marshal(base)
+}
+
+func (h *TemplateClient) mergeValues(dest map[string]interface{}, src map[string]interface{}) map[string]interface{} {
+	for k, v := range src {
+		// If the key doesn't exist already, then just set the key to that value
+		if _, exists := dest[k]; !exists {
+			dest[k] = v
+			continue
+		}
+		nextMap, ok := v.(map[string]interface{})
+		// If it isn't another map, overwrite the value
+		if !ok {
+			dest[k] = v
+			continue
+		}
+		// Edge case: If the key exists in the destination, but isn't a map
+		destMap, isMap := dest[k].(map[string]interface{})
+		// If the source map has a map for this key, prefer it
+		if !isMap {
+			dest[k] = v
+			continue
+		}
+		// If we got to this point, it is a map in both, so merge them
+		dest[k] = h.mergeValues(destMap, nextMap)
+	}
+	return dest
+}
+
+// GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template
+func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string,
+	values []string) ([]types.KubernetesResourceTemplate, error) {
+
+	var outputDir, chartPath, namespace, releaseName string
+	var retData []types.KubernetesResourceTemplate
+
+	releaseName = h.releaseName
+	namespace = h.kubeNameSpace
+
+	// verify chart path exists
+	if _, err := os.Stat(inputPath); err == nil {
+		if chartPath, err = filepath.Abs(inputPath); err != nil {
+			return retData, err
+		}
+	} else {
+		return retData, err
+	}
+
+	//Create a temp directory in the system temp folder
+	outputDir, err := ioutil.TempDir("", "helm-tmpl-")
+	if err != nil {
+		return retData, pkgerrors.Wrap(err, "Got error creating temp dir")
+	}
+	log.Printf("The o/p dir:: %s ", outputDir)
+
+	if namespace == "" {
+		namespace = "default"
+	}
+
+	// get combined values and create config
+	rawVals, err := h.processValues(valueFiles, values)
+	if err != nil {
+		return retData, err
+	}
+	config := &chart.Config{Raw: string(rawVals), Values: map[string]*chart.Value{}}
+
+	if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 {
+		return retData, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";"))
+	}
+
+	// Check chart requirements to make sure all dependencies are present in /charts
+	c, err := chartutil.Load(chartPath)
+	if err != nil {
+		return retData, pkgerrors.Errorf("Got error: %s", err.Error())
+	}
+
+	renderOpts := renderutil.Options{
+		ReleaseOptions: chartutil.ReleaseOptions{
+			Name:      releaseName,
+			IsInstall: true,
+			IsUpgrade: false,
+			Time:      timeconv.Now(),
+			Namespace: namespace,
+		},
+		KubeVersion: h.kubeVersion,
+	}
+
+	renderedTemplates, err := renderutil.Render(c, config, renderOpts)
+	if err != nil {
+		return retData, err
+	}
+
+	newRenderedTemplates := make(map[string]string)
+
+	//Some manifests can contain multiple yaml documents
+	//This step is splitting them up into multiple files
+	//Each file contains only a single k8s kind
+	for k, v := range renderedTemplates {
+		//Splits into manifest-0, manifest-1 etc
+		if filepath.Base(k) == "NOTES.txt" {
+			continue
+		}
+		rmap := releaseutil.SplitManifests(v)
+		count := 0
+		for _, v1 := range rmap {
+			key := fmt.Sprintf("%s-%d", k, count)
+			newRenderedTemplates[key] = v1
+			count = count + 1
+		}
+	}
+
+	listManifests := manifest.SplitManifests(newRenderedTemplates)
+	var manifestsToRender []manifest.Manifest
+	//render all manifests in the chart
+	manifestsToRender = listManifests
+	for _, m := range tiller.SortByKind(manifestsToRender) {
+		data := m.Content
+		b := filepath.Base(m.Name)
+		if b == "NOTES.txt" {
+			continue
+		}
+		if strings.HasPrefix(b, "_") {
+			continue
+		}
+
+		// blank template after execution
+		if h.whitespaceRegex.MatchString(data) {
+			continue
+		}
+
+		mfilePath := filepath.Join(outputDir, m.Name)
+		utils.EnsureDirectory(mfilePath)
+		err = ioutil.WriteFile(mfilePath, []byte(data), 0666)
+		if err != nil {
+			return retData, err
+		}
+
+		gvk, err := getGroupVersionKind(data)
+		if err != nil {
+			return retData, err
+		}
+
+		kres := types.KubernetesResourceTemplate{
+			GVK:      gvk,
+			FilePath: mfilePath,
+		}
+		retData = append(retData, kres)
+	}
+	return retData, nil
+}
+
+func getGroupVersionKind(data string) (schema.GroupVersionKind, error) {
+	out, err := k8syaml.ToJSON([]byte(data))
+	if err != nil {
+		return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Converting yaml to json")
+	}
+
+	simpleMeta := json.SimpleMetaFactory{}
+	gvk, err := simpleMeta.Interpret(out)
+	if err != nil {
+		return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Parsing apiversion and kind")
+	}
+
+	return *gvk, nil
+}
+
+// Resolver is an interface exposes the helm related functionalities
+type Resolver interface {
+	Resolve(appContent, appProfileContent []byte, overrideValuesOfAppStr []string, rName string) ([]types.KubernetesResourceTemplate, error)
+}
+
+// Resolve function
+func (h *TemplateClient) Resolve(appContent []byte, appProfileContent []byte, overrideValuesOfAppStr []string, rName, appName string) ([]types.KubernetesResourceTemplate, error) {
+
+	var sortedTemplates []types.KubernetesResourceTemplate
+
+	//prPath is the tmp path where the appProfileContent is extracted.
+	prPath, err := utils.ExtractTarBall(bytes.NewBuffer(appProfileContent))
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Extracting Profile Content")
+	}
+	log.Printf("The profile path:: %s", prPath)
+
+	prYamlClient, err := utils.ProcessProfileYaml(prPath, h.manifestName)
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Processing Profile Manifest")
+	}
+	log.Println("Got the profileYamlClient..")
+
+	//chartBasePath is the tmp path where the appContent(rawHelmCharts) is extracted.
+	chartBasePath, err := utils.ExtractTarBall(bytes.NewBuffer(appContent))
+	log.Printf("The chartBasePath :: %s", chartBasePath)
+
+	err = prYamlClient.CopyConfigurationOverrides(chartBasePath)
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Copying configresources to chart")
+	}
+
+	chartPath := filepath.Join(chartBasePath, appName)
+	sortedTemplates, err = h.GenerateKubernetesArtifacts(chartPath, []string{prYamlClient.GetValues()}, overrideValuesOfAppStr)
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Generate final k8s yaml")
+	}
+	return sortedTemplates, nil
+}