Resolve the helm templates

Resolving the helm templates for each of the apps
in the compositeApp by utilizing the helm libraries
and thus getting the sorted templates with all values
rendered.
This is a subtask of adding instantiation APIs.

Issue-ID: MULTICLOUD-1041
Signed-off-by: Rajamohan Raj <rajamohan.raj@intel.com>
Change-Id: Id4fddda69b5276b2409d835723b11367400ab6ea

4 files changed, 586 insertions, 0 deletions

diff --git a/src/orchestrator/utils/helm/helm.go b/src/orchestrator/utils/helm/helm.go
new file mode 100644
index 00000000..f0d15fbf
--- /dev/null
+++ b/src/orchestrator/utils/helm/helm.go
@@ -0,0 +1,320 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helm
+
+import (
+	"bytes"
+	utils "github.com/onap/multicloud-k8s/src/orchestrator/utils"
+	"github.com/onap/multicloud-k8s/src/orchestrator/utils/types"
+
+	pkgerrors "github.com/pkg/errors"
+	"log"
+
+	"fmt"
+	"io/ioutil"
+	"k8s.io/helm/pkg/strvals"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/apimachinery/pkg/util/validation"
+	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/helm/pkg/chartutil"
+	"k8s.io/helm/pkg/manifest"
+	"k8s.io/helm/pkg/proto/hapi/chart"
+	"k8s.io/helm/pkg/releaseutil"
+	"k8s.io/helm/pkg/renderutil"
+	"k8s.io/helm/pkg/tiller"
+	"k8s.io/helm/pkg/timeconv"
+)
+
+// Template is the interface for all helm templating commands
+// Any backend implementation will implement this interface and will
+// access the functionality via this.
+type Template interface {
+	GenerateKubernetesArtifacts(
+		chartPath string,
+		valueFiles []string,
+		values []string) (map[string][]string, error)
+}
+
+// TemplateClient implements the Template interface
+// It will also be used to maintain any localized state
+type TemplateClient struct {
+	whitespaceRegex *regexp.Regexp
+	kubeVersion     string
+	kubeNameSpace   string
+	releaseName     string
+	manifestName    string
+}
+
+// NewTemplateClient returns a new instance of TemplateClient
+func NewTemplateClient(k8sversion, namespace, releasename, manifestFileName string) *TemplateClient {
+	return &TemplateClient{
+		whitespaceRegex: regexp.MustCompile(`^\s*$`),
+		// defaultKubeVersion is the default value of --kube-version flag
+		kubeVersion:   k8sversion,
+		kubeNameSpace: namespace,
+		releaseName:   releasename,
+		manifestName:  manifestFileName,
+	}
+}
+
+// Combines valueFiles and values into a single values stream.
+// values takes precedence over valueFiles
+func (h *TemplateClient) processValues(valueFiles []string, values []string) ([]byte, error) {
+	base := map[string]interface{}{}
+
+	//Values files that are used for overriding the chart
+	for _, filePath := range valueFiles {
+		currentMap := map[string]interface{}{}
+
+		var bytes []byte
+		var err error
+		if strings.TrimSpace(filePath) == "-" {
+			bytes, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			bytes, err = ioutil.ReadFile(filePath)
+		}
+
+		if err != nil {
+			return []byte{}, err
+		}
+
+		if err := yaml.Unmarshal(bytes, &currentMap); err != nil {
+			return []byte{}, fmt.Errorf("failed to parse %s: %s", filePath, err)
+		}
+		// Merge with the previous map
+		base = h.mergeValues(base, currentMap)
+	}
+
+	//User specified value. Similar to ones provided by -x
+	for _, value := range values {
+		if err := strvals.ParseInto(value, base); err != nil {
+			return []byte{}, fmt.Errorf("failed parsing --set data: %s", err)
+		}
+	}
+
+	return yaml.Marshal(base)
+}
+
+func (h *TemplateClient) mergeValues(dest map[string]interface{}, src map[string]interface{}) map[string]interface{} {
+	for k, v := range src {
+		// If the key doesn't exist already, then just set the key to that value
+		if _, exists := dest[k]; !exists {
+			dest[k] = v
+			continue
+		}
+		nextMap, ok := v.(map[string]interface{})
+		// If it isn't another map, overwrite the value
+		if !ok {
+			dest[k] = v
+			continue
+		}
+		// Edge case: If the key exists in the destination, but isn't a map
+		destMap, isMap := dest[k].(map[string]interface{})
+		// If the source map has a map for this key, prefer it
+		if !isMap {
+			dest[k] = v
+			continue
+		}
+		// If we got to this point, it is a map in both, so merge them
+		dest[k] = h.mergeValues(destMap, nextMap)
+	}
+	return dest
+}
+
+// GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template
+func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string,
+	values []string) ([]types.KubernetesResourceTemplate, error) {
+
+	var outputDir, chartPath, namespace, releaseName string
+	var retData []types.KubernetesResourceTemplate
+
+	releaseName = h.releaseName
+	namespace = h.kubeNameSpace
+
+	// verify chart path exists
+	if _, err := os.Stat(inputPath); err == nil {
+		if chartPath, err = filepath.Abs(inputPath); err != nil {
+			return retData, err
+		}
+	} else {
+		return retData, err
+	}
+
+	//Create a temp directory in the system temp folder
+	outputDir, err := ioutil.TempDir("", "helm-tmpl-")
+	if err != nil {
+		return retData, pkgerrors.Wrap(err, "Got error creating temp dir")
+	}
+	log.Printf("The o/p dir:: %s ", outputDir)
+
+	if namespace == "" {
+		namespace = "default"
+	}
+
+	// get combined values and create config
+	rawVals, err := h.processValues(valueFiles, values)
+	if err != nil {
+		return retData, err
+	}
+	config := &chart.Config{Raw: string(rawVals), Values: map[string]*chart.Value{}}
+
+	if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 {
+		return retData, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";"))
+	}
+
+	// Check chart requirements to make sure all dependencies are present in /charts
+	c, err := chartutil.Load(chartPath)
+	if err != nil {
+		return retData, pkgerrors.Errorf("Got error: %s", err.Error())
+	}
+
+	renderOpts := renderutil.Options{
+		ReleaseOptions: chartutil.ReleaseOptions{
+			Name:      releaseName,
+			IsInstall: true,
+			IsUpgrade: false,
+			Time:      timeconv.Now(),
+			Namespace: namespace,
+		},
+		KubeVersion: h.kubeVersion,
+	}
+
+	renderedTemplates, err := renderutil.Render(c, config, renderOpts)
+	if err != nil {
+		return retData, err
+	}
+
+	newRenderedTemplates := make(map[string]string)
+
+	//Some manifests can contain multiple yaml documents
+	//This step is splitting them up into multiple files
+	//Each file contains only a single k8s kind
+	for k, v := range renderedTemplates {
+		//Splits into manifest-0, manifest-1 etc
+		if filepath.Base(k) == "NOTES.txt" {
+			continue
+		}
+		rmap := releaseutil.SplitManifests(v)
+		count := 0
+		for _, v1 := range rmap {
+			key := fmt.Sprintf("%s-%d", k, count)
+			newRenderedTemplates[key] = v1
+			count = count + 1
+		}
+	}
+
+	listManifests := manifest.SplitManifests(newRenderedTemplates)
+	var manifestsToRender []manifest.Manifest
+	//render all manifests in the chart
+	manifestsToRender = listManifests
+	for _, m := range tiller.SortByKind(manifestsToRender) {
+		data := m.Content
+		b := filepath.Base(m.Name)
+		if b == "NOTES.txt" {
+			continue
+		}
+		if strings.HasPrefix(b, "_") {
+			continue
+		}
+
+		// blank template after execution
+		if h.whitespaceRegex.MatchString(data) {
+			continue
+		}
+
+		mfilePath := filepath.Join(outputDir, m.Name)
+		utils.EnsureDirectory(mfilePath)
+		err = ioutil.WriteFile(mfilePath, []byte(data), 0666)
+		if err != nil {
+			return retData, err
+		}
+
+		gvk, err := getGroupVersionKind(data)
+		if err != nil {
+			return retData, err
+		}
+
+		kres := types.KubernetesResourceTemplate{
+			GVK:      gvk,
+			FilePath: mfilePath,
+		}
+		retData = append(retData, kres)
+	}
+	return retData, nil
+}
+
+func getGroupVersionKind(data string) (schema.GroupVersionKind, error) {
+	out, err := k8syaml.ToJSON([]byte(data))
+	if err != nil {
+		return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Converting yaml to json")
+	}
+
+	simpleMeta := json.SimpleMetaFactory{}
+	gvk, err := simpleMeta.Interpret(out)
+	if err != nil {
+		return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Parsing apiversion and kind")
+	}
+
+	return *gvk, nil
+}
+
+// Resolver is an interface exposes the helm related functionalities
+type Resolver interface {
+	Resolve(appContent, appProfileContent []byte, overrideValuesOfAppStr []string, rName string) ([]types.KubernetesResourceTemplate, error)
+}
+
+// Resolve function
+func (h *TemplateClient) Resolve(appContent []byte, appProfileContent []byte, overrideValuesOfAppStr []string, rName, appName string) ([]types.KubernetesResourceTemplate, error) {
+
+	var sortedTemplates []types.KubernetesResourceTemplate
+
+	//prPath is the tmp path where the appProfileContent is extracted.
+	prPath, err := utils.ExtractTarBall(bytes.NewBuffer(appProfileContent))
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Extracting Profile Content")
+	}
+	log.Printf("The profile path:: %s", prPath)
+
+	prYamlClient, err := utils.ProcessProfileYaml(prPath, h.manifestName)
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Processing Profile Manifest")
+	}
+	log.Println("Got the profileYamlClient..")
+
+	//chartBasePath is the tmp path where the appContent(rawHelmCharts) is extracted.
+	chartBasePath, err := utils.ExtractTarBall(bytes.NewBuffer(appContent))
+	log.Printf("The chartBasePath :: %s", chartBasePath)
+
+	err = prYamlClient.CopyConfigurationOverrides(chartBasePath)
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Copying configresources to chart")
+	}
+
+	chartPath := filepath.Join(chartBasePath, appName)
+	sortedTemplates, err = h.GenerateKubernetesArtifacts(chartPath, []string{prYamlClient.GetValues()}, overrideValuesOfAppStr)
+	if err != nil {
+		return sortedTemplates, pkgerrors.Wrap(err, "Generate final k8s yaml")
+	}
+	return sortedTemplates, nil
+}
diff --git a/src/orchestrator/utils/profile_yaml.go b/src/orchestrator/utils/profile_yaml.go
new file mode 100644
index 00000000..91687c0a
--- /dev/null
+++ b/src/orchestrator/utils/profile_yaml.go
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"io/ioutil"
+	"log"
+	"path/filepath"
+
+	"github.com/ghodss/yaml"
+	pkgerrors "github.com/pkg/errors"
+)
+
+/*
+#Sample Yaml format for profile manifest.yaml
+---
+version: v1
+type:
+  values: "values_override.yaml"
+  configresource:
+    - filepath: config.yaml
+      chartpath: chart/config/resources/config.yaml
+    - filepath: config2.yaml
+      chartpath: chart/config/resources/config2.yaml
+*/
+
+type overrideFiles struct {
+	FilePath  string `yaml:"filepath"`
+	ChartPath string `yaml:"chartpath"`
+}
+
+type supportedOverrides struct {
+	ConfigResource []overrideFiles `yaml:"configresource"`
+	Values         string          `yaml:"values"`
+}
+
+type profileOverride struct {
+	Version string             `yaml:"version"`
+	Type    supportedOverrides `yaml:"type"`
+}
+
+type ProfileYamlClient struct {
+	path     string
+	override profileOverride
+}
+
+func (p ProfileYamlClient) Print() {
+	log.Println(p.override)
+}
+
+//GetValues returns a path to the override values.yam
+//that was part of the profile
+func (p ProfileYamlClient) GetValues() string {
+	return filepath.Join(p.path, p.override.Type.Values)
+}
+
+//CopyConfigurationOverrides copies the various files that are
+//provided as overrides to their corresponding locations within
+//the destination chart.
+func (p ProfileYamlClient) CopyConfigurationOverrides(chartPath string) error {
+
+	//Iterate over each configresource and copy that file into
+	//the respective path in the chart.
+	for _, v := range p.override.Type.ConfigResource {
+		data, err := ioutil.ReadFile(filepath.Join(p.path, v.FilePath))
+		if err != nil {
+			return pkgerrors.Wrap(err, "Reading configuration file")
+		}
+		err = ioutil.WriteFile(filepath.Join(chartPath, v.ChartPath), data, 0644)
+		if err != nil {
+			return pkgerrors.Wrap(err, "Writing configuration file into chartpath")
+		}
+	}
+
+	return nil
+}
+
+//ProcessProfileYaml parses the manifest.yaml file that is part of the profile
+//package and creates the appropriate structures out of it.
+func ProcessProfileYaml(fpath string, manifestFileName string) (ProfileYamlClient, error) {
+
+	p := filepath.Join(fpath, manifestFileName)
+	data, err := ioutil.ReadFile(p)
+	if err != nil {
+		return ProfileYamlClient{}, pkgerrors.Wrap(err, "Reading manifest file")
+	}
+
+	out := profileOverride{}
+	err = yaml.Unmarshal(data, &out)
+	if err != nil {
+		return ProfileYamlClient{}, pkgerrors.Wrap(err, "Marshaling manifest yaml file")
+	}
+
+	return ProfileYamlClient{path: fpath, override: out}, nil
+}
diff --git a/src/orchestrator/utils/types/types.go b/src/orchestrator/utils/types/types.go
new file mode 100644
index 00000000..4ffb4180
--- /dev/null
+++ b/src/orchestrator/utils/types/types.go
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package types
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// KubernetesResourceTemplate - Represents the template that is used to create a particular
+// resource in Kubernetes
+type KubernetesResourceTemplate struct {
+	// Tracks the apiVersion and Kind of the resource
+	GVK schema.GroupVersionKind
+	// Path to the file that contains the resource info
+	FilePath string
+}
+
+// KubernetesResource is the resource that is created in Kubernetes
+// It is the type that will be used for tracking a resource.
+// Any future information such as status, time can be added here
+// for tracking.
+type KubernetesResource struct {
+	// Tracks the apiVersion and Kind of the resource
+	GVK schema.GroupVersionKind
+	// Name of resource in Kubernetes
+	Name string
+}
diff --git a/src/orchestrator/utils/util-functions.go b/src/orchestrator/utils/util-functions.go
new file mode 100644
index 00000000..13c78ba4
--- /dev/null
+++ b/src/orchestrator/utils/util-functions.go
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+
+	pkgerrors "github.com/pkg/errors"
+)
+
+//ExtractTarBall provides functionality to extract a tar.gz file
+//into a temporary location for later use.
+//It returns the path to the new location
+func ExtractTarBall(r io.Reader) (string, error) {
+	//Check if it is a valid gz
+	gzf, err := gzip.NewReader(r)
+	if err != nil {
+		return "", pkgerrors.Wrap(err, "Invalid gzip format")
+	}
+
+	//Check if it is a valid tar file
+	//Unfortunately this can only be done by inspecting all the tar contents
+	tarR := tar.NewReader(gzf)
+	first := true
+
+	outDir, _ := ioutil.TempDir("", "k8s-ext-")
+
+	for true {
+		header, err := tarR.Next()
+
+		if err == io.EOF {
+			//Check if we have just a gzip file without a tar archive inside
+			if first {
+				return "", pkgerrors.New("Empty or non-existant Tar file found")
+			}
+			//End of archive
+			break
+		}
+
+		if err != nil {
+			return "", pkgerrors.Wrap(err, "Error reading tar file")
+		}
+
+		target := filepath.Join(outDir, header.Name)
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				// Using 755 read, write, execute for owner
+				// groups and others get read and execute permissions
+				// on the folder.
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return "", pkgerrors.Wrap(err, "Creating directory")
+				}
+			}
+		case tar.TypeReg:
+			if target == outDir { // Handle '.' substituted to '' entry
+				continue
+			}
+
+			err = EnsureDirectory(target)
+			if err != nil {
+				return "", pkgerrors.Wrap(err, "Creating Directory")
+			}
+
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return "", pkgerrors.Wrap(err, "Creating file")
+			}
+
+			// copy over contents
+			if _, err := io.Copy(f, tarR); err != nil {
+				return "", pkgerrors.Wrap(err, "Copying file content")
+			}
+
+			// close for each file instead of a defer for all
+			// at the end of the function
+			f.Close()
+		}
+
+		first = false
+	}
+
+	return outDir, nil
+}
+
+//EnsureDirectory makes sure that the directories specified in the path exist
+//If not, it will create them, if possible.
+func EnsureDirectory(f string) error {
+	base := path.Dir(f)
+	_, err := os.Stat(base)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return os.MkdirAll(base, 0755)
+}