Add support for helm templates

Add support for evaluating helm templates
The interface provides a way to get a map which
contains a mapping from kubernetes kind to the
corresponding yaml file that defines a resource
of that kind. This map is then provided to the
instantiation code at instantiation time to create
in the kubernetes cluster.
P5: Use filepath.join instead of strings.join
P9: rebase with new folder structure
P10: moved the helm code into its own package.
P12: Add unit tests
     update the go.mod to use latest docker version

Issue-ID: MULTICLOUD-291
Change-Id: Ie75f5c616cc0cdc3e0ace49ff2c2f6c356a4c0d1
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>

2 files changed, 454 insertions, 0 deletions

diff --git a/src/k8splugin/internal/helm/helm.go b/src/k8splugin/internal/helm/helm.go
new file mode 100644
index 00000000..65a36d6b
--- /dev/null
+++ b/src/k8splugin/internal/helm/helm.go
@@ -0,0 +1,259 @@
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helm
+
+import (
+	"fmt"
+	"io/ioutil"
+	"k8s.io/helm/pkg/strvals"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	pkgerrors "github.com/pkg/errors"
+
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/helm/pkg/chartutil"
+	"k8s.io/helm/pkg/manifest"
+	"k8s.io/helm/pkg/proto/hapi/chart"
+	"k8s.io/helm/pkg/releaseutil"
+	"k8s.io/helm/pkg/renderutil"
+	"k8s.io/helm/pkg/tiller"
+	"k8s.io/helm/pkg/timeconv"
+)
+
+// Template is the interface for all helm templating commands
+// Any backend implementation will implement this interface and will
+// access the functionality via this.
+type Template interface {
+	GenerateKubernetesArtifacts(
+		chartPath string,
+		valueFiles []string,
+		values []string) (map[string][]string, error)
+}
+
+// TemplateClient implements the Template interface
+// It will also be used to maintain any localized state
+type TemplateClient struct {
+	whitespaceRegex *regexp.Regexp
+	kubeVersion     string
+	kubeNameSpace   string
+	releaseName     string
+}
+
+// NewTemplateClient returns a new instance of TemplateClient
+func NewTemplateClient(k8sversion, namespace, releasename string) *TemplateClient {
+	return &TemplateClient{
+		whitespaceRegex: regexp.MustCompile(`^\s*$`),
+		// defaultKubeVersion is the default value of --kube-version flag
+		kubeVersion:   k8sversion,
+		kubeNameSpace: namespace,
+		releaseName:   releasename,
+	}
+}
+
+// Combines valueFiles and values into a single values stream.
+// values takes precedence over valueFiles
+func (h *TemplateClient) processValues(valueFiles []string, values []string) ([]byte, error) {
+	base := map[string]interface{}{}
+
+	//Values files that are used for overriding the chart
+	for _, filePath := range valueFiles {
+		currentMap := map[string]interface{}{}
+
+		var bytes []byte
+		var err error
+		if strings.TrimSpace(filePath) == "-" {
+			bytes, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			bytes, err = ioutil.ReadFile(filePath)
+		}
+
+		if err != nil {
+			return []byte{}, err
+		}
+
+		if err := yaml.Unmarshal(bytes, &currentMap); err != nil {
+			return []byte{}, fmt.Errorf("failed to parse %s: %s", filePath, err)
+		}
+		// Merge with the previous map
+		base = h.mergeValues(base, currentMap)
+	}
+
+	//User specified value. Similar to ones provided by -x
+	for _, value := range values {
+		if err := strvals.ParseInto(value, base); err != nil {
+			return []byte{}, fmt.Errorf("failed parsing --set data: %s", err)
+		}
+	}
+
+	return yaml.Marshal(base)
+}
+
+func (h *TemplateClient) mergeValues(dest map[string]interface{}, src map[string]interface{}) map[string]interface{} {
+	for k, v := range src {
+		// If the key doesn't exist already, then just set the key to that value
+		if _, exists := dest[k]; !exists {
+			dest[k] = v
+			continue
+		}
+		nextMap, ok := v.(map[string]interface{})
+		// If it isn't another map, overwrite the value
+		if !ok {
+			dest[k] = v
+			continue
+		}
+		// Edge case: If the key exists in the destination, but isn't a map
+		destMap, isMap := dest[k].(map[string]interface{})
+		// If the source map has a map for this key, prefer it
+		if !isMap {
+			dest[k] = v
+			continue
+		}
+		// If we got to this point, it is a map in both, so merge them
+		dest[k] = h.mergeValues(destMap, nextMap)
+	}
+	return dest
+}
+
+func (h *TemplateClient) ensureDirectory(f string) error {
+	base := path.Dir(f)
+	_, err := os.Stat(base)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return os.MkdirAll(base, 0755)
+}
+
+// GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template
+func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string, values []string) (map[string][]string, error) {
+
+	var outputDir, chartPath, namespace, releaseName string
+	var retData map[string][]string
+
+	releaseName = h.releaseName
+	namespace = h.kubeNameSpace
+
+	// verify chart path exists
+	if _, err := os.Stat(inputPath); err == nil {
+		if chartPath, err = filepath.Abs(inputPath); err != nil {
+			return retData, err
+		}
+	} else {
+		return retData, err
+	}
+
+	//Create a temp directory in the system temp folder
+	outputDir, err := ioutil.TempDir("", "helm-tmpl-")
+	if err != nil {
+		return retData, pkgerrors.Wrap(err, "Got error creating temp dir")
+	}
+
+	if namespace == "" {
+		namespace = "default"
+	}
+
+	// get combined values and create config
+	rawVals, err := h.processValues(valueFiles, values)
+	if err != nil {
+		return retData, err
+	}
+	config := &chart.Config{Raw: string(rawVals), Values: map[string]*chart.Value{}}
+
+	if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 {
+		return retData, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";"))
+	}
+
+	// Check chart requirements to make sure all dependencies are present in /charts
+	c, err := chartutil.Load(chartPath)
+	if err != nil {
+		return retData, pkgerrors.Errorf("Got error: %s", err.Error())
+	}
+
+	renderOpts := renderutil.Options{
+		ReleaseOptions: chartutil.ReleaseOptions{
+			Name:      releaseName,
+			IsInstall: true,
+			IsUpgrade: false,
+			Time:      timeconv.Now(),
+			Namespace: namespace,
+		},
+		KubeVersion: h.kubeVersion,
+	}
+
+	renderedTemplates, err := renderutil.Render(c, config, renderOpts)
+	if err != nil {
+		return retData, err
+	}
+
+	newRenderedTemplates := make(map[string]string)
+
+	//Some manifests can contain multiple yaml documents
+	//This step is splitting them up into multiple files
+	//Each file contains only a single k8s kind
+	for k, v := range renderedTemplates {
+		//Splits into manifest-0, manifest-1 etc
+		if filepath.Base(k) == "NOTES.txt" {
+			continue
+		}
+		rmap := releaseutil.SplitManifests(v)
+		count := 0
+		for _, v1 := range rmap {
+			key := fmt.Sprintf("%s-%d", k, count)
+			newRenderedTemplates[key] = v1
+			count = count + 1
+		}
+	}
+
+	listManifests := manifest.SplitManifests(newRenderedTemplates)
+	var manifestsToRender []manifest.Manifest
+	//render all manifests in the chart
+	manifestsToRender = listManifests
+	retData = make(map[string][]string)
+	for _, m := range tiller.SortByKind(manifestsToRender) {
+		data := m.Content
+		b := filepath.Base(m.Name)
+		if b == "NOTES.txt" {
+			continue
+		}
+		if strings.HasPrefix(b, "_") {
+			continue
+		}
+
+		// blank template after execution
+		if h.whitespaceRegex.MatchString(data) {
+			continue
+		}
+
+		mfilePath := filepath.Join(outputDir, m.Name)
+		h.ensureDirectory(mfilePath)
+		err = ioutil.WriteFile(mfilePath, []byte(data), 0666)
+		if err != nil {
+			return retData, err
+		}
+
+		if val, ok := retData[m.Head.Kind]; ok {
+			retData[m.Head.Kind] = append(val, mfilePath)
+		} else {
+			retData[m.Head.Kind] = []string{mfilePath}
+		}
+	}
+	return retData, nil
+}
diff --git a/src/k8splugin/internal/helm/helm_test.go b/src/k8splugin/internal/helm/helm_test.go
new file mode 100644
index 00000000..a5bcd9c8
--- /dev/null
+++ b/src/k8splugin/internal/helm/helm_test.go
@@ -0,0 +1,195 @@
+// +build unit
+
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helm
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestProcessValues(t *testing.T) {
+
+	chartDir := "../../mock_files/mock_charts/testchart1"
+	profileDir := "../../mock_files/mock_profiles/profile1"
+
+	testCases := []struct {
+		label         string
+		valueFiles    []string
+		values        []string
+		expectedHash  string
+		expectedError string
+	}{
+		{
+			label: "Process Values with Value Files Override",
+			valueFiles: []string{
+				filepath.Join(chartDir, "values.yaml"),
+				filepath.Join(profileDir, "override_values.yaml"),
+			},
+			//Hash of a combined values.yaml file that is expected
+			expectedHash:  "c18a70f426933de3c051c996dc34fd537d0131b2d13a2112a2ecff674db6c2f9",
+			expectedError: "",
+		},
+		{
+			label: "Process Values with Values Pair Override",
+			valueFiles: []string{
+				filepath.Join(chartDir, "values.yaml"),
+			},
+			//Use the same convention as specified in helm template --set
+			values: []string{
+				"service.externalPort=82",
+			},
+			//Hash of a combined values.yaml file that is expected
+			expectedHash:  "028a3521fc9f8777ea7e67a6de0c51f2c875b88ca91734999657f0ca924ddb7a",
+			expectedError: "",
+		},
+		{
+			label: "Process Values with Both Overrides",
+			valueFiles: []string{
+				filepath.Join(chartDir, "values.yaml"),
+				filepath.Join(profileDir, "override_values.yaml"),
+			},
+			//Use the same convention as specified in helm template --set
+			//Key takes precedence over the value from override_values.yaml
+			values: []string{
+				"service.externalPort=82",
+			},
+			//Hash of a combined values.yaml file that is expected
+			expectedHash:  "516fab4ab7b76ba2ff35a97c2a79b74302543f532857b945f2fe25e717e755be",
+			expectedError: "",
+		},
+	}
+
+	h := sha256.New()
+
+	for _, testCase := range testCases {
+		t.Run(testCase.label, func(t *testing.T) {
+			tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename")
+			out, err := tc.processValues(testCase.valueFiles, testCase.values)
+			if err != nil {
+				if testCase.expectedError == "" {
+					t.Fatalf("Got an error %s", err)
+				}
+				if strings.Contains(err.Error(), testCase.expectedError) == false {
+					t.Fatalf("Got unexpected error message %s", err)
+				}
+			} else {
+				//Compute the hash of returned data and compare
+				h.Write(out)
+				gotHash := fmt.Sprintf("%x", h.Sum(nil))
+				h.Reset()
+				if gotHash != testCase.expectedHash {
+					t.Fatalf("Got unexpected values.yaml %s", out)
+				}
+			}
+		})
+	}
+}
+
+func TestGenerateKubernetesArtifacts(t *testing.T) {
+
+	chartDir := "../../mock_files/mock_charts/testchart1"
+	profileDir := "../../mock_files/mock_profiles/profile1"
+
+	testCases := []struct {
+		label           string
+		chartPath       string
+		valueFiles      []string
+		values          []string
+		expectedHashMap map[string]string
+		expectedError   string
+	}{
+		{
+			label:      "Generate artifacts without any overrides",
+			chartPath:  chartDir,
+			valueFiles: []string{},
+			values:     []string{},
+			//sha256 hash of the evaluated templates in each chart
+			expectedHashMap: map[string]string{
+				"testchart1/templates/service.yaml": "bbd7257d1f6ab958680e642a8fbbbea2002ebbaa9276fb51fbd71b4b66a772cc",
+				"subcharta/templates/service.yaml":  "570389588fffdb7193ab265888d781f3d751f3a40362533344f9aa7bb93a8bb0",
+				"subchartb/templates/service.yaml":  "5654e03d922e8ec49649b4bbda9dfc9e643b3b7c9c18b602cc7e26fd36a39c2a",
+			},
+			expectedError: "",
+		},
+		{
+			label:     "Generate artifacts with overrides",
+			chartPath: chartDir,
+			valueFiles: []string{
+				filepath.Join(profileDir, "override_values.yaml"),
+			},
+			values: []string{
+				"service.externalPort=82",
+			},
+			//sha256 hash of the evaluated templates in each chart
+			expectedHashMap: map[string]string{
+				"testchart1/templates/service.yaml": "4c5aa5d38b763fe4730fc31a759c40566a99a9c51f5e0fc7f93473c9affc2ca8",
+				"subcharta/templates/service.yaml":  "570389588fffdb7193ab265888d781f3d751f3a40362533344f9aa7bb93a8bb0",
+				"subchartb/templates/service.yaml":  "5654e03d922e8ec49649b4bbda9dfc9e643b3b7c9c18b602cc7e26fd36a39c2a",
+			},
+			expectedError: "",
+		},
+	}
+
+	h := sha256.New()
+
+	for _, testCase := range testCases {
+		t.Run(testCase.label, func(t *testing.T) {
+			tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename")
+			out, err := tc.GenerateKubernetesArtifacts(testCase.chartPath, testCase.valueFiles,
+				testCase.values)
+			if err != nil {
+				if testCase.expectedError == "" {
+					t.Fatalf("Got an error %s", err)
+				}
+				if strings.Contains(err.Error(), testCase.expectedError) == false {
+					t.Fatalf("Got unexpected error message %s", err)
+				}
+			} else {
+				//Compute the hash of returned data and compare
+				for _, v := range out {
+					for _, f := range v {
+						data, err := ioutil.ReadFile(f)
+						if err != nil {
+							t.Errorf("Unable to read file %s", v)
+						}
+						h.Write(data)
+						gotHash := fmt.Sprintf("%x", h.Sum(nil))
+						h.Reset()
+
+						//Find the right hash from expectedHashMap
+						expectedHash := ""
+						for k1, v1 := range testCase.expectedHashMap {
+							if strings.Contains(f, k1) == true {
+								expectedHash = v1
+								break
+							}
+						}
+						if gotHash != expectedHash {
+							t.Fatalf("Got unexpected hash for %s", f)
+						}
+					}
+				}
+			}
+		})
+	}
+}