summaryrefslogtreecommitdiffstats
path: root/src/k8splugin/internal
diff options
context:
space:
mode:
authorKiran Kamineni <kiran.k.kamineni@intel.com>2019-01-08 15:50:12 -0800
committerKiran Kamineni <kiran.k.kamineni@intel.com>2019-03-12 15:24:15 -0700
commit909d0d1c9a6f95dd1714b26477238d8997b3e20e (patch)
tree86c72e5518a913a6230233c1aa6de51d4f5137da /src/k8splugin/internal
parent98e83cc404987387466721aedca3fc6c97cab40b (diff)
Add support for helm templates
Add support for evaluating helm templates The interface provides a way to get a map which contains a mapping from kubernetes kind to the corresponding yaml file that defines a resource of that kind. This map is then provided to the instantiation code at instantiation time to create in the kubernetes cluster. P5: Use filepath.join instead of strings.join P9: rebase with new folder structure P10: moved the helm code into its own package. P12: Add unit tests update the go.mod to use latest docker version Issue-ID: MULTICLOUD-291 Change-Id: Ie75f5c616cc0cdc3e0ace49ff2c2f6c356a4c0d1 Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>
Diffstat (limited to 'src/k8splugin/internal')
-rw-r--r--src/k8splugin/internal/helm/helm.go259
-rw-r--r--src/k8splugin/internal/helm/helm_test.go195
2 files changed, 454 insertions, 0 deletions
diff --git a/src/k8splugin/internal/helm/helm.go b/src/k8splugin/internal/helm/helm.go
new file mode 100644
index 00000000..65a36d6b
--- /dev/null
+++ b/src/k8splugin/internal/helm/helm.go
@@ -0,0 +1,259 @@
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helm
+
+import (
+ "fmt"
+ "io/ioutil"
+ "k8s.io/helm/pkg/strvals"
+ "os"
+ "path"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "github.com/ghodss/yaml"
+ pkgerrors "github.com/pkg/errors"
+
+ "k8s.io/apimachinery/pkg/util/validation"
+ "k8s.io/helm/pkg/chartutil"
+ "k8s.io/helm/pkg/manifest"
+ "k8s.io/helm/pkg/proto/hapi/chart"
+ "k8s.io/helm/pkg/releaseutil"
+ "k8s.io/helm/pkg/renderutil"
+ "k8s.io/helm/pkg/tiller"
+ "k8s.io/helm/pkg/timeconv"
+)
+
+// Template is the interface for all helm templating commands
+// Any backend implementation will implement this interface and will
+// access the functionality via this.
+type Template interface {
+ GenerateKubernetesArtifacts(
+ chartPath string,
+ valueFiles []string,
+ values []string) (map[string][]string, error)
+}
+
+// TemplateClient implements the Template interface
+// It will also be used to maintain any localized state
+type TemplateClient struct {
+ whitespaceRegex *regexp.Regexp
+ kubeVersion string
+ kubeNameSpace string
+ releaseName string
+}
+
+// NewTemplateClient returns a new instance of TemplateClient
+func NewTemplateClient(k8sversion, namespace, releasename string) *TemplateClient {
+ return &TemplateClient{
+ whitespaceRegex: regexp.MustCompile(`^\s*$`),
+ // defaultKubeVersion is the default value of --kube-version flag
+ kubeVersion: k8sversion,
+ kubeNameSpace: namespace,
+ releaseName: releasename,
+ }
+}
+
+// Combines valueFiles and values into a single values stream.
+// values takes precedence over valueFiles
+func (h *TemplateClient) processValues(valueFiles []string, values []string) ([]byte, error) {
+ base := map[string]interface{}{}
+
+ //Values files that are used for overriding the chart
+ for _, filePath := range valueFiles {
+ currentMap := map[string]interface{}{}
+
+ var bytes []byte
+ var err error
+ if strings.TrimSpace(filePath) == "-" {
+ bytes, err = ioutil.ReadAll(os.Stdin)
+ } else {
+ bytes, err = ioutil.ReadFile(filePath)
+ }
+
+ if err != nil {
+ return []byte{}, err
+ }
+
+ if err := yaml.Unmarshal(bytes, &currentMap); err != nil {
+ return []byte{}, fmt.Errorf("failed to parse %s: %s", filePath, err)
+ }
+ // Merge with the previous map
+ base = h.mergeValues(base, currentMap)
+ }
+
+ //User specified value. Similar to ones provided by -x
+ for _, value := range values {
+ if err := strvals.ParseInto(value, base); err != nil {
+ return []byte{}, fmt.Errorf("failed parsing --set data: %s", err)
+ }
+ }
+
+ return yaml.Marshal(base)
+}
+
+func (h *TemplateClient) mergeValues(dest map[string]interface{}, src map[string]interface{}) map[string]interface{} {
+ for k, v := range src {
+ // If the key doesn't exist already, then just set the key to that value
+ if _, exists := dest[k]; !exists {
+ dest[k] = v
+ continue
+ }
+ nextMap, ok := v.(map[string]interface{})
+ // If it isn't another map, overwrite the value
+ if !ok {
+ dest[k] = v
+ continue
+ }
+ // Edge case: If the key exists in the destination, but isn't a map
+ destMap, isMap := dest[k].(map[string]interface{})
+ // If the source map has a map for this key, prefer it
+ if !isMap {
+ dest[k] = v
+ continue
+ }
+ // If we got to this point, it is a map in both, so merge them
+ dest[k] = h.mergeValues(destMap, nextMap)
+ }
+ return dest
+}
+
+func (h *TemplateClient) ensureDirectory(f string) error {
+ base := path.Dir(f)
+ _, err := os.Stat(base)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ return os.MkdirAll(base, 0755)
+}
+
+// GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template
+func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string, values []string) (map[string][]string, error) {
+
+ var outputDir, chartPath, namespace, releaseName string
+ var retData map[string][]string
+
+ releaseName = h.releaseName
+ namespace = h.kubeNameSpace
+
+ // verify chart path exists
+ if _, err := os.Stat(inputPath); err == nil {
+ if chartPath, err = filepath.Abs(inputPath); err != nil {
+ return retData, err
+ }
+ } else {
+ return retData, err
+ }
+
+ //Create a temp directory in the system temp folder
+ outputDir, err := ioutil.TempDir("", "helm-tmpl-")
+ if err != nil {
+ return retData, pkgerrors.Wrap(err, "Got error creating temp dir")
+ }
+
+ if namespace == "" {
+ namespace = "default"
+ }
+
+ // get combined values and create config
+ rawVals, err := h.processValues(valueFiles, values)
+ if err != nil {
+ return retData, err
+ }
+ config := &chart.Config{Raw: string(rawVals), Values: map[string]*chart.Value{}}
+
+ if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 {
+ return retData, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";"))
+ }
+
+ // Check chart requirements to make sure all dependencies are present in /charts
+ c, err := chartutil.Load(chartPath)
+ if err != nil {
+ return retData, pkgerrors.Errorf("Got error: %s", err.Error())
+ }
+
+ renderOpts := renderutil.Options{
+ ReleaseOptions: chartutil.ReleaseOptions{
+ Name: releaseName,
+ IsInstall: true,
+ IsUpgrade: false,
+ Time: timeconv.Now(),
+ Namespace: namespace,
+ },
+ KubeVersion: h.kubeVersion,
+ }
+
+ renderedTemplates, err := renderutil.Render(c, config, renderOpts)
+ if err != nil {
+ return retData, err
+ }
+
+ newRenderedTemplates := make(map[string]string)
+
+ //Some manifests can contain multiple yaml documents
+ //This step is splitting them up into multiple files
+ //Each file contains only a single k8s kind
+ for k, v := range renderedTemplates {
+ //Splits into manifest-0, manifest-1 etc
+ if filepath.Base(k) == "NOTES.txt" {
+ continue
+ }
+ rmap := releaseutil.SplitManifests(v)
+ count := 0
+ for _, v1 := range rmap {
+ key := fmt.Sprintf("%s-%d", k, count)
+ newRenderedTemplates[key] = v1
+ count = count + 1
+ }
+ }
+
+ listManifests := manifest.SplitManifests(newRenderedTemplates)
+ var manifestsToRender []manifest.Manifest
+ //render all manifests in the chart
+ manifestsToRender = listManifests
+ retData = make(map[string][]string)
+ for _, m := range tiller.SortByKind(manifestsToRender) {
+ data := m.Content
+ b := filepath.Base(m.Name)
+ if b == "NOTES.txt" {
+ continue
+ }
+ if strings.HasPrefix(b, "_") {
+ continue
+ }
+
+ // blank template after execution
+ if h.whitespaceRegex.MatchString(data) {
+ continue
+ }
+
+ mfilePath := filepath.Join(outputDir, m.Name)
+ h.ensureDirectory(mfilePath)
+ err = ioutil.WriteFile(mfilePath, []byte(data), 0666)
+ if err != nil {
+ return retData, err
+ }
+
+ if val, ok := retData[m.Head.Kind]; ok {
+ retData[m.Head.Kind] = append(val, mfilePath)
+ } else {
+ retData[m.Head.Kind] = []string{mfilePath}
+ }
+ }
+ return retData, nil
+}
diff --git a/src/k8splugin/internal/helm/helm_test.go b/src/k8splugin/internal/helm/helm_test.go
new file mode 100644
index 00000000..a5bcd9c8
--- /dev/null
+++ b/src/k8splugin/internal/helm/helm_test.go
@@ -0,0 +1,195 @@
+// +build unit
+
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helm
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "io/ioutil"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestProcessValues(t *testing.T) {
+
+ chartDir := "../../mock_files/mock_charts/testchart1"
+ profileDir := "../../mock_files/mock_profiles/profile1"
+
+ testCases := []struct {
+ label string
+ valueFiles []string
+ values []string
+ expectedHash string
+ expectedError string
+ }{
+ {
+ label: "Process Values with Value Files Override",
+ valueFiles: []string{
+ filepath.Join(chartDir, "values.yaml"),
+ filepath.Join(profileDir, "override_values.yaml"),
+ },
+ //Hash of a combined values.yaml file that is expected
+ expectedHash: "c18a70f426933de3c051c996dc34fd537d0131b2d13a2112a2ecff674db6c2f9",
+ expectedError: "",
+ },
+ {
+ label: "Process Values with Values Pair Override",
+ valueFiles: []string{
+ filepath.Join(chartDir, "values.yaml"),
+ },
+ //Use the same convention as specified in helm template --set
+ values: []string{
+ "service.externalPort=82",
+ },
+ //Hash of a combined values.yaml file that is expected
+ expectedHash: "028a3521fc9f8777ea7e67a6de0c51f2c875b88ca91734999657f0ca924ddb7a",
+ expectedError: "",
+ },
+ {
+ label: "Process Values with Both Overrides",
+ valueFiles: []string{
+ filepath.Join(chartDir, "values.yaml"),
+ filepath.Join(profileDir, "override_values.yaml"),
+ },
+ //Use the same convention as specified in helm template --set
+ //Key takes precedence over the value from override_values.yaml
+ values: []string{
+ "service.externalPort=82",
+ },
+ //Hash of a combined values.yaml file that is expected
+ expectedHash: "516fab4ab7b76ba2ff35a97c2a79b74302543f532857b945f2fe25e717e755be",
+ expectedError: "",
+ },
+ }
+
+ h := sha256.New()
+
+ for _, testCase := range testCases {
+ t.Run(testCase.label, func(t *testing.T) {
+ tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename")
+ out, err := tc.processValues(testCase.valueFiles, testCase.values)
+ if err != nil {
+ if testCase.expectedError == "" {
+ t.Fatalf("Got an error %s", err)
+ }
+ if strings.Contains(err.Error(), testCase.expectedError) == false {
+ t.Fatalf("Got unexpected error message %s", err)
+ }
+ } else {
+ //Compute the hash of returned data and compare
+ h.Write(out)
+ gotHash := fmt.Sprintf("%x", h.Sum(nil))
+ h.Reset()
+ if gotHash != testCase.expectedHash {
+ t.Fatalf("Got unexpected values.yaml %s", out)
+ }
+ }
+ })
+ }
+}
+
+func TestGenerateKubernetesArtifacts(t *testing.T) {
+
+ chartDir := "../../mock_files/mock_charts/testchart1"
+ profileDir := "../../mock_files/mock_profiles/profile1"
+
+ testCases := []struct {
+ label string
+ chartPath string
+ valueFiles []string
+ values []string
+ expectedHashMap map[string]string
+ expectedError string
+ }{
+ {
+ label: "Generate artifacts without any overrides",
+ chartPath: chartDir,
+ valueFiles: []string{},
+ values: []string{},
+ //sha256 hash of the evaluated templates in each chart
+ expectedHashMap: map[string]string{
+ "testchart1/templates/service.yaml": "bbd7257d1f6ab958680e642a8fbbbea2002ebbaa9276fb51fbd71b4b66a772cc",
+ "subcharta/templates/service.yaml": "570389588fffdb7193ab265888d781f3d751f3a40362533344f9aa7bb93a8bb0",
+ "subchartb/templates/service.yaml": "5654e03d922e8ec49649b4bbda9dfc9e643b3b7c9c18b602cc7e26fd36a39c2a",
+ },
+ expectedError: "",
+ },
+ {
+ label: "Generate artifacts with overrides",
+ chartPath: chartDir,
+ valueFiles: []string{
+ filepath.Join(profileDir, "override_values.yaml"),
+ },
+ values: []string{
+ "service.externalPort=82",
+ },
+ //sha256 hash of the evaluated templates in each chart
+ expectedHashMap: map[string]string{
+ "testchart1/templates/service.yaml": "4c5aa5d38b763fe4730fc31a759c40566a99a9c51f5e0fc7f93473c9affc2ca8",
+ "subcharta/templates/service.yaml": "570389588fffdb7193ab265888d781f3d751f3a40362533344f9aa7bb93a8bb0",
+ "subchartb/templates/service.yaml": "5654e03d922e8ec49649b4bbda9dfc9e643b3b7c9c18b602cc7e26fd36a39c2a",
+ },
+ expectedError: "",
+ },
+ }
+
+ h := sha256.New()
+
+ for _, testCase := range testCases {
+ t.Run(testCase.label, func(t *testing.T) {
+ tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename")
+ out, err := tc.GenerateKubernetesArtifacts(testCase.chartPath, testCase.valueFiles,
+ testCase.values)
+ if err != nil {
+ if testCase.expectedError == "" {
+ t.Fatalf("Got an error %s", err)
+ }
+ if strings.Contains(err.Error(), testCase.expectedError) == false {
+ t.Fatalf("Got unexpected error message %s", err)
+ }
+ } else {
+ //Compute the hash of returned data and compare
+ for _, v := range out {
+ for _, f := range v {
+ data, err := ioutil.ReadFile(f)
+ if err != nil {
+ t.Errorf("Unable to read file %s", v)
+ }
+ h.Write(data)
+ gotHash := fmt.Sprintf("%x", h.Sum(nil))
+ h.Reset()
+
+ //Find the right hash from expectedHashMap
+ expectedHash := ""
+ for k1, v1 := range testCase.expectedHashMap {
+ if strings.Contains(f, k1) == true {
+ expectedHash = v1
+ break
+ }
+ }
+ if gotHash != expectedHash {
+ t.Fatalf("Got unexpected hash for %s", f)
+ }
+ }
+ }
+ }
+ })
+ }
+}