From c8ba8f21b68b64b4068f188614dd7c891edf035f Mon Sep 17 00:00:00 2001 From: Rajamohan Raj Date: Fri, 3 Apr 2020 07:43:11 +0000 Subject: Test cases for resolving the helm templates Added test cases for overriding values and resolving the helm charts. Also addressed the merge conflicts and review comments Issue-ID: MULTICLOUD-1041 Signed-off-by: Rajamohan Raj Change-Id: I511e8e2e71c60e878df434370fc053f09cda1f66 --- src/orchestrator/utils/helm/helm.go | 35 +++-- src/orchestrator/utils/helm/helm_test.go | 203 ++++++++++++++++++++++++++++ src/orchestrator/utils/helm/profile_yaml.go | 109 +++++++++++++++ 3 files changed, 335 insertions(+), 12 deletions(-) create mode 100644 src/orchestrator/utils/helm/helm_test.go create mode 100644 src/orchestrator/utils/helm/profile_yaml.go (limited to 'src/orchestrator/utils/helm') diff --git a/src/orchestrator/utils/helm/helm.go b/src/orchestrator/utils/helm/helm.go index f0d15fbf..80cdfe5a 100644 --- a/src/orchestrator/utils/helm/helm.go +++ b/src/orchestrator/utils/helm/helm.go @@ -19,7 +19,6 @@ package helm import ( "bytes" utils "github.com/onap/multicloud-k8s/src/orchestrator/utils" - "github.com/onap/multicloud-k8s/src/orchestrator/utils/types" pkgerrors "github.com/pkg/errors" "log" @@ -46,6 +45,15 @@ import ( "k8s.io/helm/pkg/timeconv" ) +//KubernetesResourceTemplate - Represents the template that is used to create a particular +//resource in Kubernetes +type KubernetesResourceTemplate struct { + // Tracks the apiVersion and Kind of the resource + GVK schema.GroupVersionKind + // Path to the file that contains the resource info + FilePath string +} + // Template is the interface for all helm templating commands // Any backend implementation will implement this interface and will // access the functionality via this. @@ -144,10 +152,10 @@ func (h *TemplateClient) mergeValues(dest map[string]interface{}, src map[string // GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string, - values []string) ([]types.KubernetesResourceTemplate, error) { + values []string) ([]KubernetesResourceTemplate, error) { var outputDir, chartPath, namespace, releaseName string - var retData []types.KubernetesResourceTemplate + var retData []KubernetesResourceTemplate releaseName = h.releaseName namespace = h.kubeNameSpace @@ -255,7 +263,7 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile return retData, err } - kres := types.KubernetesResourceTemplate{ + kres := KubernetesResourceTemplate{ GVK: gvk, FilePath: mfilePath, } @@ -281,13 +289,20 @@ func getGroupVersionKind(data string) (schema.GroupVersionKind, error) { // Resolver is an interface exposes the helm related functionalities type Resolver interface { - Resolve(appContent, appProfileContent []byte, overrideValuesOfAppStr []string, rName string) ([]types.KubernetesResourceTemplate, error) + Resolve(appContent, appProfileContent []byte, overrideValuesOfAppStr []string, appName string) ([]KubernetesResourceTemplate, error) } // Resolve function -func (h *TemplateClient) Resolve(appContent []byte, appProfileContent []byte, overrideValuesOfAppStr []string, rName, appName string) ([]types.KubernetesResourceTemplate, error) { +func (h *TemplateClient) Resolve(appContent []byte, appProfileContent []byte, overrideValuesOfAppStr []string, appName string) ([]KubernetesResourceTemplate, error) { + + var sortedTemplates []KubernetesResourceTemplate - var sortedTemplates []types.KubernetesResourceTemplate + //chartBasePath is the tmp path where the appContent(rawHelmCharts) is extracted. + chartBasePath, err := utils.ExtractTarBall(bytes.NewBuffer(appContent)) + if err != nil { + return sortedTemplates, pkgerrors.Wrap(err, "Extracting appContent") + } + log.Printf("The chartBasePath :: %s", chartBasePath) //prPath is the tmp path where the appProfileContent is extracted. prPath, err := utils.ExtractTarBall(bytes.NewBuffer(appProfileContent)) @@ -296,16 +311,12 @@ func (h *TemplateClient) Resolve(appContent []byte, appProfileContent []byte, ov } log.Printf("The profile path:: %s", prPath) - prYamlClient, err := utils.ProcessProfileYaml(prPath, h.manifestName) + prYamlClient, err := ProcessProfileYaml(prPath, h.manifestName) if err != nil { return sortedTemplates, pkgerrors.Wrap(err, "Processing Profile Manifest") } log.Println("Got the profileYamlClient..") - //chartBasePath is the tmp path where the appContent(rawHelmCharts) is extracted. - chartBasePath, err := utils.ExtractTarBall(bytes.NewBuffer(appContent)) - log.Printf("The chartBasePath :: %s", chartBasePath) - err = prYamlClient.CopyConfigurationOverrides(chartBasePath) if err != nil { return sortedTemplates, pkgerrors.Wrap(err, "Copying configresources to chart") diff --git a/src/orchestrator/utils/helm/helm_test.go b/src/orchestrator/utils/helm/helm_test.go new file mode 100644 index 00000000..e9442e8a --- /dev/null +++ b/src/orchestrator/utils/helm/helm_test.go @@ -0,0 +1,203 @@ +/* + * Copyright 2020 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package helm + +import ( + "crypto/sha256" + "fmt" + + "io/ioutil" + "path/filepath" + "strings" + "testing" +) + +func TestProcessValues(t *testing.T) { + + chartDir := "../../mock_files/mock_charts/testchart2" + profileDir := "../../mock_files/mock_profiles/profile1" + + testCases := []struct { + label string + valueFiles []string + values []string + expectedHash string + expectedError string + }{ + { + label: "Process Values with Value Files Override", + valueFiles: []string{ + filepath.Join(chartDir, "values.yaml"), + filepath.Join(profileDir, "override_values.yaml"), + }, + //Hash of a combined values.yaml file that is expected + expectedHash: "c18a70f426933de3c051c996dc34fd537d0131b2d13a2112a2ecff674db6c2f9", + expectedError: "", + }, + { + label: "Process Values with Values Pair Override", + valueFiles: []string{ + filepath.Join(chartDir, "values.yaml"), + }, + //Use the same convention as specified in helm template --set + values: []string{ + "service.externalPort=82", + }, + //Hash of a combined values.yaml file that is expected + expectedHash: "028a3521fc9f8777ea7e67a6de0c51f2c875b88ca91734999657f0ca924ddb7a", + expectedError: "", + }, + { + label: "Process Values with Both Overrides", + valueFiles: []string{ + filepath.Join(chartDir, "values.yaml"), + filepath.Join(profileDir, "override_values.yaml"), + }, + //Use the same convention as specified in helm template --set + //Key takes precedence over the value from override_values.yaml + values: []string{ + "service.externalPort=82", + }, + //Hash of a combined values.yaml file that is expected + expectedHash: "516fab4ab7b76ba2ff35a97c2a79b74302543f532857b945f2fe25e717e755be", + expectedError: "", + }, + { + label: "Process complex Pair Override", + values: []string{ + "name={a,b,c}", + "servers[0].port=80", + }, + expectedError: "", + expectedHash: "50d9401b003f65c1ccfd1c5155106fff88c8201ab8b7d66bd6ffa4fe2883bead", + }, + } + + h := sha256.New() + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename", "manifest.yaml") + out, err := tc.processValues(testCase.valueFiles, testCase.values) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Got an error %s", err) + } + if strings.Contains(err.Error(), testCase.expectedError) == false { + t.Fatalf("Got unexpected error message %s", err) + } + } else { + //Compute the hash of returned data and compare + h.Write(out) + gotHash := fmt.Sprintf("%x", h.Sum(nil)) + h.Reset() + if gotHash != testCase.expectedHash { + t.Fatalf("Got unexpected hash '%s' of values.yaml:\n%s", gotHash, out) + } + } + }) + } +} + +func TestGenerateKubernetesArtifacts(t *testing.T) { + + chartDir := "../../mock_files/mock_charts/testchart2" + profileDir := "../../mock_files/mock_profiles/profile1" + + testCases := []struct { + label string + chartPath string + valueFiles []string + values []string + expectedHashMap map[string]string + expectedError string + }{ + { + label: "Generate artifacts without any overrides", + chartPath: chartDir, + valueFiles: []string{}, + values: []string{}, + //sha256 hash of the evaluated templates in each chart + expectedHashMap: map[string]string{ + "testchart2/templates/service.yaml": "fdd6a2b6795486f0dd1d8c44379afb5ffe4072c09f9cf6594738e8ded4dd872d", + "subcharta/templates/service.yaml": "570389588fffdb7193ab265888d781f3d751f3a40362533344f9aa7bb93a8bb0", + "subchartb/templates/service.yaml": "5654e03d922e8ec49649b4bbda9dfc9e643b3b7c9c18b602cc7e26fd36a39c2a", + }, + expectedError: "", + }, + { + label: "Generate artifacts with overrides", + chartPath: chartDir, + valueFiles: []string{ + filepath.Join(profileDir, "override_values.yaml"), + }, + values: []string{ + "service.externalPort=82", + }, + //sha256 hash of the evaluated templates in each chart + expectedHashMap: map[string]string{ + "testchart2/templates/service.yaml": "2bb96e791ecb6a3404bc5de3f6c4182aed881630269e2aa6766df38b0f852724", + "subcharta/templates/service.yaml": "570389588fffdb7193ab265888d781f3d751f3a40362533344f9aa7bb93a8bb0", + "subchartb/templates/service.yaml": "5654e03d922e8ec49649b4bbda9dfc9e643b3b7c9c18b602cc7e26fd36a39c2a", + }, + expectedError: "", + }, + } + + h := sha256.New() + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename", "manifest.yaml") + out, err := tc.GenerateKubernetesArtifacts(testCase.chartPath, testCase.valueFiles, + testCase.values) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Got an error %s", err) + } + if strings.Contains(err.Error(), testCase.expectedError) == false { + t.Fatalf("Got unexpected error message %s", err) + } + } else { + //Compute the hash of returned data and compare + for _, v := range out { + f := v.FilePath + data, err := ioutil.ReadFile(f) + if err != nil { + t.Errorf("Unable to read file %s", v) + } + h.Write(data) + gotHash := fmt.Sprintf("%x", h.Sum(nil)) + h.Reset() + + //Find the right hash from expectedHashMap + expectedHash := "" + for k1, v1 := range testCase.expectedHashMap { + if strings.Contains(f, k1) == true { + expectedHash = v1 + break + } + } + if gotHash != expectedHash { + t.Fatalf("Got unexpected hash for %s", f) + } + } + } + }) + } + +} diff --git a/src/orchestrator/utils/helm/profile_yaml.go b/src/orchestrator/utils/helm/profile_yaml.go new file mode 100644 index 00000000..e72fdbdc --- /dev/null +++ b/src/orchestrator/utils/helm/profile_yaml.go @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package helm + +import ( + "io/ioutil" + "log" + "path/filepath" + + "github.com/ghodss/yaml" + pkgerrors "github.com/pkg/errors" +) + +/* +#Sample Yaml format for profile manifest.yaml +--- +version: v1 +type: + values: "values_override.yaml" + configresource: + - filepath: config.yaml + chartpath: chart/config/resources/config.yaml + - filepath: config2.yaml + chartpath: chart/config/resources/config2.yaml +*/ + +type overrideFiles struct { + FilePath string `yaml:"filepath"` + ChartPath string `yaml:"chartpath"` +} + +type supportedOverrides struct { + ConfigResource []overrideFiles `yaml:"configresource"` + Values string `yaml:"values"` +} + +type profileOverride struct { + Version string `yaml:"version"` + Type supportedOverrides `yaml:"type"` +} + +type ProfileYamlClient struct { + path string + override profileOverride +} + +func (p ProfileYamlClient) Print() { + log.Println(p.override) +} + +//GetValues returns a path to the override values.yam +//that was part of the profile +func (p ProfileYamlClient) GetValues() string { + return filepath.Join(p.path, p.override.Type.Values) +} + +//CopyConfigurationOverrides copies the various files that are +//provided as overrides to their corresponding locations within +//the destination chart. +func (p ProfileYamlClient) CopyConfigurationOverrides(chartPath string) error { + + //Iterate over each configresource and copy that file into + //the respective path in the chart. + for _, v := range p.override.Type.ConfigResource { + data, err := ioutil.ReadFile(filepath.Join(p.path, v.FilePath)) + if err != nil { + return pkgerrors.Wrap(err, "Reading configuration file") + } + err = ioutil.WriteFile(filepath.Join(chartPath, v.ChartPath), data, 0644) + if err != nil { + return pkgerrors.Wrap(err, "Writing configuration file into chartpath") + } + } + + return nil +} + +//ProcessProfileYaml parses the manifest.yaml file that is part of the profile +//package and creates the appropriate structures out of it. +func ProcessProfileYaml(fpath string, manifestFileName string) (ProfileYamlClient, error) { + + p := filepath.Join(fpath, manifestFileName) + data, err := ioutil.ReadFile(p) + if err != nil { + return ProfileYamlClient{}, pkgerrors.Wrap(err, "Reading manifest file") + } + + out := profileOverride{} + err = yaml.Unmarshal(data, &out) + if err != nil { + return ProfileYamlClient{}, pkgerrors.Wrap(err, "Marshaling manifest yaml file") + } + + return ProfileYamlClient{path: fpath, override: out}, nil +} -- cgit 1.2.3-korg