diff options
Diffstat (limited to 'src/orchestrator/utils')
| -rw-r--r-- | src/orchestrator/utils/helm/helm.go | 320 | ||||
| -rw-r--r-- | src/orchestrator/utils/profile_yaml.go | 109 | ||||
| -rw-r--r-- | src/orchestrator/utils/types/types.go | 41 | ||||
| -rw-r--r-- | src/orchestrator/utils/util-functions.go | 116 |
4 files changed, 586 insertions, 0 deletions
diff --git a/src/orchestrator/utils/helm/helm.go b/src/orchestrator/utils/helm/helm.go new file mode 100644 index 00000000..f0d15fbf --- /dev/null +++ b/src/orchestrator/utils/helm/helm.go @@ -0,0 +1,320 @@ +/* + * Copyright 2020 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package helm + +import ( + "bytes" + utils "github.com/onap/multicloud-k8s/src/orchestrator/utils" + "github.com/onap/multicloud-k8s/src/orchestrator/utils/types" + + pkgerrors "github.com/pkg/errors" + "log" + + "fmt" + "io/ioutil" + "k8s.io/helm/pkg/strvals" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/ghodss/yaml" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/apimachinery/pkg/util/validation" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/helm/pkg/chartutil" + "k8s.io/helm/pkg/manifest" + "k8s.io/helm/pkg/proto/hapi/chart" + "k8s.io/helm/pkg/releaseutil" + "k8s.io/helm/pkg/renderutil" + "k8s.io/helm/pkg/tiller" + "k8s.io/helm/pkg/timeconv" +) + +// Template is the interface for all helm templating commands +// Any backend implementation will implement this interface and will +// access the functionality via this. +type Template interface { + GenerateKubernetesArtifacts( + chartPath string, + valueFiles []string, + values []string) (map[string][]string, error) +} + +// TemplateClient implements the Template interface +// It will also be used to maintain any localized state +type TemplateClient struct { + whitespaceRegex *regexp.Regexp + kubeVersion string + kubeNameSpace string + releaseName string + manifestName string +} + +// NewTemplateClient returns a new instance of TemplateClient +func NewTemplateClient(k8sversion, namespace, releasename, manifestFileName string) *TemplateClient { + return &TemplateClient{ + whitespaceRegex: regexp.MustCompile(`^\s*$`), + // defaultKubeVersion is the default value of --kube-version flag + kubeVersion: k8sversion, + kubeNameSpace: namespace, + releaseName: releasename, + manifestName: manifestFileName, + } +} + +// Combines valueFiles and values into a single values stream. +// values takes precedence over valueFiles +func (h *TemplateClient) processValues(valueFiles []string, values []string) ([]byte, error) { + base := map[string]interface{}{} + + //Values files that are used for overriding the chart + for _, filePath := range valueFiles { + currentMap := map[string]interface{}{} + + var bytes []byte + var err error + if strings.TrimSpace(filePath) == "-" { + bytes, err = ioutil.ReadAll(os.Stdin) + } else { + bytes, err = ioutil.ReadFile(filePath) + } + + if err != nil { + return []byte{}, err + } + + if err := yaml.Unmarshal(bytes, ¤tMap); err != nil { + return []byte{}, fmt.Errorf("failed to parse %s: %s", filePath, err) + } + // Merge with the previous map + base = h.mergeValues(base, currentMap) + } + + //User specified value. Similar to ones provided by -x + for _, value := range values { + if err := strvals.ParseInto(value, base); err != nil { + return []byte{}, fmt.Errorf("failed parsing --set data: %s", err) + } + } + + return yaml.Marshal(base) +} + +func (h *TemplateClient) mergeValues(dest map[string]interface{}, src map[string]interface{}) map[string]interface{} { + for k, v := range src { + // If the key doesn't exist already, then just set the key to that value + if _, exists := dest[k]; !exists { + dest[k] = v + continue + } + nextMap, ok := v.(map[string]interface{}) + // If it isn't another map, overwrite the value + if !ok { + dest[k] = v + continue + } + // Edge case: If the key exists in the destination, but isn't a map + destMap, isMap := dest[k].(map[string]interface{}) + // If the source map has a map for this key, prefer it + if !isMap { + dest[k] = v + continue + } + // If we got to this point, it is a map in both, so merge them + dest[k] = h.mergeValues(destMap, nextMap) + } + return dest +} + +// GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template +func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string, + values []string) ([]types.KubernetesResourceTemplate, error) { + + var outputDir, chartPath, namespace, releaseName string + var retData []types.KubernetesResourceTemplate + + releaseName = h.releaseName + namespace = h.kubeNameSpace + + // verify chart path exists + if _, err := os.Stat(inputPath); err == nil { + if chartPath, err = filepath.Abs(inputPath); err != nil { + return retData, err + } + } else { + return retData, err + } + + //Create a temp directory in the system temp folder + outputDir, err := ioutil.TempDir("", "helm-tmpl-") + if err != nil { + return retData, pkgerrors.Wrap(err, "Got error creating temp dir") + } + log.Printf("The o/p dir:: %s ", outputDir) + + if namespace == "" { + namespace = "default" + } + + // get combined values and create config + rawVals, err := h.processValues(valueFiles, values) + if err != nil { + return retData, err + } + config := &chart.Config{Raw: string(rawVals), Values: map[string]*chart.Value{}} + + if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 { + return retData, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";")) + } + + // Check chart requirements to make sure all dependencies are present in /charts + c, err := chartutil.Load(chartPath) + if err != nil { + return retData, pkgerrors.Errorf("Got error: %s", err.Error()) + } + + renderOpts := renderutil.Options{ + ReleaseOptions: chartutil.ReleaseOptions{ + Name: releaseName, + IsInstall: true, + IsUpgrade: false, + Time: timeconv.Now(), + Namespace: namespace, + }, + KubeVersion: h.kubeVersion, + } + + renderedTemplates, err := renderutil.Render(c, config, renderOpts) + if err != nil { + return retData, err + } + + newRenderedTemplates := make(map[string]string) + + //Some manifests can contain multiple yaml documents + //This step is splitting them up into multiple files + //Each file contains only a single k8s kind + for k, v := range renderedTemplates { + //Splits into manifest-0, manifest-1 etc + if filepath.Base(k) == "NOTES.txt" { + continue + } + rmap := releaseutil.SplitManifests(v) + count := 0 + for _, v1 := range rmap { + key := fmt.Sprintf("%s-%d", k, count) + newRenderedTemplates[key] = v1 + count = count + 1 + } + } + + listManifests := manifest.SplitManifests(newRenderedTemplates) + var manifestsToRender []manifest.Manifest + //render all manifests in the chart + manifestsToRender = listManifests + for _, m := range tiller.SortByKind(manifestsToRender) { + data := m.Content + b := filepath.Base(m.Name) + if b == "NOTES.txt" { + continue + } + if strings.HasPrefix(b, "_") { + continue + } + + // blank template after execution + if h.whitespaceRegex.MatchString(data) { + continue + } + + mfilePath := filepath.Join(outputDir, m.Name) + utils.EnsureDirectory(mfilePath) + err = ioutil.WriteFile(mfilePath, []byte(data), 0666) + if err != nil { + return retData, err + } + + gvk, err := getGroupVersionKind(data) + if err != nil { + return retData, err + } + + kres := types.KubernetesResourceTemplate{ + GVK: gvk, + FilePath: mfilePath, + } + retData = append(retData, kres) + } + return retData, nil +} + +func getGroupVersionKind(data string) (schema.GroupVersionKind, error) { + out, err := k8syaml.ToJSON([]byte(data)) + if err != nil { + return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Converting yaml to json") + } + + simpleMeta := json.SimpleMetaFactory{} + gvk, err := simpleMeta.Interpret(out) + if err != nil { + return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Parsing apiversion and kind") + } + + return *gvk, nil +} + +// Resolver is an interface exposes the helm related functionalities +type Resolver interface { + Resolve(appContent, appProfileContent []byte, overrideValuesOfAppStr []string, rName string) ([]types.KubernetesResourceTemplate, error) +} + +// Resolve function +func (h *TemplateClient) Resolve(appContent []byte, appProfileContent []byte, overrideValuesOfAppStr []string, rName, appName string) ([]types.KubernetesResourceTemplate, error) { + + var sortedTemplates []types.KubernetesResourceTemplate + + //prPath is the tmp path where the appProfileContent is extracted. + prPath, err := utils.ExtractTarBall(bytes.NewBuffer(appProfileContent)) + if err != nil { + return sortedTemplates, pkgerrors.Wrap(err, "Extracting Profile Content") + } + log.Printf("The profile path:: %s", prPath) + + prYamlClient, err := utils.ProcessProfileYaml(prPath, h.manifestName) + if err != nil { + return sortedTemplates, pkgerrors.Wrap(err, "Processing Profile Manifest") + } + log.Println("Got the profileYamlClient..") + + //chartBasePath is the tmp path where the appContent(rawHelmCharts) is extracted. + chartBasePath, err := utils.ExtractTarBall(bytes.NewBuffer(appContent)) + log.Printf("The chartBasePath :: %s", chartBasePath) + + err = prYamlClient.CopyConfigurationOverrides(chartBasePath) + if err != nil { + return sortedTemplates, pkgerrors.Wrap(err, "Copying configresources to chart") + } + + chartPath := filepath.Join(chartBasePath, appName) + sortedTemplates, err = h.GenerateKubernetesArtifacts(chartPath, []string{prYamlClient.GetValues()}, overrideValuesOfAppStr) + if err != nil { + return sortedTemplates, pkgerrors.Wrap(err, "Generate final k8s yaml") + } + return sortedTemplates, nil +} diff --git a/src/orchestrator/utils/profile_yaml.go b/src/orchestrator/utils/profile_yaml.go new file mode 100644 index 00000000..91687c0a --- /dev/null +++ b/src/orchestrator/utils/profile_yaml.go @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "io/ioutil" + "log" + "path/filepath" + + "github.com/ghodss/yaml" + pkgerrors "github.com/pkg/errors" +) + +/* +#Sample Yaml format for profile manifest.yaml +--- +version: v1 +type: + values: "values_override.yaml" + configresource: + - filepath: config.yaml + chartpath: chart/config/resources/config.yaml + - filepath: config2.yaml + chartpath: chart/config/resources/config2.yaml +*/ + +type overrideFiles struct { + FilePath string `yaml:"filepath"` + ChartPath string `yaml:"chartpath"` +} + +type supportedOverrides struct { + ConfigResource []overrideFiles `yaml:"configresource"` + Values string `yaml:"values"` +} + +type profileOverride struct { + Version string `yaml:"version"` + Type supportedOverrides `yaml:"type"` +} + +type ProfileYamlClient struct { + path string + override profileOverride +} + +func (p ProfileYamlClient) Print() { + log.Println(p.override) +} + +//GetValues returns a path to the override values.yam +//that was part of the profile +func (p ProfileYamlClient) GetValues() string { + return filepath.Join(p.path, p.override.Type.Values) +} + +//CopyConfigurationOverrides copies the various files that are +//provided as overrides to their corresponding locations within +//the destination chart. +func (p ProfileYamlClient) CopyConfigurationOverrides(chartPath string) error { + + //Iterate over each configresource and copy that file into + //the respective path in the chart. + for _, v := range p.override.Type.ConfigResource { + data, err := ioutil.ReadFile(filepath.Join(p.path, v.FilePath)) + if err != nil { + return pkgerrors.Wrap(err, "Reading configuration file") + } + err = ioutil.WriteFile(filepath.Join(chartPath, v.ChartPath), data, 0644) + if err != nil { + return pkgerrors.Wrap(err, "Writing configuration file into chartpath") + } + } + + return nil +} + +//ProcessProfileYaml parses the manifest.yaml file that is part of the profile +//package and creates the appropriate structures out of it. +func ProcessProfileYaml(fpath string, manifestFileName string) (ProfileYamlClient, error) { + + p := filepath.Join(fpath, manifestFileName) + data, err := ioutil.ReadFile(p) + if err != nil { + return ProfileYamlClient{}, pkgerrors.Wrap(err, "Reading manifest file") + } + + out := profileOverride{} + err = yaml.Unmarshal(data, &out) + if err != nil { + return ProfileYamlClient{}, pkgerrors.Wrap(err, "Marshaling manifest yaml file") + } + + return ProfileYamlClient{path: fpath, override: out}, nil +} diff --git a/src/orchestrator/utils/types/types.go b/src/orchestrator/utils/types/types.go new file mode 100644 index 00000000..4ffb4180 --- /dev/null +++ b/src/orchestrator/utils/types/types.go @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// KubernetesResourceTemplate - Represents the template that is used to create a particular +// resource in Kubernetes +type KubernetesResourceTemplate struct { + // Tracks the apiVersion and Kind of the resource + GVK schema.GroupVersionKind + // Path to the file that contains the resource info + FilePath string +} + +// KubernetesResource is the resource that is created in Kubernetes +// It is the type that will be used for tracking a resource. +// Any future information such as status, time can be added here +// for tracking. +type KubernetesResource struct { + // Tracks the apiVersion and Kind of the resource + GVK schema.GroupVersionKind + // Name of resource in Kubernetes + Name string +} diff --git a/src/orchestrator/utils/util-functions.go b/src/orchestrator/utils/util-functions.go new file mode 100644 index 00000000..13c78ba4 --- /dev/null +++ b/src/orchestrator/utils/util-functions.go @@ -0,0 +1,116 @@ +/* + * Copyright 2020 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "archive/tar" + "compress/gzip" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + + pkgerrors "github.com/pkg/errors" +) + +//ExtractTarBall provides functionality to extract a tar.gz file +//into a temporary location for later use. +//It returns the path to the new location +func ExtractTarBall(r io.Reader) (string, error) { + //Check if it is a valid gz + gzf, err := gzip.NewReader(r) + if err != nil { + return "", pkgerrors.Wrap(err, "Invalid gzip format") + } + + //Check if it is a valid tar file + //Unfortunately this can only be done by inspecting all the tar contents + tarR := tar.NewReader(gzf) + first := true + + outDir, _ := ioutil.TempDir("", "k8s-ext-") + + for true { + header, err := tarR.Next() + + if err == io.EOF { + //Check if we have just a gzip file without a tar archive inside + if first { + return "", pkgerrors.New("Empty or non-existant Tar file found") + } + //End of archive + break + } + + if err != nil { + return "", pkgerrors.Wrap(err, "Error reading tar file") + } + + target := filepath.Join(outDir, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + if _, err := os.Stat(target); err != nil { + // Using 755 read, write, execute for owner + // groups and others get read and execute permissions + // on the folder. + if err := os.MkdirAll(target, 0755); err != nil { + return "", pkgerrors.Wrap(err, "Creating directory") + } + } + case tar.TypeReg: + if target == outDir { // Handle '.' substituted to '' entry + continue + } + + err = EnsureDirectory(target) + if err != nil { + return "", pkgerrors.Wrap(err, "Creating Directory") + } + + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return "", pkgerrors.Wrap(err, "Creating file") + } + + // copy over contents + if _, err := io.Copy(f, tarR); err != nil { + return "", pkgerrors.Wrap(err, "Copying file content") + } + + // close for each file instead of a defer for all + // at the end of the function + f.Close() + } + + first = false + } + + return outDir, nil +} + +//EnsureDirectory makes sure that the directories specified in the path exist +//If not, it will create them, if possible. +func EnsureDirectory(f string) error { + base := path.Dir(f) + _, err := os.Stat(base) + if err != nil && !os.IsNotExist(err) { + return err + } + return os.MkdirAll(base, 0755) +} |