summaryrefslogtreecommitdiffstats
path: root/src/orchestrator/utils
diff options
context:
space:
mode:
Diffstat (limited to 'src/orchestrator/utils')
-rw-r--r--src/orchestrator/utils/helm/helm.go320
-rw-r--r--src/orchestrator/utils/profile_yaml.go109
-rw-r--r--src/orchestrator/utils/types/types.go41
-rw-r--r--src/orchestrator/utils/util-functions.go116
4 files changed, 586 insertions, 0 deletions
diff --git a/src/orchestrator/utils/helm/helm.go b/src/orchestrator/utils/helm/helm.go
new file mode 100644
index 00000000..f0d15fbf
--- /dev/null
+++ b/src/orchestrator/utils/helm/helm.go
@@ -0,0 +1,320 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helm
+
+import (
+ "bytes"
+ utils "github.com/onap/multicloud-k8s/src/orchestrator/utils"
+ "github.com/onap/multicloud-k8s/src/orchestrator/utils/types"
+
+ pkgerrors "github.com/pkg/errors"
+ "log"
+
+ "fmt"
+ "io/ioutil"
+ "k8s.io/helm/pkg/strvals"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "github.com/ghodss/yaml"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/runtime/serializer/json"
+ "k8s.io/apimachinery/pkg/util/validation"
+ k8syaml "k8s.io/apimachinery/pkg/util/yaml"
+ "k8s.io/helm/pkg/chartutil"
+ "k8s.io/helm/pkg/manifest"
+ "k8s.io/helm/pkg/proto/hapi/chart"
+ "k8s.io/helm/pkg/releaseutil"
+ "k8s.io/helm/pkg/renderutil"
+ "k8s.io/helm/pkg/tiller"
+ "k8s.io/helm/pkg/timeconv"
+)
+
+// Template is the interface for all helm templating commands
+// Any backend implementation will implement this interface and will
+// access the functionality via this.
+type Template interface {
+ GenerateKubernetesArtifacts(
+ chartPath string,
+ valueFiles []string,
+ values []string) (map[string][]string, error)
+}
+
+// TemplateClient implements the Template interface
+// It will also be used to maintain any localized state
+type TemplateClient struct {
+ whitespaceRegex *regexp.Regexp
+ kubeVersion string
+ kubeNameSpace string
+ releaseName string
+ manifestName string
+}
+
+// NewTemplateClient returns a new instance of TemplateClient
+func NewTemplateClient(k8sversion, namespace, releasename, manifestFileName string) *TemplateClient {
+ return &TemplateClient{
+ whitespaceRegex: regexp.MustCompile(`^\s*$`),
+ // defaultKubeVersion is the default value of --kube-version flag
+ kubeVersion: k8sversion,
+ kubeNameSpace: namespace,
+ releaseName: releasename,
+ manifestName: manifestFileName,
+ }
+}
+
+// Combines valueFiles and values into a single values stream.
+// values takes precedence over valueFiles
+func (h *TemplateClient) processValues(valueFiles []string, values []string) ([]byte, error) {
+ base := map[string]interface{}{}
+
+ //Values files that are used for overriding the chart
+ for _, filePath := range valueFiles {
+ currentMap := map[string]interface{}{}
+
+ var bytes []byte
+ var err error
+ if strings.TrimSpace(filePath) == "-" {
+ bytes, err = ioutil.ReadAll(os.Stdin)
+ } else {
+ bytes, err = ioutil.ReadFile(filePath)
+ }
+
+ if err != nil {
+ return []byte{}, err
+ }
+
+ if err := yaml.Unmarshal(bytes, &currentMap); err != nil {
+ return []byte{}, fmt.Errorf("failed to parse %s: %s", filePath, err)
+ }
+ // Merge with the previous map
+ base = h.mergeValues(base, currentMap)
+ }
+
+ //User specified value. Similar to ones provided by -x
+ for _, value := range values {
+ if err := strvals.ParseInto(value, base); err != nil {
+ return []byte{}, fmt.Errorf("failed parsing --set data: %s", err)
+ }
+ }
+
+ return yaml.Marshal(base)
+}
+
+func (h *TemplateClient) mergeValues(dest map[string]interface{}, src map[string]interface{}) map[string]interface{} {
+ for k, v := range src {
+ // If the key doesn't exist already, then just set the key to that value
+ if _, exists := dest[k]; !exists {
+ dest[k] = v
+ continue
+ }
+ nextMap, ok := v.(map[string]interface{})
+ // If it isn't another map, overwrite the value
+ if !ok {
+ dest[k] = v
+ continue
+ }
+ // Edge case: If the key exists in the destination, but isn't a map
+ destMap, isMap := dest[k].(map[string]interface{})
+ // If the source map has a map for this key, prefer it
+ if !isMap {
+ dest[k] = v
+ continue
+ }
+ // If we got to this point, it is a map in both, so merge them
+ dest[k] = h.mergeValues(destMap, nextMap)
+ }
+ return dest
+}
+
+// GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template
+func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string,
+ values []string) ([]types.KubernetesResourceTemplate, error) {
+
+ var outputDir, chartPath, namespace, releaseName string
+ var retData []types.KubernetesResourceTemplate
+
+ releaseName = h.releaseName
+ namespace = h.kubeNameSpace
+
+ // verify chart path exists
+ if _, err := os.Stat(inputPath); err == nil {
+ if chartPath, err = filepath.Abs(inputPath); err != nil {
+ return retData, err
+ }
+ } else {
+ return retData, err
+ }
+
+ //Create a temp directory in the system temp folder
+ outputDir, err := ioutil.TempDir("", "helm-tmpl-")
+ if err != nil {
+ return retData, pkgerrors.Wrap(err, "Got error creating temp dir")
+ }
+ log.Printf("The o/p dir:: %s ", outputDir)
+
+ if namespace == "" {
+ namespace = "default"
+ }
+
+ // get combined values and create config
+ rawVals, err := h.processValues(valueFiles, values)
+ if err != nil {
+ return retData, err
+ }
+ config := &chart.Config{Raw: string(rawVals), Values: map[string]*chart.Value{}}
+
+ if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 {
+ return retData, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";"))
+ }
+
+ // Check chart requirements to make sure all dependencies are present in /charts
+ c, err := chartutil.Load(chartPath)
+ if err != nil {
+ return retData, pkgerrors.Errorf("Got error: %s", err.Error())
+ }
+
+ renderOpts := renderutil.Options{
+ ReleaseOptions: chartutil.ReleaseOptions{
+ Name: releaseName,
+ IsInstall: true,
+ IsUpgrade: false,
+ Time: timeconv.Now(),
+ Namespace: namespace,
+ },
+ KubeVersion: h.kubeVersion,
+ }
+
+ renderedTemplates, err := renderutil.Render(c, config, renderOpts)
+ if err != nil {
+ return retData, err
+ }
+
+ newRenderedTemplates := make(map[string]string)
+
+ //Some manifests can contain multiple yaml documents
+ //This step is splitting them up into multiple files
+ //Each file contains only a single k8s kind
+ for k, v := range renderedTemplates {
+ //Splits into manifest-0, manifest-1 etc
+ if filepath.Base(k) == "NOTES.txt" {
+ continue
+ }
+ rmap := releaseutil.SplitManifests(v)
+ count := 0
+ for _, v1 := range rmap {
+ key := fmt.Sprintf("%s-%d", k, count)
+ newRenderedTemplates[key] = v1
+ count = count + 1
+ }
+ }
+
+ listManifests := manifest.SplitManifests(newRenderedTemplates)
+ var manifestsToRender []manifest.Manifest
+ //render all manifests in the chart
+ manifestsToRender = listManifests
+ for _, m := range tiller.SortByKind(manifestsToRender) {
+ data := m.Content
+ b := filepath.Base(m.Name)
+ if b == "NOTES.txt" {
+ continue
+ }
+ if strings.HasPrefix(b, "_") {
+ continue
+ }
+
+ // blank template after execution
+ if h.whitespaceRegex.MatchString(data) {
+ continue
+ }
+
+ mfilePath := filepath.Join(outputDir, m.Name)
+ utils.EnsureDirectory(mfilePath)
+ err = ioutil.WriteFile(mfilePath, []byte(data), 0666)
+ if err != nil {
+ return retData, err
+ }
+
+ gvk, err := getGroupVersionKind(data)
+ if err != nil {
+ return retData, err
+ }
+
+ kres := types.KubernetesResourceTemplate{
+ GVK: gvk,
+ FilePath: mfilePath,
+ }
+ retData = append(retData, kres)
+ }
+ return retData, nil
+}
+
+func getGroupVersionKind(data string) (schema.GroupVersionKind, error) {
+ out, err := k8syaml.ToJSON([]byte(data))
+ if err != nil {
+ return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Converting yaml to json")
+ }
+
+ simpleMeta := json.SimpleMetaFactory{}
+ gvk, err := simpleMeta.Interpret(out)
+ if err != nil {
+ return schema.GroupVersionKind{}, pkgerrors.Wrap(err, "Parsing apiversion and kind")
+ }
+
+ return *gvk, nil
+}
+
+// Resolver is an interface exposes the helm related functionalities
+type Resolver interface {
+ Resolve(appContent, appProfileContent []byte, overrideValuesOfAppStr []string, rName string) ([]types.KubernetesResourceTemplate, error)
+}
+
+// Resolve function
+func (h *TemplateClient) Resolve(appContent []byte, appProfileContent []byte, overrideValuesOfAppStr []string, rName, appName string) ([]types.KubernetesResourceTemplate, error) {
+
+ var sortedTemplates []types.KubernetesResourceTemplate
+
+ //prPath is the tmp path where the appProfileContent is extracted.
+ prPath, err := utils.ExtractTarBall(bytes.NewBuffer(appProfileContent))
+ if err != nil {
+ return sortedTemplates, pkgerrors.Wrap(err, "Extracting Profile Content")
+ }
+ log.Printf("The profile path:: %s", prPath)
+
+ prYamlClient, err := utils.ProcessProfileYaml(prPath, h.manifestName)
+ if err != nil {
+ return sortedTemplates, pkgerrors.Wrap(err, "Processing Profile Manifest")
+ }
+ log.Println("Got the profileYamlClient..")
+
+ //chartBasePath is the tmp path where the appContent(rawHelmCharts) is extracted.
+ chartBasePath, err := utils.ExtractTarBall(bytes.NewBuffer(appContent))
+ log.Printf("The chartBasePath :: %s", chartBasePath)
+
+ err = prYamlClient.CopyConfigurationOverrides(chartBasePath)
+ if err != nil {
+ return sortedTemplates, pkgerrors.Wrap(err, "Copying configresources to chart")
+ }
+
+ chartPath := filepath.Join(chartBasePath, appName)
+ sortedTemplates, err = h.GenerateKubernetesArtifacts(chartPath, []string{prYamlClient.GetValues()}, overrideValuesOfAppStr)
+ if err != nil {
+ return sortedTemplates, pkgerrors.Wrap(err, "Generate final k8s yaml")
+ }
+ return sortedTemplates, nil
+}
diff --git a/src/orchestrator/utils/profile_yaml.go b/src/orchestrator/utils/profile_yaml.go
new file mode 100644
index 00000000..91687c0a
--- /dev/null
+++ b/src/orchestrator/utils/profile_yaml.go
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+ "io/ioutil"
+ "log"
+ "path/filepath"
+
+ "github.com/ghodss/yaml"
+ pkgerrors "github.com/pkg/errors"
+)
+
+/*
+#Sample Yaml format for profile manifest.yaml
+---
+version: v1
+type:
+ values: "values_override.yaml"
+ configresource:
+ - filepath: config.yaml
+ chartpath: chart/config/resources/config.yaml
+ - filepath: config2.yaml
+ chartpath: chart/config/resources/config2.yaml
+*/
+
+type overrideFiles struct {
+ FilePath string `yaml:"filepath"`
+ ChartPath string `yaml:"chartpath"`
+}
+
+type supportedOverrides struct {
+ ConfigResource []overrideFiles `yaml:"configresource"`
+ Values string `yaml:"values"`
+}
+
+type profileOverride struct {
+ Version string `yaml:"version"`
+ Type supportedOverrides `yaml:"type"`
+}
+
+type ProfileYamlClient struct {
+ path string
+ override profileOverride
+}
+
+func (p ProfileYamlClient) Print() {
+ log.Println(p.override)
+}
+
+//GetValues returns a path to the override values.yam
+//that was part of the profile
+func (p ProfileYamlClient) GetValues() string {
+ return filepath.Join(p.path, p.override.Type.Values)
+}
+
+//CopyConfigurationOverrides copies the various files that are
+//provided as overrides to their corresponding locations within
+//the destination chart.
+func (p ProfileYamlClient) CopyConfigurationOverrides(chartPath string) error {
+
+ //Iterate over each configresource and copy that file into
+ //the respective path in the chart.
+ for _, v := range p.override.Type.ConfigResource {
+ data, err := ioutil.ReadFile(filepath.Join(p.path, v.FilePath))
+ if err != nil {
+ return pkgerrors.Wrap(err, "Reading configuration file")
+ }
+ err = ioutil.WriteFile(filepath.Join(chartPath, v.ChartPath), data, 0644)
+ if err != nil {
+ return pkgerrors.Wrap(err, "Writing configuration file into chartpath")
+ }
+ }
+
+ return nil
+}
+
+//ProcessProfileYaml parses the manifest.yaml file that is part of the profile
+//package and creates the appropriate structures out of it.
+func ProcessProfileYaml(fpath string, manifestFileName string) (ProfileYamlClient, error) {
+
+ p := filepath.Join(fpath, manifestFileName)
+ data, err := ioutil.ReadFile(p)
+ if err != nil {
+ return ProfileYamlClient{}, pkgerrors.Wrap(err, "Reading manifest file")
+ }
+
+ out := profileOverride{}
+ err = yaml.Unmarshal(data, &out)
+ if err != nil {
+ return ProfileYamlClient{}, pkgerrors.Wrap(err, "Marshaling manifest yaml file")
+ }
+
+ return ProfileYamlClient{path: fpath, override: out}, nil
+}
diff --git a/src/orchestrator/utils/types/types.go b/src/orchestrator/utils/types/types.go
new file mode 100644
index 00000000..4ffb4180
--- /dev/null
+++ b/src/orchestrator/utils/types/types.go
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package types
+
+import (
+ "k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// KubernetesResourceTemplate - Represents the template that is used to create a particular
+// resource in Kubernetes
+type KubernetesResourceTemplate struct {
+ // Tracks the apiVersion and Kind of the resource
+ GVK schema.GroupVersionKind
+ // Path to the file that contains the resource info
+ FilePath string
+}
+
+// KubernetesResource is the resource that is created in Kubernetes
+// It is the type that will be used for tracking a resource.
+// Any future information such as status, time can be added here
+// for tracking.
+type KubernetesResource struct {
+ // Tracks the apiVersion and Kind of the resource
+ GVK schema.GroupVersionKind
+ // Name of resource in Kubernetes
+ Name string
+}
diff --git a/src/orchestrator/utils/util-functions.go b/src/orchestrator/utils/util-functions.go
new file mode 100644
index 00000000..13c78ba4
--- /dev/null
+++ b/src/orchestrator/utils/util-functions.go
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+ "archive/tar"
+ "compress/gzip"
+ "io"
+ "io/ioutil"
+ "os"
+ "path"
+ "path/filepath"
+
+ pkgerrors "github.com/pkg/errors"
+)
+
+//ExtractTarBall provides functionality to extract a tar.gz file
+//into a temporary location for later use.
+//It returns the path to the new location
+func ExtractTarBall(r io.Reader) (string, error) {
+ //Check if it is a valid gz
+ gzf, err := gzip.NewReader(r)
+ if err != nil {
+ return "", pkgerrors.Wrap(err, "Invalid gzip format")
+ }
+
+ //Check if it is a valid tar file
+ //Unfortunately this can only be done by inspecting all the tar contents
+ tarR := tar.NewReader(gzf)
+ first := true
+
+ outDir, _ := ioutil.TempDir("", "k8s-ext-")
+
+ for true {
+ header, err := tarR.Next()
+
+ if err == io.EOF {
+ //Check if we have just a gzip file without a tar archive inside
+ if first {
+ return "", pkgerrors.New("Empty or non-existant Tar file found")
+ }
+ //End of archive
+ break
+ }
+
+ if err != nil {
+ return "", pkgerrors.Wrap(err, "Error reading tar file")
+ }
+
+ target := filepath.Join(outDir, header.Name)
+
+ switch header.Typeflag {
+ case tar.TypeDir:
+ if _, err := os.Stat(target); err != nil {
+ // Using 755 read, write, execute for owner
+ // groups and others get read and execute permissions
+ // on the folder.
+ if err := os.MkdirAll(target, 0755); err != nil {
+ return "", pkgerrors.Wrap(err, "Creating directory")
+ }
+ }
+ case tar.TypeReg:
+ if target == outDir { // Handle '.' substituted to '' entry
+ continue
+ }
+
+ err = EnsureDirectory(target)
+ if err != nil {
+ return "", pkgerrors.Wrap(err, "Creating Directory")
+ }
+
+ f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+ if err != nil {
+ return "", pkgerrors.Wrap(err, "Creating file")
+ }
+
+ // copy over contents
+ if _, err := io.Copy(f, tarR); err != nil {
+ return "", pkgerrors.Wrap(err, "Copying file content")
+ }
+
+ // close for each file instead of a defer for all
+ // at the end of the function
+ f.Close()
+ }
+
+ first = false
+ }
+
+ return outDir, nil
+}
+
+//EnsureDirectory makes sure that the directories specified in the path exist
+//If not, it will create them, if possible.
+func EnsureDirectory(f string) error {
+ base := path.Dir(f)
+ _, err := os.Stat(base)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ return os.MkdirAll(base, 0755)
+}