From 3766e380c40dc1e4c839372dcdc0c71a972ffa70 Mon Sep 17 00:00:00 2001 From: Lukasz Rajewski Date: Mon, 4 Oct 2021 21:56:02 +0200 Subject: Fixed installation of CRD resources Issue-ID: MULTICLOUD-1397 Signed-off-by: Lukasz Rajewski Change-Id: Id8e653f1b5c61278ee2d64da409ac5b0685b36b8 --- src/k8splugin/internal/app/client_test.go | 3 +- src/k8splugin/internal/app/config_backend.go | 8 +++- src/k8splugin/internal/app/instance.go | 22 +++++++--- src/k8splugin/internal/helm/helm.go | 63 ++++++++++++++++++++-------- src/k8splugin/internal/helm/helm_test.go | 5 ++- src/k8splugin/internal/rb/profile.go | 25 +++++------ src/k8splugin/internal/rb/profile_test.go | 2 +- src/k8splugin/plugins/generic/plugin.go | 31 +++++++++++++- 8 files changed, 117 insertions(+), 42 deletions(-) (limited to 'src') diff --git a/src/k8splugin/internal/app/client_test.go b/src/k8splugin/internal/app/client_test.go index 0ba244d2..f51c15fc 100644 --- a/src/k8splugin/internal/app/client_test.go +++ b/src/k8splugin/internal/app/client_test.go @@ -15,13 +15,14 @@ package app import ( "encoding/base64" - "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils" "io/ioutil" "os" "plugin" "reflect" "testing" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/connection" "github.com/onap/multicloud-k8s/src/k8splugin/internal/db" "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" diff --git a/src/k8splugin/internal/app/config_backend.go b/src/k8splugin/internal/app/config_backend.go index 4fedb386..be11e8ce 100644 --- a/src/k8splugin/internal/app/config_backend.go +++ b/src/k8splugin/internal/app/config_backend.go @@ -631,14 +631,18 @@ var resolve = func(rbName, rbVersion, profileName string, p Config, releaseName finalReleaseName) chartPath := filepath.Join(chartBasePath, t.ChartName) - resTemplates, _, err = helmClient.GenerateKubernetesArtifacts(chartPath, + resTemplates, crdList, _, err := helmClient.GenerateKubernetesArtifacts(chartPath, []string{outputfile.Name()}, nil) if err != nil { return configResourceList{}, pkgerrors.Wrap(err, "Generate final k8s yaml") } + for _, tmp := range resTemplates { + crdList = append(crdList, tmp) + } + crl := configResourceList{ - resourceTemplates: resTemplates, + resourceTemplates: crdList, profile: profile, } diff --git a/src/k8splugin/internal/app/instance.go b/src/k8splugin/internal/app/instance.go index b7f382ad..63fe042b 100644 --- a/src/k8splugin/internal/app/instance.go +++ b/src/k8splugin/internal/app/instance.go @@ -30,7 +30,6 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/resource" @@ -225,7 +224,7 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { } //Execute the kubernetes create command - sortedTemplates, hookList, releaseName, err := rb.NewProfileClient().Resolve(i.RBName, i.RBVersion, i.ProfileName, overrideValues, i.ReleaseName) + sortedTemplates, crdList, hookList, releaseName, err := rb.NewProfileClient().Resolve(i.RBName, i.RBVersion, i.ProfileName, overrideValues, i.ReleaseName) if err != nil { return InstanceResponse{}, pkgerrors.Wrap(err, "Error resolving helm charts") } @@ -245,6 +244,12 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { log.Printf(" Kind: %s", t.GVK.Kind) } + log.Printf("Crd rss info") + for _, t := range crdList { + log.Printf(" Path: %s", t.FilePath) + log.Printf(" Kind: %s", t.GVK.Kind) + } + log.Printf("Hook info") for _, h := range hookList { log.Printf(" Name: %s", h.Hook.Name) @@ -280,6 +285,15 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Namespace") } + if len(crdList) > 0 { + log.Printf("Pre-Installing CRDs") + _, err = k8sClient.createResources(crdList, profile.Namespace) + + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Pre-Installing CRDs") + } + } + hookClient := NewHookClient(profile.Namespace, id, v.storeName, v.tagInst) if len(hookClient.getHookByEvent(hookList, release.HookPreInstall)) != 0 { err = hookClient.ExecHook(k8sClient, hookList, release.HookPreInstall, preInstallTimeOut, 0, &dbData) @@ -561,8 +575,6 @@ func (v *InstanceClient) checkRssStatus(rss helm.KubernetesResource, k8sClient K parsedRes = new(corev1.Service) case "DaemonSet": parsedRes = new(appsv1.DaemonSet) - case "CustomResourceDefinition": - parsedRes = new(apiextv1.CustomResourceDefinition) case "StatefulSet": parsedRes = new(appsv1.StatefulSet) case "ReplicationController": @@ -791,7 +803,7 @@ func (v *InstanceClient) RecoverCreateOrDelete(id string) error { ID: id, } log.Printf(" Resolving template for release %s", instance.Request.ReleaseName) - _, hookList, _, err := rb.NewProfileClient().Resolve(instance.Request.RBName, instance.Request.RBVersion, instance.Request.ProfileName, overrideValues, instance.Request.ReleaseName) + _, _, hookList, _, err := rb.NewProfileClient().Resolve(instance.Request.RBName, instance.Request.RBVersion, instance.Request.ProfileName, overrideValues, instance.Request.ReleaseName) instance.Hooks = hookList err = db.DBconn.Update(v.storeName, key, v.tagInst, instance) if err != nil { diff --git a/src/k8splugin/internal/helm/helm.go b/src/k8splugin/internal/helm/helm.go index 849674a9..6064b2c8 100644 --- a/src/k8splugin/internal/helm/helm.go +++ b/src/k8splugin/internal/helm/helm.go @@ -19,13 +19,14 @@ package helm import ( "fmt" - "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils" "io/ioutil" "os" "path/filepath" "regexp" "strings" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils" + pkgerrors "github.com/pkg/errors" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" @@ -47,7 +48,7 @@ type Template interface { GenerateKubernetesArtifacts( chartPath string, valueFiles []string, - values []string) ([]KubernetesResourceTemplate, []*Hook, error) + values []string) ([]KubernetesResourceTemplate, []KubernetesResourceTemplate, []*Hook, error) } // TemplateClient implements the Template interface @@ -90,10 +91,11 @@ func (h *TemplateClient) processValues(valueFiles []string, values []string) (ma // GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string, - values []string) ([]KubernetesResourceTemplate, []*Hook, error) { + values []string) ([]KubernetesResourceTemplate, []KubernetesResourceTemplate, []*Hook, error) { var outputDir, chartPath, namespace, releaseName string var retData []KubernetesResourceTemplate + var crdData []KubernetesResourceTemplate var hookList []*Hook releaseName = h.releaseName @@ -102,16 +104,16 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile // verify chart path exists if _, err := os.Stat(inputPath); err == nil { if chartPath, err = filepath.Abs(inputPath); err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } } else { - return retData, hookList, err + return retData, crdData, hookList, err } //Create a temp directory in the system temp folder outputDir, err := ioutil.TempDir("", "helm-tmpl-") if err != nil { - return retData, hookList, pkgerrors.Wrap(err, "Got error creating temp dir") + return retData, crdData, hookList, pkgerrors.Wrap(err, "Got error creating temp dir") } if namespace == "" { @@ -121,11 +123,11 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile // get combined values and create config rawVals, err := h.processValues(valueFiles, values) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 { - return retData, hookList, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";")) + return retData, crdData, hookList, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";")) } // Initialize the install client @@ -133,27 +135,52 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile client.DryRun = true client.ClientOnly = true client.ReleaseName = releaseName - client.IncludeCRDs = true + client.IncludeCRDs = false client.DisableHooks = true //to ensure no duplicates in case of defined pre/post install hooks // Check chart dependencies to make sure all are present in /charts chartRequested, err := loader.Load(chartPath) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } if chartRequested.Metadata.Type != "" && chartRequested.Metadata.Type != "application" { - return retData, hookList, fmt.Errorf( + return retData, crdData, hookList, fmt.Errorf( "chart %q has an unsupported type and is not installable: %q", chartRequested.Metadata.Name, chartRequested.Metadata.Type, ) } + for _, crd := range chartRequested.CRDObjects() { + if strings.HasPrefix(crd.Name, "_") { + continue + } + filePath := filepath.Join(outputDir, crd.Name) + data := string(crd.File.Data) + // blank template after execution + if h.emptyRegex.MatchString(data) { + continue + } + utils.EnsureDirectory(filePath) + err = ioutil.WriteFile(filePath, []byte(crd.File.Data), 0600) + if err != nil { + return retData, crdData, hookList, err + } + gvk, err := getGroupVersionKind(data) + if err != nil { + return retData, crdData, hookList, err + } + kres := KubernetesResourceTemplate{ + GVK: gvk, + FilePath: filePath, + } + crdData = append(crdData, kres) + } client.Namespace = namespace release, err := client.Run(chartRequested, rawVals) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } // SplitManifests returns integer-sortable so that manifests get output // in the same order as the input by `BySplitManifestsOrder`. @@ -161,7 +188,7 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile // We won't get any meaningful hooks from here _, m, err := releaseutil.SortManifests(rmap, nil, releaseutil.InstallOrder) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } for _, k := range m { data := k.Content @@ -180,11 +207,11 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile utils.EnsureDirectory(mfilePath) err = ioutil.WriteFile(mfilePath, []byte(k.Content), 0600) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } gvk, err := getGroupVersionKind(data) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } kres := KubernetesResourceTemplate{ GVK: gvk, @@ -197,15 +224,15 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile utils.EnsureDirectory(hFilePath) err = ioutil.WriteFile(hFilePath, []byte(h.Manifest), 0600) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } gvk, err := getGroupVersionKind(h.Manifest) if err != nil { - return retData, hookList, err + return retData, crdData, hookList, err } hookList = append(hookList, &Hook{*h, KubernetesResourceTemplate{gvk, hFilePath}}) } - return retData, hookList, nil + return retData, crdData, hookList, nil } func getGroupVersionKind(data string) (schema.GroupVersionKind, error) { diff --git a/src/k8splugin/internal/helm/helm_test.go b/src/k8splugin/internal/helm/helm_test.go index 29d446fa..b805b59b 100644 --- a/src/k8splugin/internal/helm/helm_test.go +++ b/src/k8splugin/internal/helm/helm_test.go @@ -20,11 +20,12 @@ package helm import ( "crypto/sha256" "fmt" - "gopkg.in/yaml.v2" "io/ioutil" "path/filepath" "strings" "testing" + + "gopkg.in/yaml.v2" ) func TestProcessValues(t *testing.T) { @@ -202,7 +203,7 @@ func TestGenerateKubernetesArtifacts(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename") - out, hooks, err := tc.GenerateKubernetesArtifacts(testCase.chartPath, testCase.valueFiles, + out, _, hooks, err := tc.GenerateKubernetesArtifacts(testCase.chartPath, testCase.valueFiles, testCase.values) if err != nil { if testCase.expectedError == "" { diff --git a/src/k8splugin/internal/rb/profile.go b/src/k8splugin/internal/rb/profile.go index 3db6c40f..f9ac56bd 100644 --- a/src/k8splugin/internal/rb/profile.go +++ b/src/k8splugin/internal/rb/profile.go @@ -271,9 +271,10 @@ func (v *ProfileClient) Download(rbName, rbVersion, prName string) ([]byte, erro //Resolve returns the path where the helm chart merged with //configuration overrides resides and final ReleaseName picked for instantiation func (v *ProfileClient) Resolve(rbName string, rbVersion string, - profileName string, values []string, overrideReleaseName string) ([]helm.KubernetesResourceTemplate, []*helm.Hook, string, error) { + profileName string, values []string, overrideReleaseName string) ([]helm.KubernetesResourceTemplate, []helm.KubernetesResourceTemplate, []*helm.Hook, string, error) { var sortedTemplates []helm.KubernetesResourceTemplate + var crdList []helm.KubernetesResourceTemplate var hookList []*helm.Hook var finalReleaseName string @@ -281,40 +282,40 @@ func (v *ProfileClient) Resolve(rbName string, rbVersion string, //If everything seems okay, then download the definition prData, err := v.Download(rbName, rbVersion, profileName) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Profile") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Profile") } prPath, err := ExtractTarBall(bytes.NewBuffer(prData)) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Profile Content") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Profile Content") } prYamlClient, err := ProcessProfileYaml(prPath, v.manifestName) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Processing Profile Manifest") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Processing Profile Manifest") } definitionClient := NewDefinitionClient() definition, err := definitionClient.Get(rbName, rbVersion) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Definition Metadata") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Definition Metadata") } defData, err := definitionClient.Download(rbName, rbVersion) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Definition") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Definition") } chartBasePath, err := ExtractTarBall(bytes.NewBuffer(defData)) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Definition Charts") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Definition Charts") } //Get the definition ID and download its contents profile, err := v.Get(rbName, rbVersion, profileName) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Profile") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Profile") } //Copy the profile configresources to the chart locations @@ -324,7 +325,7 @@ func (v *ProfileClient) Resolve(rbName string, rbVersion string, // chartpath: chart/config/resources/config.yaml err = prYamlClient.CopyConfigurationOverrides(chartBasePath) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Copying configresources to chart") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Copying configresources to chart") } if overrideReleaseName == "" { @@ -338,14 +339,14 @@ func (v *ProfileClient) Resolve(rbName string, rbVersion string, finalReleaseName) chartPath := filepath.Join(chartBasePath, definition.ChartName) - sortedTemplates, hookList, err = helmClient.GenerateKubernetesArtifacts(chartPath, + sortedTemplates, crdList, hookList, err = helmClient.GenerateKubernetesArtifacts(chartPath, []string{prYamlClient.GetValues()}, values) if err != nil { - return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Generate final k8s yaml") + return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Generate final k8s yaml") } - return sortedTemplates, hookList, finalReleaseName, nil + return sortedTemplates, crdList, hookList, finalReleaseName, nil } // Returns an empty profile with the following contents diff --git a/src/k8splugin/internal/rb/profile_test.go b/src/k8splugin/internal/rb/profile_test.go index 3c40c2c9..2a9dc4fd 100644 --- a/src/k8splugin/internal/rb/profile_test.go +++ b/src/k8splugin/internal/rb/profile_test.go @@ -773,7 +773,7 @@ func TestResolveProfile(t *testing.T) { t.Run(testCase.label, func(t *testing.T) { db.DBconn = testCase.mockdb impl := NewProfileClient() - data, _, releaseName, err := impl.Resolve(testCase.rbname, + data, _, _, releaseName, err := impl.Resolve(testCase.rbname, testCase.rbversion, testCase.prname, []string{}, testCase.releaseName) defer cleanup(data) if err != nil { diff --git a/src/k8splugin/plugins/generic/plugin.go b/src/k8splugin/plugins/generic/plugin.go index 5815b74f..a210f6d6 100644 --- a/src/k8splugin/plugins/generic/plugin.go +++ b/src/k8splugin/plugins/generic/plugin.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" pkgerrors "github.com/pkg/errors" + "github.com/prometheus/common/log" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -305,7 +306,18 @@ func (g genericPlugin) Create(yamlFilePath string, namespace string, client plug if err != nil { return "", pkgerrors.Wrap(err, "Mapping kind to resource error") } - + if gvk.Kind == "CustomResourceDefinition" { + //according the helm spec, CRD is created only once, and we raise only warn if we try to do it once more + resource := helm.KubernetesResource{} + resource.GVK = gvk + resource.Name = unstruct.GetName() + name, err := g.Get(resource, namespace, client) + if err == nil && name == resource.Name { + //CRD update is not supported according to Helm spec + log.Warn(fmt.Sprintf("CRD %s create will be skipped. It already exists", name)) + return name, nil + } + } //Add the tracking label to all resources created here labels := unstruct.GetLabels() //Check if labels exist for this object @@ -362,6 +374,18 @@ func (g genericPlugin) Update(yamlFilePath string, namespace string, client plug return "", pkgerrors.Wrap(err, "Mapping kind to resource error") } + if gvk.Kind == "CustomResourceDefinition" { + resource := helm.KubernetesResource{} + resource.GVK = gvk + resource.Name = unstruct.GetName() + name, err := g.Get(resource, namespace, client) + if err == nil && name == resource.Name { + //CRD update is not supported according to Helm spec + log.Warn(fmt.Sprintf("CRD %s update will be skipped", name)) + return name, nil + } + } + //Add the tracking label to all resources created here labels := unstruct.GetLabels() //Check if labels exist for this object @@ -463,6 +487,11 @@ func (g genericPlugin) Delete(resource helm.KubernetesResource, namespace string opts := metav1.DeleteOptions{ PropagationPolicy: &deletePolicy, } + if resource.GVK.Kind == "CustomResourceDefinition" { + //CRD deletion is not supported according to Helm spec + log.Warn(fmt.Sprintf("CRD %s deletion will be skipped", resource.Name)) + return nil + } switch mapping.Scope.Name() { case meta.RESTScopeNameNamespace: -- cgit 1.2.3-korg