Fixed installation of CRD resources

Issue-ID: MULTICLOUD-1397
Signed-off-by: Lukasz Rajewski <lukasz.rajewski@orange.com>
Change-Id: Id8e653f1b5c61278ee2d64da409ac5b0685b36b8

8 files changed, 117 insertions, 42 deletions

diff --git a/src/k8splugin/internal/app/client_test.go b/src/k8splugin/internal/app/client_test.go
index 0ba244d2..f51c15fc 100644
--- a/src/k8splugin/internal/app/client_test.go
+++ b/src/k8splugin/internal/app/client_test.go
@@ -15,13 +15,14 @@ package app
 
 import (
 	"encoding/base64"
-	"github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
 	"io/ioutil"
 	"os"
 	"plugin"
 	"reflect"
 	"testing"
 
+	"github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
+
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/connection"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
diff --git a/src/k8splugin/internal/app/config_backend.go b/src/k8splugin/internal/app/config_backend.go
index 4fedb386..be11e8ce 100644
--- a/src/k8splugin/internal/app/config_backend.go
+++ b/src/k8splugin/internal/app/config_backend.go
@@ -631,14 +631,18 @@ var resolve = func(rbName, rbVersion, profileName string, p Config, releaseName
 		finalReleaseName)
 
 	chartPath := filepath.Join(chartBasePath, t.ChartName)
-	resTemplates, _, err = helmClient.GenerateKubernetesArtifacts(chartPath,
+	resTemplates, crdList, _, err := helmClient.GenerateKubernetesArtifacts(chartPath,
 		[]string{outputfile.Name()},
 		nil)
 	if err != nil {
 		return configResourceList{}, pkgerrors.Wrap(err, "Generate final k8s yaml")
 	}
+	for _, tmp := range resTemplates {
+		crdList = append(crdList, tmp)
+	}
+
 	crl := configResourceList{
-		resourceTemplates: resTemplates,
+		resourceTemplates: crdList,
 		profile:           profile,
 	}
 
diff --git a/src/k8splugin/internal/app/instance.go b/src/k8splugin/internal/app/instance.go
index b7f382ad..63fe042b 100644
--- a/src/k8splugin/internal/app/instance.go
+++ b/src/k8splugin/internal/app/instance.go
@@ -30,7 +30,6 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
-	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/cli-runtime/pkg/resource"
@@ -225,7 +224,7 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) {
 	}
 
 	//Execute the kubernetes create command
-	sortedTemplates, hookList, releaseName, err := rb.NewProfileClient().Resolve(i.RBName, i.RBVersion, i.ProfileName, overrideValues, i.ReleaseName)
+	sortedTemplates, crdList, hookList, releaseName, err := rb.NewProfileClient().Resolve(i.RBName, i.RBVersion, i.ProfileName, overrideValues, i.ReleaseName)
 	if err != nil {
 		return InstanceResponse{}, pkgerrors.Wrap(err, "Error resolving helm charts")
 	}
@@ -245,6 +244,12 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) {
 		log.Printf("    Kind: %s", t.GVK.Kind)
 	}
 
+	log.Printf("Crd rss info")
+	for _, t := range crdList {
+		log.Printf("  Path: %s", t.FilePath)
+		log.Printf("    Kind: %s", t.GVK.Kind)
+	}
+
 	log.Printf("Hook info")
 	for _, h := range hookList {
 		log.Printf("  Name: %s", h.Hook.Name)
@@ -280,6 +285,15 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) {
 		return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Namespace")
 	}
 
+	if len(crdList) > 0 {
+		log.Printf("Pre-Installing CRDs")
+		_, err = k8sClient.createResources(crdList, profile.Namespace)
+
+		if err != nil {
+			return InstanceResponse{}, pkgerrors.Wrap(err, "Pre-Installing CRDs")
+		}
+	}
+
 	hookClient := NewHookClient(profile.Namespace, id, v.storeName, v.tagInst)
 	if len(hookClient.getHookByEvent(hookList, release.HookPreInstall)) != 0 {
 		err = hookClient.ExecHook(k8sClient, hookList, release.HookPreInstall, preInstallTimeOut, 0, &dbData)
@@ -561,8 +575,6 @@ func (v *InstanceClient) checkRssStatus(rss helm.KubernetesResource, k8sClient K
 		parsedRes = new(corev1.Service)
 	case "DaemonSet":
 		parsedRes = new(appsv1.DaemonSet)
-	case "CustomResourceDefinition":
-		parsedRes = new(apiextv1.CustomResourceDefinition)
 	case "StatefulSet":
 		parsedRes = new(appsv1.StatefulSet)
 	case "ReplicationController":
@@ -791,7 +803,7 @@ func (v *InstanceClient) RecoverCreateOrDelete(id string) error {
 		ID: id,
 	}
 	log.Printf("  Resolving template for release %s", instance.Request.ReleaseName)
-	_, hookList, _, err := rb.NewProfileClient().Resolve(instance.Request.RBName, instance.Request.RBVersion, instance.Request.ProfileName, overrideValues, instance.Request.ReleaseName)
+	_, _, hookList, _, err := rb.NewProfileClient().Resolve(instance.Request.RBName, instance.Request.RBVersion, instance.Request.ProfileName, overrideValues, instance.Request.ReleaseName)
 	instance.Hooks = hookList
 	err = db.DBconn.Update(v.storeName, key, v.tagInst, instance)
 	if err != nil {
diff --git a/src/k8splugin/internal/helm/helm.go b/src/k8splugin/internal/helm/helm.go
index 849674a9..6064b2c8 100644
--- a/src/k8splugin/internal/helm/helm.go
+++ b/src/k8splugin/internal/helm/helm.go
@@ -19,13 +19,14 @@ package helm
 
 import (
 	"fmt"
-	"github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 
+	"github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
+
 	pkgerrors "github.com/pkg/errors"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -47,7 +48,7 @@ type Template interface {
 	GenerateKubernetesArtifacts(
 		chartPath string,
 		valueFiles []string,
-		values []string) ([]KubernetesResourceTemplate, []*Hook, error)
+		values []string) ([]KubernetesResourceTemplate, []KubernetesResourceTemplate, []*Hook, error)
 }
 
 // TemplateClient implements the Template interface
@@ -90,10 +91,11 @@ func (h *TemplateClient) processValues(valueFiles []string, values []string) (ma
 
 // GenerateKubernetesArtifacts a mapping of type to fully evaluated helm template
 func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFiles []string,
-	values []string) ([]KubernetesResourceTemplate, []*Hook, error) {
+	values []string) ([]KubernetesResourceTemplate, []KubernetesResourceTemplate, []*Hook, error) {
 
 	var outputDir, chartPath, namespace, releaseName string
 	var retData []KubernetesResourceTemplate
+	var crdData []KubernetesResourceTemplate
 	var hookList []*Hook
 
 	releaseName = h.releaseName
@@ -102,16 +104,16 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile
 	// verify chart path exists
 	if _, err := os.Stat(inputPath); err == nil {
 		if chartPath, err = filepath.Abs(inputPath); err != nil {
-			return retData, hookList, err
+			return retData, crdData, hookList, err
 		}
 	} else {
-		return retData, hookList, err
+		return retData, crdData, hookList, err
 	}
 
 	//Create a temp directory in the system temp folder
 	outputDir, err := ioutil.TempDir("", "helm-tmpl-")
 	if err != nil {
-		return retData, hookList, pkgerrors.Wrap(err, "Got error creating temp dir")
+		return retData, crdData, hookList, pkgerrors.Wrap(err, "Got error creating temp dir")
 	}
 
 	if namespace == "" {
@@ -121,11 +123,11 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile
 	// get combined values and create config
 	rawVals, err := h.processValues(valueFiles, values)
 	if err != nil {
-		return retData, hookList, err
+		return retData, crdData, hookList, err
 	}
 
 	if msgs := validation.IsDNS1123Label(releaseName); releaseName != "" && len(msgs) > 0 {
-		return retData, hookList, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";"))
+		return retData, crdData, hookList, fmt.Errorf("release name %s is not a valid DNS label: %s", releaseName, strings.Join(msgs, ";"))
 	}
 
 	// Initialize the install client
@@ -133,27 +135,52 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile
 	client.DryRun = true
 	client.ClientOnly = true
 	client.ReleaseName = releaseName
-	client.IncludeCRDs = true
+	client.IncludeCRDs = false
 	client.DisableHooks = true //to ensure no duplicates in case of defined pre/post install hooks
 
 	// Check chart dependencies to make sure all are present in /charts
 	chartRequested, err := loader.Load(chartPath)
 	if err != nil {
-		return retData, hookList, err
+		return retData, crdData, hookList, err
 	}
 
 	if chartRequested.Metadata.Type != "" && chartRequested.Metadata.Type != "application" {
-		return retData, hookList, fmt.Errorf(
+		return retData, crdData, hookList, fmt.Errorf(
 			"chart %q has an unsupported type and is not installable: %q",
 			chartRequested.Metadata.Name,
 			chartRequested.Metadata.Type,
 		)
 	}
 
+	for _, crd := range chartRequested.CRDObjects() {
+		if strings.HasPrefix(crd.Name, "_") {
+			continue
+		}
+		filePath := filepath.Join(outputDir, crd.Name)
+		data := string(crd.File.Data)
+		// blank template after execution
+		if h.emptyRegex.MatchString(data) {
+			continue
+		}
+		utils.EnsureDirectory(filePath)
+		err = ioutil.WriteFile(filePath, []byte(crd.File.Data), 0600)
+		if err != nil {
+			return retData, crdData, hookList, err
+		}
+		gvk, err := getGroupVersionKind(data)
+		if err != nil {
+			return retData, crdData, hookList, err
+		}
+		kres := KubernetesResourceTemplate{
+			GVK:      gvk,
+			FilePath: filePath,
+		}
+		crdData = append(crdData, kres)
+	}
 	client.Namespace = namespace
 	release, err := client.Run(chartRequested, rawVals)
 	if err != nil {
-		return retData, hookList, err
+		return retData, crdData, hookList, err
 	}
 	// SplitManifests returns integer-sortable so that manifests get output
 	// in the same order as the input by `BySplitManifestsOrder`.
@@ -161,7 +188,7 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile
 	// We won't get any meaningful hooks from here
 	_, m, err := releaseutil.SortManifests(rmap, nil, releaseutil.InstallOrder)
 	if err != nil {
-		return retData, hookList, err
+		return retData, crdData, hookList, err
 	}
 	for _, k := range m {
 		data := k.Content
@@ -180,11 +207,11 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile
 		utils.EnsureDirectory(mfilePath)
 		err = ioutil.WriteFile(mfilePath, []byte(k.Content), 0600)
 		if err != nil {
-			return retData, hookList, err
+			return retData, crdData, hookList, err
 		}
 		gvk, err := getGroupVersionKind(data)
 		if err != nil {
-			return retData, hookList, err
+			return retData, crdData, hookList, err
 		}
 		kres := KubernetesResourceTemplate{
 			GVK:      gvk,
@@ -197,15 +224,15 @@ func (h *TemplateClient) GenerateKubernetesArtifacts(inputPath string, valueFile
 		utils.EnsureDirectory(hFilePath)
 		err = ioutil.WriteFile(hFilePath, []byte(h.Manifest), 0600)
 		if err != nil {
-			return retData, hookList, err
+			return retData, crdData, hookList, err
 		}
 		gvk, err := getGroupVersionKind(h.Manifest)
 		if err != nil {
-			return retData, hookList, err
+			return retData, crdData, hookList, err
 		}
 		hookList = append(hookList, &Hook{*h, KubernetesResourceTemplate{gvk, hFilePath}})
 	}
-	return retData, hookList, nil
+	return retData, crdData, hookList, nil
 }
 
 func getGroupVersionKind(data string) (schema.GroupVersionKind, error) {
diff --git a/src/k8splugin/internal/helm/helm_test.go b/src/k8splugin/internal/helm/helm_test.go
index 29d446fa..b805b59b 100644
--- a/src/k8splugin/internal/helm/helm_test.go
+++ b/src/k8splugin/internal/helm/helm_test.go
@@ -20,11 +20,12 @@ package helm
 import (
 	"crypto/sha256"
 	"fmt"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"gopkg.in/yaml.v2"
 )
 
 func TestProcessValues(t *testing.T) {
@@ -202,7 +203,7 @@ func TestGenerateKubernetesArtifacts(t *testing.T) {
 	for _, testCase := range testCases {
 		t.Run(testCase.label, func(t *testing.T) {
 			tc := NewTemplateClient("1.12.3", "testnamespace", "testreleasename")
-			out, hooks, err := tc.GenerateKubernetesArtifacts(testCase.chartPath, testCase.valueFiles,
+			out, _, hooks, err := tc.GenerateKubernetesArtifacts(testCase.chartPath, testCase.valueFiles,
 				testCase.values)
 			if err != nil {
 				if testCase.expectedError == "" {
diff --git a/src/k8splugin/internal/rb/profile.go b/src/k8splugin/internal/rb/profile.go
index 3db6c40f..f9ac56bd 100644
--- a/src/k8splugin/internal/rb/profile.go
+++ b/src/k8splugin/internal/rb/profile.go
@@ -271,9 +271,10 @@ func (v *ProfileClient) Download(rbName, rbVersion, prName string) ([]byte, erro
 //Resolve returns the path where the helm chart merged with
 //configuration overrides resides and final ReleaseName picked for instantiation
 func (v *ProfileClient) Resolve(rbName string, rbVersion string,
-	profileName string, values []string, overrideReleaseName string) ([]helm.KubernetesResourceTemplate, []*helm.Hook, string, error) {
+	profileName string, values []string, overrideReleaseName string) ([]helm.KubernetesResourceTemplate, []helm.KubernetesResourceTemplate, []*helm.Hook, string, error) {
 
 	var sortedTemplates []helm.KubernetesResourceTemplate
+	var crdList []helm.KubernetesResourceTemplate
 	var hookList []*helm.Hook
 	var finalReleaseName string
 
@@ -281,40 +282,40 @@ func (v *ProfileClient) Resolve(rbName string, rbVersion string,
 	//If everything seems okay, then download the definition
 	prData, err := v.Download(rbName, rbVersion, profileName)
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Profile")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Profile")
 	}
 
 	prPath, err := ExtractTarBall(bytes.NewBuffer(prData))
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Profile Content")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Profile Content")
 	}
 
 	prYamlClient, err := ProcessProfileYaml(prPath, v.manifestName)
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Processing Profile Manifest")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Processing Profile Manifest")
 	}
 
 	definitionClient := NewDefinitionClient()
 
 	definition, err := definitionClient.Get(rbName, rbVersion)
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Definition Metadata")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Definition Metadata")
 	}
 
 	defData, err := definitionClient.Download(rbName, rbVersion)
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Definition")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Downloading Definition")
 	}
 
 	chartBasePath, err := ExtractTarBall(bytes.NewBuffer(defData))
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Definition Charts")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Extracting Definition Charts")
 	}
 
 	//Get the definition ID and download its contents
 	profile, err := v.Get(rbName, rbVersion, profileName)
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Profile")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Getting Profile")
 	}
 
 	//Copy the profile configresources to the chart locations
@@ -324,7 +325,7 @@ func (v *ProfileClient) Resolve(rbName string, rbVersion string,
 	//   chartpath: chart/config/resources/config.yaml
 	err = prYamlClient.CopyConfigurationOverrides(chartBasePath)
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Copying configresources to chart")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Copying configresources to chart")
 	}
 
 	if overrideReleaseName == "" {
@@ -338,14 +339,14 @@ func (v *ProfileClient) Resolve(rbName string, rbVersion string,
 		finalReleaseName)
 
 	chartPath := filepath.Join(chartBasePath, definition.ChartName)
-	sortedTemplates, hookList, err = helmClient.GenerateKubernetesArtifacts(chartPath,
+	sortedTemplates, crdList, hookList, err = helmClient.GenerateKubernetesArtifacts(chartPath,
 		[]string{prYamlClient.GetValues()},
 		values)
 	if err != nil {
-		return sortedTemplates, hookList, finalReleaseName, pkgerrors.Wrap(err, "Generate final k8s yaml")
+		return sortedTemplates, crdList, hookList, finalReleaseName, pkgerrors.Wrap(err, "Generate final k8s yaml")
 	}
 
-	return sortedTemplates, hookList, finalReleaseName, nil
+	return sortedTemplates, crdList, hookList, finalReleaseName, nil
 }
 
 // Returns an empty profile with the following contents
diff --git a/src/k8splugin/internal/rb/profile_test.go b/src/k8splugin/internal/rb/profile_test.go
index 3c40c2c9..2a9dc4fd 100644
--- a/src/k8splugin/internal/rb/profile_test.go
+++ b/src/k8splugin/internal/rb/profile_test.go
@@ -773,7 +773,7 @@ func TestResolveProfile(t *testing.T) {
 		t.Run(testCase.label, func(t *testing.T) {
 			db.DBconn = testCase.mockdb
 			impl := NewProfileClient()
-			data, _, releaseName, err := impl.Resolve(testCase.rbname,
+			data, _, _, releaseName, err := impl.Resolve(testCase.rbname,
 				testCase.rbversion, testCase.prname, []string{}, testCase.releaseName)
 			defer cleanup(data)
 			if err != nil {
diff --git a/src/k8splugin/plugins/generic/plugin.go b/src/k8splugin/plugins/generic/plugin.go
index 5815b74f..a210f6d6 100644
--- a/src/k8splugin/plugins/generic/plugin.go
+++ b/src/k8splugin/plugins/generic/plugin.go
@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	pkgerrors "github.com/pkg/errors"
+	"github.com/prometheus/common/log"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -305,7 +306,18 @@ func (g genericPlugin) Create(yamlFilePath string, namespace string, client plug
 	if err != nil {
 		return "", pkgerrors.Wrap(err, "Mapping kind to resource error")
 	}
-
+	if gvk.Kind == "CustomResourceDefinition" {
+		//according the helm spec, CRD is created only once, and we raise only warn if we try to do it once more
+		resource := helm.KubernetesResource{}
+		resource.GVK = gvk
+		resource.Name = unstruct.GetName()
+		name, err := g.Get(resource, namespace, client)
+		if err == nil && name == resource.Name {
+			//CRD update is not supported according to Helm spec
+			log.Warn(fmt.Sprintf("CRD %s create will be skipped. It already exists", name))
+			return name, nil
+		}
+	}
 	//Add the tracking label to all resources created here
 	labels := unstruct.GetLabels()
 	//Check if labels exist for this object
@@ -362,6 +374,18 @@ func (g genericPlugin) Update(yamlFilePath string, namespace string, client plug
 		return "", pkgerrors.Wrap(err, "Mapping kind to resource error")
 	}
 
+	if gvk.Kind == "CustomResourceDefinition" {
+		resource := helm.KubernetesResource{}
+		resource.GVK = gvk
+		resource.Name = unstruct.GetName()
+		name, err := g.Get(resource, namespace, client)
+		if err == nil && name == resource.Name {
+			//CRD update is not supported according to Helm spec
+			log.Warn(fmt.Sprintf("CRD %s update will be skipped", name))
+			return name, nil
+		}
+	}
+
 	//Add the tracking label to all resources created here
 	labels := unstruct.GetLabels()
 	//Check if labels exist for this object
@@ -463,6 +487,11 @@ func (g genericPlugin) Delete(resource helm.KubernetesResource, namespace string
 	opts := metav1.DeleteOptions{
 		PropagationPolicy: &deletePolicy,
 	}
+	if resource.GVK.Kind == "CustomResourceDefinition" {
+		//CRD deletion is not supported according to Helm spec
+		log.Warn(fmt.Sprintf("CRD %s deletion will be skipped", resource.Name))
+		return nil
+	}
 
 	switch mapping.Scope.Name() {
 	case meta.RESTScopeNameNamespace: