From a73b42b9c3877f1a34939d85941482f7f5c44db9 Mon Sep 17 00:00:00 2001 From: Lukasz Rajewski Date: Mon, 7 Feb 2022 19:34:25 +0100 Subject: Upgrade handler implementation Implementation of the upgrade handler for the instance. As a result, exsting instance is modified, resources upated and lefovers removed from the cluster. Issue-ID: MULTICLOUD-1444 Signed-off-by: Lukasz Rajewski Change-Id: I4122ee12d9332eaeb5ee016446b3da2bbe94bd2d --- src/k8splugin/api/api.go | 1 + src/k8splugin/api/brokerhandler.go | 2 +- src/k8splugin/api/instancehandler.go | 56 ++- src/k8splugin/api/instancehandler_test.go | 2 +- src/k8splugin/internal/app/client.go | 57 ++- src/k8splugin/internal/app/config_backend.go | 2 +- src/k8splugin/internal/app/hook.go | 38 +- src/k8splugin/internal/app/instance.go | 420 ++++++++++++++++++--- src/k8splugin/internal/app/instance_test.go | 7 +- .../internal/namegenerator/namegenerator.go | 3 + 10 files changed, 505 insertions(+), 83 deletions(-) diff --git a/src/k8splugin/api/api.go b/src/k8splugin/api/api.go index 4e84de70..64c83e03 100644 --- a/src/k8splugin/api/api.go +++ b/src/k8splugin/api/api.go @@ -53,6 +53,7 @@ func NewRouter(defClient rb.DefinitionManager, //Want to get full Data -> add query param: /install/{instID}?full=true instRouter.HandleFunc("/instance/{instID}", instHandler.getHandler).Methods("GET") instRouter.HandleFunc("/instance/{instID}/status", instHandler.statusHandler).Methods("GET") + instRouter.HandleFunc("/instance/{instID}/upgrade", instHandler.upgradeHandler).Methods("POST") instRouter.HandleFunc("/instance/{instID}/query", instHandler.queryHandler).Methods("GET") instRouter.HandleFunc("/instance/{instID}/query", instHandler.queryHandler). Queries("ApiVersion", "{ApiVersion}", diff --git a/src/k8splugin/api/brokerhandler.go b/src/k8splugin/api/brokerhandler.go index b4803101..ecfde8c4 100644 --- a/src/k8splugin/api/brokerhandler.go +++ b/src/k8splugin/api/brokerhandler.go @@ -175,7 +175,7 @@ func (b brokerInstanceHandler) createHandler(w http.ResponseWriter, r *http.Requ log.Info("Instance API Payload", log.Fields{ "payload": instReq, }) - resp, err := b.client.Create(instReq) + resp, err := b.client.Create(instReq, "") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/src/k8splugin/api/instancehandler.go b/src/k8splugin/api/instancehandler.go index e07bfcb0..6d1fd7b3 100644 --- a/src/k8splugin/api/instancehandler.go +++ b/src/k8splugin/api/instancehandler.go @@ -95,7 +95,7 @@ func (i instanceHandler) createHandler(w http.ResponseWriter, r *http.Request) { return } - resp, err := i.client.Create(resource) + resp, err := i.client.Create(resource, "") if err != nil { log.Error("Error Creating Resource", log.Fields{ "error": err, @@ -118,6 +118,60 @@ func (i instanceHandler) createHandler(w http.ResponseWriter, r *http.Request) { } } +func (i instanceHandler) upgradeHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + id := vars["instID"] + var resource app.UpgradeRequest + + err := json.NewDecoder(r.Body).Decode(&resource) + switch { + case err == io.EOF: + log.Error("Body Empty", log.Fields{ + "error": io.EOF, + }) + http.Error(w, "Body empty", http.StatusBadRequest) + return + case err != nil: + log.Error("Error unmarshaling Body", log.Fields{ + "error": err, + }) + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + + // Check body for expected parameters + err = i.validateBody(resource) + if err != nil { + log.Error("Invalid Parameters in Body", log.Fields{ + "error": err, + }) + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + + resp, err := i.client.Upgrade(id, resource) + if err != nil { + log.Error("Error Upgrading Resource", log.Fields{ + "error": err, + "resource": resource, + }) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + err = json.NewEncoder(w).Encode(resp) + if err != nil { + log.Error("Error Marshaling Response", log.Fields{ + "error": err, + "response": resp, + }) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + // getHandler retrieves information about an instance via the ID func (i instanceHandler) getHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) diff --git a/src/k8splugin/api/instancehandler_test.go b/src/k8splugin/api/instancehandler_test.go index faec1329..444b6695 100644 --- a/src/k8splugin/api/instancehandler_test.go +++ b/src/k8splugin/api/instancehandler_test.go @@ -48,7 +48,7 @@ type mockInstanceClient struct { err error } -func (m *mockInstanceClient) Create(inp app.InstanceRequest) (app.InstanceResponse, error) { +func (m *mockInstanceClient) Create(inp app.InstanceRequest, newId string) (app.InstanceResponse, error) { if m.err != nil { return app.InstanceResponse{}, m.err } diff --git a/src/k8splugin/internal/app/client.go b/src/k8splugin/internal/app/client.go index a2868cd5..3aabda22 100644 --- a/src/k8splugin/internal/app/client.go +++ b/src/k8splugin/internal/app/client.go @@ -19,6 +19,7 @@ package app import ( "context" + "encoding/json" "io/ioutil" appsv1 "k8s.io/api/apps/v1" @@ -77,6 +78,35 @@ type ResourceStatus struct { Status unstructured.Unstructured `json:"status"` } +type ResourceStatusKey struct { + Name string `json:"name"` + GVK schema.GroupVersionKind `json:"GVK"` +} + +// We will use json marshalling to convert to string to +// preserve the underlying structure. +func (rs ResourceStatus) Key() string { + key := ResourceStatusKey{ + Name: rs.Name, + GVK: rs.GVK, + } + out, err := json.Marshal(key) + if err != nil { + return "" + } + + return string(out) +} + +func (rs ResourceStatus) Value() string { + out, err := json.Marshal(rs.Status) + if err != nil { + return "" + } + + return string(out) +} + func (k *KubernetesClient) getObjTypeForHook(kind string) (runtime.Object, error) { switch kind { case "Job": @@ -463,7 +493,7 @@ func (k *KubernetesClient) CreateKind(resTempl helm.KubernetesResourceTemplate, } func (k *KubernetesClient) updateKind(resTempl helm.KubernetesResourceTemplate, - namespace string) (helm.KubernetesResource, error) { + namespace string, createIfDoNotExist bool) (helm.KubernetesResource, error) { if _, err := os.Stat(resTempl.FilePath); os.IsNotExist(err) { return helm.KubernetesResource{}, pkgerrors.New("File " + resTempl.FilePath + " does not exists") @@ -480,12 +510,21 @@ func (k *KubernetesClient) updateKind(resTempl helm.KubernetesResourceTemplate, updatedResourceName, err := pluginImpl.Update(resTempl.FilePath, namespace, k) if err != nil { - log.Error("Error Updating Resource", log.Fields{ - "error": err, - "gvk": resTempl.GVK, - "filepath": resTempl.FilePath, - }) - return helm.KubernetesResource{}, pkgerrors.Wrap(err, "Error in plugin "+resTempl.GVK.Kind+" plugin") + var failed = true + if createIfDoNotExist && strings.Contains(err.Error(), "not found") == true { + updatedResourceName, err = pluginImpl.Create(resTempl.FilePath, namespace, k) + if err == nil { + failed = false + } + } + if failed { + log.Error("Error Updating Resource", log.Fields{ + "error": err, + "gvk": resTempl.GVK, + "filepath": resTempl.FilePath, + }) + return helm.KubernetesResource{}, pkgerrors.Wrap(err, "Error in plugin "+resTempl.GVK.Kind+" plugin") + } } log.Info("Updated Kubernetes Resource", log.Fields{ @@ -521,7 +560,7 @@ func (k *KubernetesClient) createResources(sortedTemplates []helm.KubernetesReso } func (k *KubernetesClient) updateResources(sortedTemplates []helm.KubernetesResourceTemplate, - namespace string) ([]helm.KubernetesResource, error) { + namespace string, createIfDoNotExist bool) ([]helm.KubernetesResource, error) { err := k.ensureNamespace(namespace) if err != nil { @@ -530,7 +569,7 @@ func (k *KubernetesClient) updateResources(sortedTemplates []helm.KubernetesReso var updatedResources []helm.KubernetesResource for _, resTempl := range sortedTemplates { - resUpdated, err := k.updateKind(resTempl, namespace) + resUpdated, err := k.updateKind(resTempl, namespace, createIfDoNotExist) if err != nil { return nil, pkgerrors.Wrapf(err, "Error updating kind: %+v", resTempl.GVK) } diff --git a/src/k8splugin/internal/app/config_backend.go b/src/k8splugin/internal/app/config_backend.go index 4dcbeb57..80230896 100644 --- a/src/k8splugin/internal/app/config_backend.go +++ b/src/k8splugin/internal/app/config_backend.go @@ -558,7 +558,7 @@ func scheduleResources(c chan configResourceList) { var status string = "" if err != nil { // assuming - the err represent the resource already exist, so going for update - resProceeded, err = k8sClient.updateResources(resToCreateOrUpdate, inst.Namespace) + resProceeded, err = k8sClient.updateResources(resToCreateOrUpdate, inst.Namespace, false) if err != nil { log.Printf("Error Creating resources: %s", errCreate.Error()) log.Printf("Error Updating resources: %s", err.Error()) diff --git a/src/k8splugin/internal/app/hook.go b/src/k8splugin/internal/app/hook.go index ebf5f8e3..3a4af8a9 100644 --- a/src/k8splugin/internal/app/hook.go +++ b/src/k8splugin/internal/app/hook.go @@ -15,12 +15,13 @@ package app import ( "fmt" - "github.com/onap/multicloud-k8s/src/k8splugin/internal/db" - "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" - "helm.sh/helm/v3/pkg/release" "log" "strings" "time" + + "github.com/onap/multicloud-k8s/src/k8splugin/internal/db" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" + "helm.sh/helm/v3/pkg/release" ) // Timeout used when deleting resources with a hook-delete-policy. @@ -28,25 +29,30 @@ const defaultHookDeleteTimeoutInSeconds = int64(60) // HookClient implements the Helm Hook interface type HookClient struct { - kubeNameSpace string - id string - dbStoreName string - dbTagInst string + kubeNameSpace string + id string + dbStoreName string + dbTagInst string } -type MultiCloudHook struct{ +type MultiCloudHook struct { release.Hook Group string Version string } +type HookTimeoutInfo struct { + preInstallTimeOut, postInstallTimeOut, preDeleteTimeout, + postDeleteTimeout, preUpgradeTimeout, postUpgradeTimeout int64 +} + // NewHookClient returns a new instance of HookClient func NewHookClient(namespace, id, dbStoreName, dbTagInst string) *HookClient { return &HookClient{ kubeNameSpace: namespace, - id: id, - dbStoreName: dbStoreName, - dbTagInst: dbTagInst, + id: id, + dbStoreName: dbStoreName, + dbTagInst: dbTagInst, } } @@ -69,7 +75,7 @@ func (hc *HookClient) ExecHook( hook release.HookEvent, timeout int64, startIndex int, - dbData *InstanceDbData) (error){ + dbData *InstanceDbData) error { executingHooks := hc.getHookByEvent(hs, hook) key := InstanceKey{ ID: hc.id, @@ -91,7 +97,7 @@ func (hc *HookClient) ExecHook( //update DB here before the creation of the hook, if the plugin quits //-> when it comes back, it will continue from next hook and consider that this one is done if dbData != nil { - dbData.HookProgress = fmt.Sprintf("%d/%d", index + 1, len(executingHooks)) + dbData.HookProgress = fmt.Sprintf("%d/%d", index+1, len(executingHooks)) err := db.DBconn.Update(hc.dbStoreName, key, hc.dbTagInst, dbData) if err != nil { return err @@ -103,7 +109,7 @@ func (hc *HookClient) ExecHook( FilePath: h.KRT.FilePath, } createdHook, err := k8sClient.CreateKind(resTempl, hc.kubeNameSpace) - if err != nil { + if err != nil { log.Printf(" Instance: %s, Warning: %s hook %s, filePath: %s, error: %s", hc.id, hook, h.Hook.Name, h.KRT.FilePath, err) hc.deleteHookByPolicy(h, release.HookFailed, k8sClient) return err @@ -148,7 +154,7 @@ func (hc *HookClient) deleteHookByPolicy(h *helm.Hook, policy release.HookDelete if strings.Contains(errHookDelete.Error(), "not found") { return nil } else { - log.Printf(" Instance: %s, Warning: hook %s, filePath %s could not be deleted: %s", hc.id, h.Hook.Name, h.KRT.FilePath ,errHookDelete) + log.Printf(" Instance: %s, Warning: hook %s, filePath %s could not be deleted: %s", hc.id, h.Hook.Name, h.KRT.FilePath, errHookDelete) return errHookDelete } } else { @@ -180,4 +186,4 @@ func hookHasDeletePolicy(h *helm.Hook, policy release.HookDeletePolicy) bool { } } return false -} \ No newline at end of file +} diff --git a/src/k8splugin/internal/app/instance.go b/src/k8splugin/internal/app/instance.go index 71042f08..e78eea77 100644 --- a/src/k8splugin/internal/app/instance.go +++ b/src/k8splugin/internal/app/instance.go @@ -1,7 +1,7 @@ /* * Copyright 2018 Intel Corporation, Inc * Copyright © 2021 Samsung Electronics - * Copyright © 2021 Orange + * Copyright © 2022 Orange * Copyright © 2021 Nokia Bell Labs * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "log" + "reflect" "strconv" "strings" "time" @@ -57,6 +58,17 @@ type InstanceRequest struct { OverrideValues map[string]string `json:"override-values"` } +// UpgradeRequest contains the parameters needed for instantiation +// of profiles +type UpgradeRequest struct { + RBName string `json:"rb-name"` + RBVersion string `json:"rb-version"` + ProfileName string `json:"profile-name"` + CloudRegion string `json:"cloud-region"` + Labels map[string]string `json:"labels"` + OverrideValues map[string]string `json:"override-values"` +} + // InstanceResponse contains the response from instantiation type InstanceResponse struct { ID string `json:"id"` @@ -81,6 +93,8 @@ type InstanceDbData struct { PostInstallTimeout int64 `json:"PostInstallTimeout"` PreDeleteTimeout int64 `json:"PreDeleteTimeout"` PostDeleteTimeout int64 `json:"PostDeleteTimeout"` + PreUpgradeTimeout int64 `json:"PreUpgradeTimeout"` + PostUpgradeTimeout int64 `json:"PostUpgradeTimeout"` } // InstanceMiniResponse contains the response from instantiation @@ -103,7 +117,8 @@ type InstanceStatus struct { // InstanceManager is an interface exposes the instantiation functionality type InstanceManager interface { - Create(i InstanceRequest) (InstanceResponse, error) + Create(i InstanceRequest, newId string) (InstanceResponse, error) + Upgrade(id string, u UpgradeRequest) (InstanceResponse, error) Get(id string) (InstanceResponse, error) GetFull(id string) (InstanceDbData, error) Status(id string) (InstanceStatus, error) @@ -159,86 +174,119 @@ func resolveModelFromInstance(instanceID string) (rbName, rbVersion, profileName return resp.Request.RBName, resp.Request.RBVersion, resp.Request.ProfileName, resp.ReleaseName, nil } -// Create an instance of rb on the cluster in the database -func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { - // Name is required - if i.RBName == "" || i.RBVersion == "" || i.ProfileName == "" || i.CloudRegion == "" { - return InstanceResponse{}, - pkgerrors.New("RBName, RBversion, ProfileName, CloudRegion are required to create a new instance") - } - - //Check if profile exists - profile, err := rb.NewProfileClient().Get(i.RBName, i.RBVersion, i.ProfileName) - if err != nil { - return InstanceResponse{}, pkgerrors.New("Unable to find Profile to create instance") - } - +func getOverridesAndHookInfo(i InstanceRequest) ([]string, HookTimeoutInfo, error) { //Convert override values from map to array of strings of the following format //foo=bar overrideValues := []string{} - var preInstallTimeOut, postInstallTimeOut, preDeleteTimeout, postDeleteTimeout int64 + var hookTimeoutInfo = HookTimeoutInfo{} + var err error = nil if i.OverrideValues != nil { preInstallTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-pre-install-timeout"] if !ok { preInstallTimeOutStr = "60" } - preInstallTimeOut, err = strconv.ParseInt(preInstallTimeOutStr, 10, 64) + hookTimeoutInfo.preInstallTimeOut, err = strconv.ParseInt(preInstallTimeOutStr, 10, 64) if err != nil { - return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-pre-install-timeout") + return overrideValues, hookTimeoutInfo, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-pre-install-timeout") } postInstallTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-post-install-timeout"] if !ok { postInstallTimeOutStr = "600" } - postInstallTimeOut, err = strconv.ParseInt(postInstallTimeOutStr, 10, 64) + hookTimeoutInfo.postInstallTimeOut, err = strconv.ParseInt(postInstallTimeOutStr, 10, 64) if err != nil { - return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-post-install-timeout") + return overrideValues, hookTimeoutInfo, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-post-install-timeout") } preDeleteTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-pre-delete-timeout"] if !ok { preDeleteTimeOutStr = "60" } - preDeleteTimeout, err = strconv.ParseInt(preDeleteTimeOutStr, 10, 64) + hookTimeoutInfo.preDeleteTimeout, err = strconv.ParseInt(preDeleteTimeOutStr, 10, 64) if err != nil { - return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-pre-delete-timeout") + return overrideValues, hookTimeoutInfo, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-pre-delete-timeout") } postDeleteTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-post-delete-timeout"] if !ok { postDeleteTimeOutStr = "600" } - postDeleteTimeout, err = strconv.ParseInt(postDeleteTimeOutStr, 10, 64) + hookTimeoutInfo.postDeleteTimeout, err = strconv.ParseInt(postDeleteTimeOutStr, 10, 64) + if err != nil { + return overrideValues, hookTimeoutInfo, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-post-delete-timeout") + } + + preUpgradeTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-pre-upgrade-timeout"] + if !ok { + preUpgradeTimeOutStr = "60" + } + hookTimeoutInfo.preUpgradeTimeout, err = strconv.ParseInt(preUpgradeTimeOutStr, 10, 64) + if err != nil { + return overrideValues, hookTimeoutInfo, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-pre-upgrade-timeout") + } + + postUpgradeTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-post-upgrade-timeout"] + if !ok { + postUpgradeTimeOutStr = "600" + } + hookTimeoutInfo.postUpgradeTimeout, err = strconv.ParseInt(postUpgradeTimeOutStr, 10, 64) if err != nil { - return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-post-delete-timeout") + return overrideValues, hookTimeoutInfo, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-post-upgrade-timeout") } for k, v := range i.OverrideValues { overrideValues = append(overrideValues, k+"="+v) } } else { - preInstallTimeOut = 60 - postInstallTimeOut = 600 - preDeleteTimeout = 60 - postDeleteTimeout = 600 + hookTimeoutInfo.preInstallTimeOut = 60 + hookTimeoutInfo.postInstallTimeOut = 600 + hookTimeoutInfo.preDeleteTimeout = 60 + hookTimeoutInfo.postDeleteTimeout = 600 + hookTimeoutInfo.preUpgradeTimeout = 60 + hookTimeoutInfo.postUpgradeTimeout = 600 + } + return overrideValues, hookTimeoutInfo, nil +} + +// Create an instance of rb on the cluster in the database +func (v *InstanceClient) Create(i InstanceRequest, newId string) (InstanceResponse, error) { + // Name is required + if i.RBName == "" || i.RBVersion == "" || i.ProfileName == "" || i.CloudRegion == "" { + return InstanceResponse{}, + pkgerrors.New("RBName, RBversion, ProfileName, CloudRegion are required to create a new instance") + } + + //Check if profile exists + profile, err := rb.NewProfileClient().Get(i.RBName, i.RBVersion, i.ProfileName) + if err != nil { + return InstanceResponse{}, pkgerrors.New("Unable to find Profile to create instance") } - id := namegenerator.Generate() + overrideValues, hookTimeoutInfo, err := getOverridesAndHookInfo(i) - overrideValues = append(overrideValues, "k8s-rb-instance-id="+id) + var generatedId string = "" + var finalId string = "" + if newId == "" { + generatedId = namegenerator.Generate() + finalId = generatedId + } else { + finalId = newId + } + + overrideValues = append(overrideValues, "k8s-rb-instance-id="+finalId) //Execute the kubernetes create command sortedTemplates, crdList, hookList, releaseName, err := rb.NewProfileClient().Resolve(i.RBName, i.RBVersion, i.ProfileName, overrideValues, i.ReleaseName) if err != nil { - namegenerator.Release(id) + namegenerator.Release(generatedId) return InstanceResponse{}, pkgerrors.Wrap(err, "Error resolving helm charts") } k8sClient := KubernetesClient{} - err = k8sClient.Init(i.CloudRegion, id) + err = k8sClient.Init(i.CloudRegion, finalId) if err != nil { - namegenerator.Release(id) + namegenerator.Release(generatedId) return InstanceResponse{}, pkgerrors.Wrap(err, "Getting CloudRegion Information") } @@ -262,7 +310,7 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { log.Printf(" DeletePolicies: %s", h.Hook.DeletePolicies) } dbData := InstanceDbData{ - ID: id, + ID: finalId, Request: i, Namespace: profile.Namespace, ReleaseName: releaseName, @@ -270,24 +318,26 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { Resources: []helm.KubernetesResource{}, Hooks: hookList, HookProgress: "", - PreInstallTimeout: preInstallTimeOut, - PostInstallTimeout: postInstallTimeOut, - PreDeleteTimeout: preDeleteTimeout, - PostDeleteTimeout: postDeleteTimeout, + PreInstallTimeout: hookTimeoutInfo.preInstallTimeOut, + PostInstallTimeout: hookTimeoutInfo.postInstallTimeOut, + PreDeleteTimeout: hookTimeoutInfo.preDeleteTimeout, + PostDeleteTimeout: hookTimeoutInfo.postDeleteTimeout, + PreUpgradeTimeout: hookTimeoutInfo.preUpgradeTimeout, + PostUpgradeTimeout: hookTimeoutInfo.postUpgradeTimeout, } err = k8sClient.ensureNamespace(profile.Namespace) if err != nil { - namegenerator.Release(id) + namegenerator.Release(generatedId) return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Namespace") } key := InstanceKey{ - ID: id, + ID: finalId, } err = db.DBconn.Create(v.storeName, key, v.tagInst, dbData) if err != nil { - namegenerator.Release(id) + namegenerator.Release(generatedId) return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Instance DB Entry") } @@ -300,16 +350,16 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { } } - hookClient := NewHookClient(profile.Namespace, id, v.storeName, v.tagInst) + hookClient := NewHookClient(profile.Namespace, finalId, v.storeName, v.tagInst) if len(hookClient.getHookByEvent(hookList, release.HookPreInstall)) != 0 { - err = hookClient.ExecHook(k8sClient, hookList, release.HookPreInstall, preInstallTimeOut, 0, &dbData) + err = hookClient.ExecHook(k8sClient, hookList, release.HookPreInstall, hookTimeoutInfo.preInstallTimeOut, 0, &dbData) if err != nil { log.Printf("Error running preinstall hooks for release %s, Error: %s. Stop here", releaseName, err) err2 := db.DBconn.Delete(v.storeName, key, v.tagInst) if err2 != nil { log.Printf("Error cleaning failed instance in DB, please check DB.") } else { - namegenerator.Release(id) + namegenerator.Release(generatedId) } return InstanceResponse{}, pkgerrors.Wrap(err, "Error running preinstall hooks") } @@ -322,7 +372,7 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { if err2 != nil { log.Printf("Delete Instance DB Entry for release %s has error.", releaseName) } else { - namegenerator.Release(id) + namegenerator.Release(generatedId) } return InstanceResponse{}, pkgerrors.Wrap(err, "Update Instance DB Entry") } @@ -334,13 +384,13 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { log.Printf("[Instance] Reverting created resources on Error: %s", err.Error()) k8sClient.deleteResources(helm.GetReverseK8sResources(createdResources), profile.Namespace) } - log.Printf(" Instance: %s, Main rss are failed, skip post-install and remove instance in DB", id) + log.Printf(" Instance: %s, Main rss are failed, skip post-install and remove instance in DB", finalId) //main rss creation failed -> remove instance in DB err2 := db.DBconn.Delete(v.storeName, key, v.tagInst) if err2 != nil { log.Printf("Delete Instance DB Entry for release %s has error.", releaseName) } else { - namegenerator.Release(id) + namegenerator.Release(generatedId) } return InstanceResponse{}, pkgerrors.Wrap(err, "Create Kubernetes Resources") } @@ -354,7 +404,7 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { //Compose the return response resp := InstanceResponse{ - ID: id, + ID: finalId, Request: i, Namespace: profile.Namespace, ReleaseName: releaseName, @@ -366,10 +416,272 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { go func() { dbData.Status = "POST-INSTALL" dbData.HookProgress = "" - err = hookClient.ExecHook(k8sClient, hookList, release.HookPostInstall, postInstallTimeOut, 0, &dbData) + err = hookClient.ExecHook(k8sClient, hookList, release.HookPostInstall, hookTimeoutInfo.postInstallTimeOut, 0, &dbData) if err != nil { dbData.Status = "POST-INSTALL-FAILED" - log.Printf(" Instance: %s, Error running postinstall hooks error: %s", id, err) + log.Printf(" Instance: %s, Error running postinstall hooks error: %s", finalId, err) + } else { + dbData.Status = "DONE" + } + err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData) + if err != nil { + log.Printf("Update Instance DB Entry for release %s has error.", releaseName) + } + }() + } else { + dbData.Status = "DONE" + err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData) + if err != nil { + log.Printf("Update Instance DB Entry for release %s has error.", releaseName) + } + } + + return resp, nil +} + +// Simplified function to retrieve model data from instance ID +func upgradeRequestToInstanceRequest(instance InstanceResponse, u UpgradeRequest) InstanceRequest { + i := InstanceRequest{} + i.CloudRegion = u.CloudRegion + i.Labels = u.Labels + i.OverrideValues = u.OverrideValues + i.ProfileName = u.ProfileName + i.RBName = u.RBName + i.RBVersion = u.RBVersion + i.ReleaseName = instance.ReleaseName + + return i +} + +// Upgrade an instance of rb on the cluster in the database +func (v *InstanceClient) Upgrade(id string, u UpgradeRequest) (InstanceResponse, error) { + key := InstanceKey{ + ID: id, + } + value, err := db.DBconn.Read(v.storeName, key, v.tagInst) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Upgrade Instance") + } + if value == nil { //value is a byte array + return InstanceResponse{}, pkgerrors.New("Status is not available") + } + currentInstance := InstanceResponse{} + currentInstanceFull := InstanceDbData{} + err = db.DBconn.Unmarshal(value, ¤tInstance) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Demarshaling Instance Value") + } + err = db.DBconn.Unmarshal(value, ¤tInstanceFull) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Demarshaling Instance Value") + } + i := upgradeRequestToInstanceRequest(currentInstance, u) + + // Required parameters + if i.RBName == "" || i.RBVersion == "" || i.ProfileName == "" || i.CloudRegion == "" { + return InstanceResponse{}, + pkgerrors.New("RBName, RBversion, ProfileName, CloudRegion are required to upgrade the instance") + } + + if reflect.DeepEqual(i, currentInstance.Request) && currentInstanceFull.Status == "DONE" { + log.Printf("Nothing to do for instance upgrade") + return currentInstance, nil + } + + if currentInstance.Request.CloudRegion != u.CloudRegion { + newInstance, err := v.Create(i, "") + if err == nil { + err = v.Delete(id) + if err == nil { + newInstanceDb, err := v.GetFull(newInstance.ID) + oldKey := InstanceKey{ + ID: newInstance.ID, + } + err2 := db.DBconn.Delete(v.storeName, oldKey, v.tagInst) + if err2 != nil { + log.Printf("Delete of the temporal instance from the DB has failed %s", err2.Error()) + } + namegenerator.Release(newInstance.ID) + newInstanceDb.ID = id + newInstance.ID = id + err = db.DBconn.Create(v.storeName, key, v.tagInst, newInstanceDb) + if err != nil { + return newInstance, pkgerrors.Wrap(err, "Create Instance DB Entry after update failed") + } + return newInstance, nil + } else { + err2 := v.Delete(newInstance.ID) + if err2 != nil { + log.Printf("Delete of the instance from the new region failed with error %s", err2.Error()) + } + return InstanceResponse{}, pkgerrors.Wrap(err, "Deletion of instance in the old region failed") + } + } else { + return InstanceResponse{}, pkgerrors.Wrap(err, "Creation of instance in new region failed") + } + } + + //Check if profile exists + profile, err := rb.NewProfileClient().Get(i.RBName, i.RBVersion, i.ProfileName) + if err != nil { + return InstanceResponse{}, pkgerrors.New("Unable to find Profile to create instance") + } + + overrideValues, hookTimeoutInfo, err := getOverridesAndHookInfo(i) + + overrideValues = append(overrideValues, "k8s-rb-instance-id="+id) + + //Execute the kubernetes create command + sortedTemplates, crdList, hookList, releaseName, err := rb.NewProfileClient().Resolve(i.RBName, i.RBVersion, i.ProfileName, overrideValues, i.ReleaseName) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Error resolving helm charts") + } + + k8sClient := KubernetesClient{} + err = k8sClient.Init(i.CloudRegion, id) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Getting CloudRegion Information") + } + + log.Printf("Main rss info") + for _, t := range sortedTemplates { + log.Printf(" Path: %s", t.FilePath) + log.Printf(" Kind: %s", t.GVK.Kind) + } + + log.Printf("Crd rss info") + for _, t := range crdList { + log.Printf(" Path: %s", t.FilePath) + log.Printf(" Kind: %s", t.GVK.Kind) + } + + log.Printf("Hook info") + for _, h := range hookList { + log.Printf(" Name: %s", h.Hook.Name) + log.Printf(" Events: %s", h.Hook.Events) + log.Printf(" Weight: %d", h.Hook.Weight) + log.Printf(" DeletePolicies: %s", h.Hook.DeletePolicies) + } + dbData := InstanceDbData{ + ID: id, + Request: i, + Namespace: profile.Namespace, + ReleaseName: releaseName, + Status: "PRE-UPGRADE", + Resources: []helm.KubernetesResource{}, + Hooks: hookList, + HookProgress: "", + PreInstallTimeout: hookTimeoutInfo.preInstallTimeOut, + PostInstallTimeout: hookTimeoutInfo.postInstallTimeOut, + PreDeleteTimeout: hookTimeoutInfo.preDeleteTimeout, + PostDeleteTimeout: hookTimeoutInfo.postDeleteTimeout, + PreUpgradeTimeout: hookTimeoutInfo.preUpgradeTimeout, + PostUpgradeTimeout: hookTimeoutInfo.postUpgradeTimeout, + } + + err = k8sClient.ensureNamespace(profile.Namespace) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Namespace") + } + + err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Updating Instance DB Entry") + } + + if len(crdList) > 0 { + log.Printf("Pre-Installing CRDs") + _, err = k8sClient.createResources(crdList, profile.Namespace) + + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Pre-Installing CRDs") + } + } + + hookClient := NewHookClient(profile.Namespace, id, v.storeName, v.tagInst) + if len(hookClient.getHookByEvent(hookList, release.HookPreUpgrade)) != 0 { + err = hookClient.ExecHook(k8sClient, hookList, release.HookPreUpgrade, hookTimeoutInfo.preUpgradeTimeout, 0, &dbData) + if err != nil { + log.Printf("Error running preupgrade hooks for release %s, Error: %s. Stop here", releaseName, err) + return InstanceResponse{}, pkgerrors.Wrap(err, "Error running preupgrade hooks") + } + } + + dbData.Status = "UPGRADING" + err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Update Instance DB Entry") + } + + upgradedResources, err := k8sClient.updateResources(sortedTemplates, profile.Namespace, true) + if err != nil { + log.Printf(" Instance: %s, Main rss are failed, skip post-upgrade", id) + return InstanceResponse{}, pkgerrors.Wrap(err, "Upgrade Kubernetes Resources") + } + + var resToDelete = make([]helm.KubernetesResource, 0) + for _, pastRes := range currentInstance.Resources { + var exists = false + for _, res := range upgradedResources { + if res.Name == pastRes.Name && res.GVK == pastRes.GVK { + if profile.Namespace == currentInstance.Namespace { + exists = true + break + } else { + status1, err := k8sClient.GetResourceStatus(res, profile.Namespace) + status2, err2 := k8sClient.GetResourceStatus(pastRes, currentInstance.Namespace) + if err == nil && err2 == nil && status1.Value() == status2.Value() { + //only when resource is namespace-less + exists = true + break + } + } + } + } + if !exists { + resToDelete = append(resToDelete, pastRes) + } + } + + err = k8sClient.deleteResources(helm.GetReverseK8sResources(resToDelete), currentInstance.Namespace) + + configClient := NewConfigClient() + configList, err := configClient.List(id) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Cannot retrieve former configuration list") + } + for _, config := range configList { + err = configClient.DeleteAll(id, config.ConfigName, true) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Failed to delete config after upgrade") + } + } + + dbData.Status = "UPGRADED" + dbData.Resources = upgradedResources + err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData) + if err != nil { + return InstanceResponse{}, pkgerrors.Wrap(err, "Update Instance DB Entry") + } + + //Compose the return response + resp := InstanceResponse{ + ID: id, + Request: i, + Namespace: profile.Namespace, + ReleaseName: releaseName, + Resources: upgradedResources, + Hooks: hookList, + } + + if len(hookClient.getHookByEvent(hookList, release.HookPostUpgrade)) != 0 { + go func() { + dbData.Status = "POST-UPGRADE" + dbData.HookProgress = "" + err = hookClient.ExecHook(k8sClient, hookList, release.HookPostUpgrade, hookTimeoutInfo.postUpgradeTimeout, 0, &dbData) + if err != nil { + dbData.Status = "POST-UPGRADE-FAILED" + log.Printf(" Instance: %s, Error running postupgrade hooks error: %s", id, err) } else { dbData.Status = "DONE" } @@ -423,6 +735,12 @@ func (v *InstanceClient) GetFull(id string) (InstanceDbData, error) { if resp.PostDeleteTimeout == 0 { resp.PostDeleteTimeout = 600 } + if resp.PreUpgradeTimeout == 0 { + resp.PreInstallTimeout = 60 + } + if resp.PostUpgradeTimeout == 0 { + resp.PostDeleteTimeout = 600 + } return resp, nil } diff --git a/src/k8splugin/internal/app/instance_test.go b/src/k8splugin/internal/app/instance_test.go index 890c4c99..86955fa3 100644 --- a/src/k8splugin/internal/app/instance_test.go +++ b/src/k8splugin/internal/app/instance_test.go @@ -16,13 +16,14 @@ package app import ( "encoding/base64" - "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils" "io/ioutil" "log" "reflect" "sort" "testing" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/connection" "github.com/onap/multicloud-k8s/src/k8splugin/internal/db" "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" @@ -172,7 +173,7 @@ func TestInstanceCreate(t *testing.T) { CloudRegion: "mock_connection", } - ir, err := ic.Create(input) + ir, err := ic.Create(input, "") if err != nil { t.Fatalf("TestInstanceCreate returned an error (%s)", err) } @@ -879,7 +880,7 @@ func TestInstanceWithHookCreate(t *testing.T) { CloudRegion: "mock_connection", } - ir, err := ic.Create(input) + ir, err := ic.Create(input, "") if err != nil { t.Fatalf("TestInstanceWithHookCreate returned an error (%s)", err) } diff --git a/src/k8splugin/internal/namegenerator/namegenerator.go b/src/k8splugin/internal/namegenerator/namegenerator.go index 0a49633a..8eb89546 100644 --- a/src/k8splugin/internal/namegenerator/namegenerator.go +++ b/src/k8splugin/internal/namegenerator/namegenerator.go @@ -142,6 +142,9 @@ func (c *cache) generateName() string { } func (c *cache) releaseName(name string) { + if name == "" { + return + } c.mux.Lock() defer c.mux.Unlock() -- cgit 1.2.3-korg