Further fixes for config delete operation

The issue was related with insufficient handlijg of
different versions of config vs their delete operation
handled by the plugin.

Issue-ID: MULTICLOUD-1332
Signed-off-by: Lukasz Rajewski <lukasz.rajewski@orange.com>
Change-Id: I90d896720fa89ebd66cb3290cdd9401272f5e3fd

7 files changed, 309 insertions, 105 deletions

diff --git a/src/k8splugin/internal/app/client.go b/src/k8splugin/internal/app/client.go
index 9813333e..06c4c464 100644
--- a/src/k8splugin/internal/app/client.go
+++ b/src/k8splugin/internal/app/client.go
@@ -20,25 +20,30 @@ package app
 import (
 	"context"
 	"io/ioutil"
+
 	appsv1 "k8s.io/api/apps/v1"
+
 	//appsv1beta1 "k8s.io/api/apps/v1beta1"
 	//appsv1beta2 "k8s.io/api/apps/v1beta2"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+
 	//extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
 	//apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	//apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"os"
 	"strings"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	logger "log"
+
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/config"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/connection"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
 	log "github.com/onap/multicloud-k8s/src/k8splugin/internal/logutils"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin"
-	logger "log"
 
 	pkgerrors "github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -545,9 +550,18 @@ func (k *KubernetesClient) DeleteKind(resource helm.KubernetesResource, namespac
 		return pkgerrors.Wrap(err, "Error loading plugin")
 	}
 
-	err = pluginImpl.Delete(resource, namespace, k)
-	if err != nil {
-		return pkgerrors.Wrap(err, "Error deleting "+resource.Name)
+	name, err := pluginImpl.Get(resource, namespace, k)
+
+	if (err == nil && name == resource.Name) || (err != nil && strings.Contains(err.Error(), "not found") == false) {
+		err = pluginImpl.Delete(resource, namespace, k)
+		if err != nil {
+			return pkgerrors.Wrap(err, "Error deleting "+resource.Name)
+		}
+	} else {
+		log.Warn("Resource does not exist, Skipping delete", log.Fields{
+			"gvk":      resource.GVK,
+			"resource": resource.Name,
+		})
 	}
 
 	return nil
diff --git a/src/k8splugin/internal/app/config.go b/src/k8splugin/internal/app/config.go
index a25ab543..8952c16d 100644
--- a/src/k8splugin/internal/app/config.go
+++ b/src/k8splugin/internal/app/config.go
@@ -18,10 +18,11 @@
 package app
 
 import (
+	"log"
 	"strconv"
 	"strings"
 
-	"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
+	"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
 
 	pkgerrors "github.com/pkg/errors"
 )
@@ -66,7 +67,9 @@ type ConfigManager interface {
 	Help() map[string]string
 	Update(instanceID, configName string, p Config) (ConfigResult, error)
 	Delete(instanceID, configName string) (ConfigResult, error)
+	DeleteAll(instanceID, configName string) error
 	Rollback(instanceID string, configName string, p ConfigRollback) error
+	Cleanup(instanceID string) error
 	Tagit(instanceID string, configName string, p ConfigTagit) error
 }
 
@@ -94,7 +97,7 @@ func (v *ConfigClient) Help() map[string]string {
 
 // Create an entry for the config in the database
 func (v *ConfigClient) Create(instanceID string, p Config) (ConfigResult, error) {
-
+	log.Printf("[Config Create] Instance %s", instanceID)
 	// Check required fields
 	if p.ConfigName == "" || p.TemplateName == "" || len(p.Values) == 0 {
 		return ConfigResult{}, pkgerrors.New("Incomplete Configuration Provided")
@@ -120,10 +123,12 @@ func (v *ConfigClient) Create(instanceID string, p Config) (ConfigResult, error)
 	// Acquire per profile Mutex
 	lock.Lock()
 	defer lock.Unlock()
-	err = applyConfig(instanceID, p, profileChannel, "POST")
+	var appliedResources ([]helm.KubernetesResource)
+	appliedResources, err = applyConfig(instanceID, p, profileChannel, "POST", nil)
 	if err != nil {
 		return ConfigResult{}, pkgerrors.Wrap(err, "Apply Config failed")
 	}
+	log.Printf("POST result: %s", appliedResources)
 	// Create Config DB Entry
 	err = cs.createConfig(p)
 	if err != nil {
@@ -134,7 +139,7 @@ func (v *ConfigClient) Create(instanceID string, p Config) (ConfigResult, error)
 		instanceID: instanceID,
 		configName: p.ConfigName,
 	}
-	version, err := cvs.createConfigVersion(p, Config{}, "POST")
+	version, err := cvs.createConfigVersion(p, Config{}, "POST", appliedResources)
 	if err != nil {
 		return ConfigResult{}, pkgerrors.Wrap(err, "Create Config Version DB Entry")
 	}
@@ -154,7 +159,7 @@ func (v *ConfigClient) Create(instanceID string, p Config) (ConfigResult, error)
 
 // Update an entry for the config in the database
 func (v *ConfigClient) Update(instanceID, configName string, p Config) (ConfigResult, error) {
-
+	log.Printf("[Config Update] Instance %s Config %s", instanceID, configName)
 	// Check required fields
 	if len(p.Values) == 0 {
 		return ConfigResult{}, pkgerrors.New("Incomplete Configuration Provided")
@@ -177,10 +182,12 @@ func (v *ConfigClient) Update(instanceID, configName string, p Config) (ConfigRe
 	// Acquire per profile Mutex
 	lock.Lock()
 	defer lock.Unlock()
-	err = applyConfig(instanceID, p, profileChannel, "PUT")
+	var appliedResources ([]helm.KubernetesResource)
+	appliedResources, err = applyConfig(instanceID, p, profileChannel, "PUT", nil)
 	if err != nil {
 		return ConfigResult{}, pkgerrors.Wrap(err, "Apply Config  failed")
 	}
+	log.Printf("PUT result: %s", appliedResources)
 	// Update Config DB Entry
 	configPrev, err := cs.updateConfig(p)
 	if err != nil {
@@ -191,7 +198,7 @@ func (v *ConfigClient) Update(instanceID, configName string, p Config) (ConfigRe
 		instanceID: instanceID,
 		configName: configName,
 	}
-	version, err := cvs.createConfigVersion(p, configPrev, "PUT")
+	version, err := cvs.createConfigVersion(p, configPrev, "PUT", appliedResources)
 	if err != nil {
 		return ConfigResult{}, pkgerrors.Wrap(err, "Create Config Version DB Entry")
 	}
@@ -247,8 +254,49 @@ func (v *ConfigClient) List(instanceID string) ([]Config, error) {
 }
 
 // Delete the Config from database
-func (v *ConfigClient) Delete(instanceID, configName string) (ConfigResult, error) {
+func (v *ConfigClient) DeleteAll(instanceID, configName string) error {
+	log.Printf("[Config Delete All] Instance %s Config %s", instanceID, configName)
+	// Check if Config exists
+	cs := ConfigStore{
+		instanceID: instanceID,
+		configName: configName,
+	}
+	_, err := cs.getConfig()
+	if err != nil {
+		return pkgerrors.Wrap(err, "Update Error - Config doesn't exist")
+	}
+	// Get Version Entry in DB for Config
+	cvs := ConfigVersionStore{
+		instanceID: instanceID,
+		configName: configName,
+	}
+	currentVersion, err := cvs.getCurrentVersion(configName)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Current version get failed")
+	}
+	_, _, action, _, err := cvs.getConfigVersion(configName, currentVersion)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Config version get failed")
+	}
+
+	if action != "DELETE" {
+		_, err = v.Delete(instanceID, configName)
+		if err != nil {
+			return pkgerrors.Wrap(err, "Config  DELETE version failed")
+		}
+	}
+	// Delete Config from DB
+	_, err = cs.deleteConfig()
+	if err != nil {
+		return pkgerrors.Wrap(err, "Delete Config DB Entry")
+	}
+	cvs.cleanupIstanceTags(configName)
+	return nil
+}
 
+// Apply update with delete operation
+func (v *ConfigClient) Delete(instanceID, configName string) (ConfigResult, error) {
+	log.Printf("[Config Delete] Instance %s Config %s", instanceID, configName)
 	// Resolving rbName, Version, etc. not to break response
 	rbName, rbVersion, profileName, _, err := resolveModelFromInstance(instanceID)
 	if err != nil {
@@ -261,29 +309,39 @@ func (v *ConfigClient) Delete(instanceID, configName string) (ConfigResult, erro
 	}
 	p, err := cs.getConfig()
 	if err != nil {
-		return ConfigResult{}, pkgerrors.Wrap(err, "Update Error - Config doesn't exist")
+		return ConfigResult{}, pkgerrors.Wrap(err, "Delete Error - Config doesn't exist")
 	}
 	lock, profileChannel := getProfileData(instanceID)
 	// Acquire per profile Mutex
 	lock.Lock()
 	defer lock.Unlock()
-	err = applyConfig(instanceID, p, profileChannel, "DELETE")
-	if err != nil {
-		return ConfigResult{}, pkgerrors.Wrap(err, "Apply Config  failed")
-	}
-	// Delete Config from DB
-	configPrev, err := cs.deleteConfig()
-	if err != nil {
-		return ConfigResult{}, pkgerrors.Wrap(err, "Delete Config DB Entry")
-	}
 	// Create Version Entry in DB for Config
 	cvs := ConfigVersionStore{
 		instanceID: instanceID,
 		configName: configName,
 	}
-	version, err := cvs.createConfigVersion(Config{}, configPrev, "DELETE")
+	currentVersion, err := cvs.getCurrentVersion(configName)
+	if err != nil {
+		return ConfigResult{}, pkgerrors.Wrap(err, "Current version get failed")
+	}
+	_, _, _, resources, err := cvs.getConfigVersion(configName, currentVersion)
+	if err != nil {
+		return ConfigResult{}, pkgerrors.Wrap(err, "Config version get failed")
+	}
+
+	_, err = applyConfig(instanceID, p, profileChannel, "DELETE", resources)
+	if err != nil {
+		return ConfigResult{}, pkgerrors.Wrap(err, "Apply Config  failed")
+	}
+	log.Printf("DELETE resources: [%s]", resources)
+	// Update Config from DB
+	configPrev, err := cs.updateConfig(p)
 	if err != nil {
-		return ConfigResult{}, pkgerrors.Wrap(err, "Delete Config Version DB Entry")
+		return ConfigResult{}, pkgerrors.Wrap(err, "Update Config DB Entry")
+	}
+	version, err := cvs.createConfigVersion(p, configPrev, "DELETE", []helm.KubernetesResource{})
+	if err != nil {
+		return ConfigResult{}, pkgerrors.Wrap(err, "Create Delete Config Version DB Entry")
 	}
 
 	// Create Result structure
@@ -301,7 +359,7 @@ func (v *ConfigClient) Delete(instanceID, configName string) (ConfigResult, erro
 
 // Rollback starts from current version and rollbacks to the version desired
 func (v *ConfigClient) Rollback(instanceID string, configName string, rback ConfigRollback) error {
-
+	log.Printf("[Config Rollback] Instance %s Config %s", instanceID, configName)
 	var reqVersion string
 	var err error
 
@@ -342,40 +400,35 @@ func (v *ConfigClient) Rollback(instanceID string, configName string, rback Conf
 
 	//Rollback all the intermettinent configurations
 	for i := currentVersion; i > rollbackIndex; i-- {
-		configNew, configPrev, action, err := cvs.getConfigVersion(configName, i)
+		configNew, configPrev, _, resources, err := cvs.getConfigVersion(configName, i)
 		if err != nil {
 			return pkgerrors.Wrap(err, "Rollback Get Config Version")
 		}
+		_, _, prevAction, _, err := cvs.getConfigVersion(configName, i-1)
+		if err != nil {
+			return pkgerrors.Wrap(err, "Rollback Get Prev Config Version")
+		}
 		cs := ConfigStore{
 			instanceID: instanceID,
 			configName: configNew.ConfigName,
 		}
-		if action == "PUT" {
-			// PUT is proceeded by PUT or POST
-			err = applyConfig(instanceID, configPrev, profileChannel, "PUT")
+		if prevAction != "DELETE" {
+			appliedResources, err := applyConfig(instanceID, configPrev, profileChannel, prevAction, nil)
 			if err != nil {
 				return pkgerrors.Wrap(err, "Apply Config  failed")
 			}
+			log.Printf("%s result: %s", prevAction, appliedResources)
 			_, err = cs.updateConfig(configPrev)
 			if err != nil {
 				return pkgerrors.Wrap(err, "Update Config DB Entry")
 			}
-		} else if action == "POST" {
+		} else {
 			// POST is always preceeded by Config not existing
-			err = applyConfig(instanceID, configNew, profileChannel, "DELETE")
-			if err != nil {
-				return pkgerrors.Wrap(err, "Delete Config  failed")
-			}
-			_, err = cs.deleteConfig()
-			if err != nil {
-				return pkgerrors.Wrap(err, "Delete Config DB Entry")
-			}
-		} else if action == "DELETE" {
-			// DELETE is proceeded by PUT or POST
-			err = applyConfig(instanceID, configPrev, profileChannel, "PUT")
+			_, err := applyConfig(instanceID, configNew, profileChannel, prevAction, resources)
 			if err != nil {
 				return pkgerrors.Wrap(err, "Delete Config  failed")
 			}
+			log.Printf("DELETE resources: %s", resources)
 			_, err = cs.updateConfig(configPrev)
 			if err != nil {
 				return pkgerrors.Wrap(err, "Update Config DB Entry")
@@ -394,11 +447,7 @@ func (v *ConfigClient) Rollback(instanceID string, configName string, rback Conf
 
 // Tagit tags the current version with the tag provided
 func (v *ConfigClient) Tagit(instanceID string, configName string, tag ConfigTagit) error {
-
-	rbName, rbVersion, profileName, _, err := resolveModelFromInstance(instanceID)
-	if err != nil {
-		return pkgerrors.Wrap(err, "Retrieving model info")
-	}
+	log.Printf("[Config Tag It] Instance %s Config %s", instanceID, configName)
 	lock, _ := getProfileData(instanceID)
 	// Acquire per profile Mutex
 	lock.Lock()
@@ -408,38 +457,52 @@ func (v *ConfigClient) Tagit(instanceID string, configName string, tag ConfigTag
 		instanceID: instanceID,
 		configName: configName,
 	}
-	currentVersion, err := cvs.getCurrentVersion(configName)
-	if err != nil {
-		return pkgerrors.Wrap(err, "Get Current Config Version ")
-	}
-	tagKey := constructKey(rbName, rbVersion, profileName, instanceID, v.tagTag, configName, tag.TagName)
-
-	err = db.Etcd.Put(tagKey, strconv.Itoa(int(currentVersion)))
+	err := cvs.tagCurrentVersion(configName, tag.TagName)
 	if err != nil {
-		return pkgerrors.Wrap(err, "TagIt store DB")
+		return pkgerrors.Wrap(err, "Tag of current version failed")
 	}
 	return nil
 }
 
 // GetTagVersion returns the version associated with the tag
 func (v *ConfigClient) GetTagVersion(instanceID, configName string, tagName string) (string, error) {
-
-	rbName, rbVersion, profileName, _, err := resolveModelFromInstance(instanceID)
+	log.Printf("[Config Get Tag Version] Instance %s Config %s", instanceID, configName)
+	cvs := ConfigVersionStore{
+		instanceID: instanceID,
+		configName: configName,
+	}
+	value, err := cvs.getTagVersion(configName, tagName)
 	if err != nil {
-		return "", pkgerrors.Wrap(err, "Retrieving model info")
+		return "", pkgerrors.Wrap(err, "Tag of current version failed")
 	}
-	tagKey := constructKey(rbName, rbVersion, profileName, instanceID, v.tagTag, configName, tagName)
 
-	value, err := db.Etcd.Get(tagKey)
+	return value, nil
+}
+
+// Cleanup version used only when instance is being deleted. We do not pass errors and we try to delete data
+func (v *ConfigClient) Cleanup(instanceID string) error {
+	log.Printf("[Config Cleanup] Instance %s", instanceID)
+	configs, err := v.List(instanceID)
+
 	if err != nil {
-		return "", pkgerrors.Wrap(err, "Config DB Entry Not found")
+		return pkgerrors.Wrap(err, "Retrieving active config list info")
+	}
+
+	for _, config := range configs {
+		err = v.DeleteAll(instanceID, config.ConfigName)
+		if err != nil {
+			log.Printf("Config %s delete failed: %s", config.ConfigName, err.Error())
+		}
 	}
-	return string(value), nil
+
+	removeProfileData(instanceID)
+
+	return nil
 }
 
 // ApplyAllConfig starts from first configuration version and applies all versions in sequence
 func (v *ConfigClient) ApplyAllConfig(instanceID string, configName string) error {
-
+	log.Printf("[Config Apply All] Instance %s Config %s", instanceID, configName)
 	lock, profileChannel := getProfileData(instanceID)
 	// Acquire per profile Mutex
 	lock.Lock()
@@ -459,14 +522,19 @@ func (v *ConfigClient) ApplyAllConfig(instanceID string, configName string) erro
 	//Apply all configurations
 	var i uint
 	for i = 1; i <= currentVersion; i++ {
-		configNew, _, action, err := cvs.getConfigVersion(configName, i)
+		configNew, _, action, resources, err := cvs.getConfigVersion(configName, i)
 		if err != nil {
 			return pkgerrors.Wrap(err, "Get Config Version")
 		}
-		err = applyConfig(instanceID, configNew, profileChannel, action)
+		if action != "DELETE" {
+			resources = nil
+		}
+		var appliedResources ([]helm.KubernetesResource)
+		appliedResources, err = applyConfig(instanceID, configNew, profileChannel, action, resources)
 		if err != nil {
 			return pkgerrors.Wrap(err, "Apply Config  failed")
 		}
+		log.Printf("%s result: %s", action, appliedResources)
 	}
 	return nil
 }
diff --git a/src/k8splugin/internal/app/config_backend.go b/src/k8splugin/internal/app/config_backend.go
index 3e5d8a3f..4fedb386 100644
--- a/src/k8splugin/internal/app/config_backend.go
+++ b/src/k8splugin/internal/app/config_backend.go
@@ -38,9 +38,10 @@ import (
 
 //ConfigStore contains the values that will be stored in the database
 type configVersionDBContent struct {
-	ConfigNew  Config `json:"config-new"`
-	ConfigPrev Config `json:"config-prev"`
-	Action     string `json:"action"` // CRUD opration for this config
+	ConfigNew  Config                    `json:"config-new"`
+	ConfigPrev Config                    `json:"config-prev"`
+	Action     string                    `json:"action"` // CRUD opration for this config
+	Resources  []helm.KubernetesResource `json:"resources"`
 }
 
 //ConfigStore to Store the Config
@@ -57,7 +58,8 @@ type ConfigVersionStore struct {
 
 type configResourceList struct {
 	resourceTemplates []helm.KubernetesResourceTemplate
-	createdResources  []helm.KubernetesResource
+	resources         []helm.KubernetesResource
+	updatedResources  chan []helm.KubernetesResource
 	profile           rb.Profile
 	action            string
 }
@@ -72,6 +74,7 @@ const (
 	storeName  = "config"
 	tagCounter = "counter"
 	tagVersion = "configversion"
+	tagName    = "configtag"
 	tagConfig  = "configdata"
 )
 
@@ -223,8 +226,37 @@ func (c ConfigStore) deleteConfig() (Config, error) {
 	return configPrev, nil
 }
 
+//Cleanup stored data in etcd before instance is being deleted
+func (c ConfigVersionStore) cleanupIstanceTags(configName string) error {
+
+	rbName, rbVersion, profileName, _, err := resolveModelFromInstance(c.instanceID)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Retrieving model info")
+	}
+
+	versionKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagVersion, configName)
+	err = db.Etcd.DeletePrefix(versionKey)
+	if err != nil {
+		log.Printf("Deleting versions of instance failed: %s", err.Error())
+	}
+
+	counterKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagCounter, configName)
+	err = db.Etcd.DeletePrefix(counterKey)
+	if err != nil {
+		log.Printf("Deleting counters of instance failed: %s", err.Error())
+	}
+
+	nameKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagName, configName)
+	err = db.Etcd.DeletePrefix(nameKey)
+	if err != nil {
+		log.Printf("Deleting counters of instance failed: %s", err.Error())
+	}
+
+	return nil
+}
+
 // Create a version for the configuration. If previous config provided that is also stored
-func (c ConfigVersionStore) createConfigVersion(configNew, configPrev Config, action string) (uint, error) {
+func (c ConfigVersionStore) createConfigVersion(configNew, configPrev Config, action string, resources []helm.KubernetesResource) (uint, error) {
 
 	configName := ""
 	if configNew.ConfigName != "" {
@@ -249,6 +281,7 @@ func (c ConfigVersionStore) createConfigVersion(configNew, configPrev Config, ac
 	cs.Action = action
 	cs.ConfigNew = configNew
 	cs.ConfigPrev = configPrev
+	cs.Resources = resources //[]helm.KubernetesResource{}
 
 	configValue, err := db.Serialize(cs)
 	if err != nil {
@@ -288,27 +321,27 @@ func (c ConfigVersionStore) deleteConfigVersion(configName string) error {
 
 // Read the specified version of the configuration and return its prev and current value.
 // Also returns the action for the config version
-func (c ConfigVersionStore) getConfigVersion(configName string, version uint) (Config, Config, string, error) {
+func (c ConfigVersionStore) getConfigVersion(configName string, version uint) (Config, Config, string, []helm.KubernetesResource, error) {
 
 	rbName, rbVersion, profileName, _, err := resolveModelFromInstance(c.instanceID)
 	if err != nil {
-		return Config{}, Config{}, "", pkgerrors.Wrap(err, "Retrieving model info")
+		return Config{}, Config{}, "", []helm.KubernetesResource{}, pkgerrors.Wrap(err, "Retrieving model info")
 	}
 	versionKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagVersion, configName, strconv.Itoa(int(version)))
 	configBytes, err := db.Etcd.Get(versionKey)
 	if err != nil {
-		return Config{}, Config{}, "", pkgerrors.Wrap(err, "Get Config Version ")
+		return Config{}, Config{}, "", []helm.KubernetesResource{}, pkgerrors.Wrap(err, "Get Config Version ")
 	}
 
 	if configBytes != nil {
 		pr := configVersionDBContent{}
 		err = db.DeSerialize(string(configBytes), &pr)
 		if err != nil {
-			return Config{}, Config{}, "", pkgerrors.Wrap(err, "DeSerialize Config Version")
+			return Config{}, Config{}, "", []helm.KubernetesResource{}, pkgerrors.Wrap(err, "DeSerialize Config Version")
 		}
-		return pr.ConfigNew, pr.ConfigPrev, pr.Action, nil
+		return pr.ConfigNew, pr.ConfigPrev, pr.Action, pr.Resources, nil
 	}
-	return Config{}, Config{}, "", pkgerrors.Wrap(err, "Invalid data ")
+	return Config{}, Config{}, "", []helm.KubernetesResource{}, pkgerrors.Wrap(err, "Invalid data ")
 }
 
 // Get the counter for the version
@@ -318,7 +351,7 @@ func (c ConfigVersionStore) getCurrentVersion(configName string) (uint, error) {
 	if err != nil {
 		return 0, pkgerrors.Wrap(err, "Retrieving model info")
 	}
-	cfgKey := constructKey(rbName, rbVersion, profileName, c.instanceID, configName, tagCounter)
+	cfgKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagCounter, configName)
 
 	value, err := db.Etcd.Get(cfgKey)
 	if err != nil {
@@ -344,7 +377,7 @@ func (c ConfigVersionStore) updateVersion(configName string, counter uint) error
 	if err != nil {
 		return pkgerrors.Wrap(err, "Retrieving model info")
 	}
-	cfgKey := constructKey(rbName, rbVersion, profileName, c.instanceID, configName, tagCounter)
+	cfgKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagCounter, configName)
 	err = db.Etcd.Put(cfgKey, strconv.Itoa(int(counter)))
 	if err != nil {
 		return pkgerrors.Wrap(err, "Counter DB Entry")
@@ -386,45 +419,87 @@ func (c ConfigVersionStore) decrementVersion(configName string) error {
 	return nil
 }
 
+// Get tag version
+func (c ConfigVersionStore) getTagVersion(configName, tagNameValue string) (string, error) {
+	rbName, rbVersion, profileName, _, err := resolveModelFromInstance(c.instanceID)
+	if err != nil {
+		return "", pkgerrors.Wrap(err, "Retrieving model info")
+	}
+	tagKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagName, configName, tagNameValue)
+
+	value, err := db.Etcd.Get(tagKey)
+	if err != nil {
+		return "", pkgerrors.Wrap(err, "Config DB Entry Not found")
+	}
+	return string(value), nil
+}
+
+// Tag current version
+func (c ConfigVersionStore) tagCurrentVersion(configName, tagNameValue string) error {
+	currentVersion, err := c.getCurrentVersion(configName)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Get Current Config Version ")
+	}
+	rbName, rbVersion, profileName, _, err := resolveModelFromInstance(c.instanceID)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Retrieving model info")
+	}
+	tagKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagName, configName, tagNameValue)
+
+	err = db.Etcd.Put(tagKey, strconv.Itoa(int(currentVersion)))
+	if err != nil {
+		return pkgerrors.Wrap(err, "TagIt store DB")
+	}
+	return nil
+}
+
 // Apply Config
-func applyConfig(instanceID string, p Config, pChannel chan configResourceList, action string) error {
+func applyConfig(instanceID string, p Config, pChannel chan configResourceList, action string, resources []helm.KubernetesResource) ([]helm.KubernetesResource, error) {
 
 	rbName, rbVersion, profileName, releaseName, err := resolveModelFromInstance(instanceID)
 	if err != nil {
-		return pkgerrors.Wrap(err, "Retrieving model info")
+		return []helm.KubernetesResource{}, pkgerrors.Wrap(err, "Retrieving model info")
 	}
 	// Get Template and Resolve the template with values
 	crl, err := resolve(rbName, rbVersion, profileName, p, releaseName)
 	if err != nil {
-		return pkgerrors.Wrap(err, "Resolve Config")
+		return []helm.KubernetesResource{}, pkgerrors.Wrap(err, "Resolve Config")
 	}
+	var updatedResources (chan []helm.KubernetesResource) = make(chan []helm.KubernetesResource)
 	crl.action = action
+	crl.resources = resources
+	crl.updatedResources = updatedResources
 	// Send the configResourceList to the channel. Using select for non-blocking channel
+	log.Printf("Before Sent to goroutine %v", crl.profile)
 	select {
 	case pChannel <- crl:
 		log.Printf("Message Sent to goroutine %v", crl.profile)
 	default:
 	}
 
-	return nil
+	var resultResources []helm.KubernetesResource = <-updatedResources
+	return resultResources, nil
 }
 
 // Per Profile Go routine to apply the configuration to Cloud Region
 func scheduleResources(c chan configResourceList) {
 	// Keep thread running
+	log.Printf("[scheduleResources]: START thread")
 	for {
 		data := <-c
 		//TODO: ADD Check to see if Application running
 		ic := NewInstanceClient()
 		resp, err := ic.Find(data.profile.RBName, data.profile.RBVersion, data.profile.ProfileName, nil)
-		if err != nil || len(resp) == 0 {
+		if (err != nil || len(resp) == 0) && data.action != "STOP" {
 			log.Println("Error finding a running instance. Retrying later...")
-			time.Sleep(time.Second * 10)
+			data.updatedResources <- []helm.KubernetesResource{}
 			continue
 		}
+		breakThread := false
 		switch {
 		case data.action == "POST":
 			log.Printf("[scheduleResources]: POST %v %v", data.profile, data.resourceTemplates)
+			var resources []helm.KubernetesResource
 			for _, inst := range resp {
 				k8sClient := KubernetesClient{}
 				err = k8sClient.Init(inst.Request.CloudRegion, inst.ID)
@@ -434,11 +509,11 @@ func scheduleResources(c chan configResourceList) {
 					continue
 				}
 				//assuming - the resource is not exist already
-				data.createdResources, err = k8sClient.createResources(data.resourceTemplates, inst.Namespace)
+				resources, err = k8sClient.createResources(data.resourceTemplates, inst.Namespace)
 				errCreate := err
 				if err != nil {
 					// assuming - the err represent the resource is already exist, so going for update
-					data.createdResources, err = k8sClient.updateResources(data.resourceTemplates, inst.Namespace)
+					resources, err = k8sClient.updateResources(data.resourceTemplates, inst.Namespace)
 					if err != nil {
 						log.Printf("Error Creating resources: %s", errCreate.Error())
 						log.Printf("Error Updating resources: %s", err.Error())
@@ -446,8 +521,10 @@ func scheduleResources(c chan configResourceList) {
 					}
 				}
 			}
+			data.updatedResources <- resources
 		case data.action == "PUT":
 			log.Printf("[scheduleResources]: PUT %v %v", data.profile, data.resourceTemplates)
+			var resources []helm.KubernetesResource
 			for _, inst := range resp {
 				k8sClient := KubernetesClient{}
 				err = k8sClient.Init(inst.Request.CloudRegion, inst.ID)
@@ -456,14 +533,16 @@ func scheduleResources(c chan configResourceList) {
 					//Move onto the next cloud region
 					continue
 				}
-				data.createdResources, err = k8sClient.updateResources(data.resourceTemplates, inst.Namespace)
+
+				resources, err = k8sClient.updateResources(data.resourceTemplates, inst.Namespace)
 				if err != nil {
 					log.Printf("Error Updating resources: %s", err.Error())
 					continue
 				}
 			}
+			data.updatedResources <- resources
 		case data.action == "DELETE":
-			log.Printf("[scheduleResources]: DELETE %v %v", data.profile, data.resourceTemplates)
+			log.Printf("[scheduleResources]: DELETE %v %v", data.profile, data.resources)
 			for _, inst := range resp {
 				k8sClient := KubernetesClient{}
 				err = k8sClient.Init(inst.Request.CloudRegion, inst.ID)
@@ -472,14 +551,22 @@ func scheduleResources(c chan configResourceList) {
 					//Move onto the next cloud region
 					continue
 				}
-				err = k8sClient.deleteResources(data.createdResources, inst.Namespace)
+				err = k8sClient.deleteResources(data.resources, inst.Namespace)
 				if err != nil {
 					log.Printf("Error Deleting resources: %s", err.Error())
 					continue
 				}
 			}
+			data.updatedResources <- []helm.KubernetesResource{}
+
+		case data.action == "STOP":
+			breakThread = true
+		}
+		if breakThread {
+			break
 		}
 	}
+	log.Printf("[scheduleResources]: STOP thread")
 }
 
 //Resolve returns the path where the helm chart merged with
@@ -570,6 +657,25 @@ func getProfileData(key string) (*sync.Mutex, chan configResourceList) {
 	if !ok {
 		profileData.resourceChannel[key] = make(chan configResourceList)
 		go scheduleResources(profileData.resourceChannel[key])
+		time.Sleep(time.Second * 5)
 	}
 	return profileData.profileLockMap[key], profileData.resourceChannel[key]
 }
+
+func removeProfileData(key string) {
+	profileData.Lock()
+	defer profileData.Unlock()
+	_, ok := profileData.profileLockMap[key]
+	if ok {
+		delete(profileData.profileLockMap, key)
+	}
+	_, ok = profileData.resourceChannel[key]
+	if ok {
+		log.Printf("Stop config thread for %s", key)
+		crl := configResourceList{
+			action: "STOP",
+		}
+		profileData.resourceChannel[key] <- crl
+		delete(profileData.resourceChannel, key)
+	}
+}
diff --git a/src/k8splugin/internal/app/config_test.go b/src/k8splugin/internal/app/config_test.go
index dc4779d8..0cc3c3ce 100644
--- a/src/k8splugin/internal/app/config_test.go
+++ b/src/k8splugin/internal/app/config_test.go
@@ -319,3 +319,6 @@ func TestRollbackConfig(t *testing.T) {
 		})
 	}
 }
+
+func main() {
+}
diff --git a/src/k8splugin/internal/app/instance.go b/src/k8splugin/internal/app/instance.go
index 0f1f3d7e..b7f382ad 100644
--- a/src/k8splugin/internal/app/instance.go
+++ b/src/k8splugin/internal/app/instance.go
@@ -709,7 +709,8 @@ func (v *InstanceClient) Delete(id string) error {
 		return nil
 	} else if inst.Status != "DONE" {
 		//Recover is ongoing, do nothing here
-		return nil
+		//return nil
+		//TODO: implement recovery
 	}
 
 	k8sClient := KubernetesClient{}
@@ -718,12 +719,6 @@ func (v *InstanceClient) Delete(id string) error {
 		return pkgerrors.Wrap(err, "Getting CloudRegion Information")
 	}
 
-	configClient := NewConfigClient()
-	configs, err := configClient.List(id)
-	if err != nil {
-		return pkgerrors.Wrap(err, "Getting Configs Information")
-	}
-
 	inst.Status = "PRE-DELETE"
 	inst.HookProgress = ""
 	err = db.DBconn.Update(v.storeName, key, v.tagInst, inst)
@@ -751,15 +746,10 @@ func (v *InstanceClient) Delete(id string) error {
 		log.Printf("Update Instance DB Entry for release %s has error.", inst.ReleaseName)
 	}
 
-	if len(configs) > 0 {
-		log.Printf("Deleting config resources first")
-		for _, config := range configs {
-			log.Printf("Deleting Config %s Resources", config.ConfigName)
-			_, err = configClient.Delete(id, config.ConfigName)
-			if err != nil {
-				return pkgerrors.Wrap(err, "Deleting Config Resources")
-			}
-		}
+	configClient := NewConfigClient()
+	err = configClient.Cleanup(id)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Cleanup Config Resources")
 	}
 
 	err = k8sClient.deleteResources(inst.Resources, inst.Namespace)
diff --git a/src/k8splugin/internal/db/etcd.go b/src/k8splugin/internal/db/etcd.go
index a435b435..e455cc1a 100644
--- a/src/k8splugin/internal/db/etcd.go
+++ b/src/k8splugin/internal/db/etcd.go
@@ -39,6 +39,7 @@ type EtcdStore interface {
 	GetAll(key string) ([][]byte, error)
 	Put(key, value string) error
 	Delete(key string) error
+	DeletePrefix(keyPrefix string) error
 }
 
 // EtcdClient for Etcd
@@ -151,3 +152,16 @@ func (e EtcdClient) Delete(key string) error {
 	}
 	return nil
 }
+
+// Delete values by prefix from Etcd DB
+func (e EtcdClient) DeletePrefix(keyPrefix string) error {
+
+	if e.cli == nil {
+		return pkgerrors.Errorf("Etcd Client not initialized")
+	}
+	_, err := e.cli.Delete(context.Background(), keyPrefix, clientv3.WithPrefix())
+	if err != nil {
+		return pkgerrors.Errorf("Delete prefix failed etcd entry:%s", err.Error())
+	}
+	return nil
+}
diff --git a/src/k8splugin/internal/db/etcd_testing.go b/src/k8splugin/internal/db/etcd_testing.go
index 9dfcad82..4b4dfe3e 100644
--- a/src/k8splugin/internal/db/etcd_testing.go
+++ b/src/k8splugin/internal/db/etcd_testing.go
@@ -55,3 +55,12 @@ func (c *MockEtcdClient) Delete(key string) error {
 	delete(c.Items, key)
 	return c.Err
 }
+
+func (c *MockEtcdClient) DeletePrefix(key string) error {
+	for kvKey := range c.Items {
+		if strings.HasPrefix(kvKey, key) {
+			delete(c.Items, key)
+		}
+	}
+	return c.Err
+}