4 files changed, 81 insertions, 155 deletions
diff --git a/src/dcm/pkg/module/apply.go b/src/dcm/pkg/module/apply.go
index 3770457a..0eaa75ab 100644
--- a/src/dcm/pkg/module/apply.go
+++ b/src/dcm/pkg/module/apply.go
@@ -84,6 +84,19 @@ type RoleRef struct {
 	ApiGroup string `yaml:"apiGroup"`
 }
 
+func cleanupCompositeApp(context appcontext.AppContext, err error, reason string, details []string) error {
+	cleanuperr := context.DeleteCompositeApp()
+	newerr := pkgerrors.Wrap(err, reason)
+	if cleanuperr != nil {
+		log.Warn("Error cleaning AppContext, ", log.Fields{
+			"Related details": details,
+		})
+		// this would be useful: https://godoc.org/go.uber.org/multierr
+		return pkgerrors.Wrap(err, "After previous error, cleaning the AppContext also failed.")
+	}
+	return newerr
+}
+
 func createNamespace(logicalcloud LogicalCloud) (string, string, error) {
 
 	name := logicalcloud.Specification.NameSpace
@@ -329,11 +342,11 @@ func Apply(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 	}
 
 	// Check if there was a previous context for this logical cloud
-	ac, cid, err := lcclient.GetLogicalCloudContext(project, logicalCloudName)
+	ac, cid, err := lcclient.util.GetLogicalCloudContext(lcclient.storeName, lckey, lcclient.tagMeta, project, logicalCloudName)
 	if cid != "" {
 		// Make sure rsync status for this logical cloud is Terminated,
 		// otherwise we can't re-apply logical cloud yet
-		acStatus, _ := getAppContextStatus(ac)
+		acStatus, _ := lcclient.util.GetAppContextStatus(ac)
 		switch acStatus.Status {
 		case appcontext.AppContextStatusEnum.Terminated:
 			// We now know Logical Cloud has terminated, so let's update the entry before we process the apply
@@ -397,126 +410,60 @@ func Apply(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 
 	appHandle, err := context.AddApp(handle, APP)
 	if err != nil {
-		cleanuperr := context.DeleteCompositeApp()
-		if cleanuperr != nil {
-			log.Warn("Error cleaning AppContext CompositeApp create failure", log.Fields{
-				"logical-cloud": logicalCloudName,
-			})
-		}
-		return pkgerrors.Wrap(err, "Error adding App to AppContext")
+		return cleanupCompositeApp(context, err, "Error adding App to AppContext", []string{logicalCloudName, ctxVal.(string)})
 	}
 
 	// Iterate through cluster list and add all the clusters
 	for _, cluster := range clusterList {
 		clusterName := strings.Join([]string{cluster.Specification.ClusterProvider, "+", cluster.Specification.ClusterName}, "")
 		clusterHandle, err := context.AddCluster(appHandle, clusterName)
+		// pre-build array to pass to cleanupCompositeApp() [for performance]
+		details := []string{logicalCloudName, clusterName, ctxVal.(string)}
 
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add cluster failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding Cluster to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding Cluster to AppContext", details)
 		}
 
 		// Add namespace resource to each cluster
 		_, err = context.AddResource(clusterHandle, namespaceName, namespace)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add namespace resource failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding Namespace Resource to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding Namespace Resource to AppContext", details)
 		}
 
 		// Add csr resource to each cluster
 		csrHandle, err := context.AddResource(clusterHandle, csrName, csr)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add CSR resource failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding CSR Resource to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding CSR Resource to AppContext", details)
 		}
 
 		// Add csr approval as a subresource of csr:
 		_, err = context.AddLevelValue(csrHandle, "subresource/approval", approval)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add CSR approval failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error approving CSR via AppContext")
+			return cleanupCompositeApp(context, err, "Error approving CSR via AppContext", details)
 		}
 
 		// Add private key to MongoDB
 		err = db.DBconn.Insert("orchestrator", lckey, nil, "privatekey", key)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after DB insert failure", log.Fields{
-					"logical-cloud": logicalcloud.MetaData.LogicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding private key to DB")
+			return cleanupCompositeApp(context, err, "Error adding private key to DB", details)
 		}
 
 		// Add Role resource to each cluster
 		_, err = context.AddResource(clusterHandle, roleName, role)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add role resource failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding role Resource to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding role Resource to AppContext", details)
 		}
 
 		// Add RoleBinding resource to each cluster
 		_, err = context.AddResource(clusterHandle, roleBindingName, roleBinding)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add roleBinding resource failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding roleBinding Resource to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding roleBinding Resource to AppContext", details)
 		}
 
 		// Add quota resource to each cluster
 		_, err = context.AddResource(clusterHandle, quotaName, quota)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add quota resource failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding quota Resource to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding quota Resource to AppContext", details)
 		}
 
 		// Add Subresource Order and Subresource Dependency
@@ -541,77 +488,42 @@ func Apply(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 			return pkgerrors.Wrap(err, "Error creating resource order JSON")
 		}
 		appDependency, err := json.Marshal(map[string]map[string]string{"appdependency": map[string]string{APP: "go"}})
-
 		if err != nil {
 			return pkgerrors.Wrap(err, "Error creating resource dependency JSON")
 		}
 
+		// Add Resource-level Order and Dependency
 		_, err = context.AddInstruction(clusterHandle, "resource", "order", string(resOrder))
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add instruction  failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding instruction order to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding instruction order to AppContext", details)
 		}
-
 		_, err = context.AddInstruction(clusterHandle, "resource", "dependency", string(resDependency))
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add instruction  failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding instruction dependency to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding instruction dependency to AppContext", details)
 		}
-
 		_, err = context.AddInstruction(csrHandle, "subresource", "order", string(subresOrder))
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add instruction  failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding instruction order to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding instruction order to AppContext", details)
 		}
-
 		_, err = context.AddInstruction(csrHandle, "subresource", "dependency", string(subresDependency))
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Warn("Error cleaning AppContext after add instruction  failure", log.Fields{
-					"cluster-provider": cluster.Specification.ClusterProvider,
-					"cluster":          cluster.Specification.ClusterName,
-					"logical-cloud":    logicalCloudName,
-				})
-			}
-			return pkgerrors.Wrap(err, "Error adding instruction dependency to AppContext")
+			return cleanupCompositeApp(context, err, "Error adding instruction dependency to AppContext", details)
 		}
 
 		// Add App-level Order and Dependency
 		_, err = context.AddInstruction(handle, "app", "order", string(appOrder))
+		if err != nil {
+			return cleanupCompositeApp(context, err, "Error adding app-level order to AppContext", details)
+		}
 		_, err = context.AddInstruction(handle, "app", "dependency", string(appDependency))
+		if err != nil {
+			return cleanupCompositeApp(context, err, "Error adding app-level dependency to AppContext", details)
+		}
 	}
 	// save the context in the logicalcloud db record
 	err = db.DBconn.Insert("orchestrator", lckey, nil, "lccontext", ctxVal)
 	if err != nil {
-		cleanuperr := context.DeleteCompositeApp()
-		if cleanuperr != nil {
-			log.Warn("Error cleaning AppContext after DB insert failure", log.Fields{
-				"logical-cloud": logicalcloud.MetaData.LogicalCloudName,
-			})
-		}
-		return pkgerrors.Wrap(err, "Error adding AppContext to DB")
+		return cleanupCompositeApp(context, err, "Error adding AppContext to DB", []string{logicalCloudName, ctxVal.(string)})
 	}
 
 	// call resource synchronizer to instantiate the CRs in the cluster
@@ -632,8 +544,12 @@ func Terminate(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 	logicalCloudName := logicalcloud.MetaData.LogicalCloudName
 
 	lcclient := NewLogicalCloudClient()
+	lckey := LogicalCloudKey{
+		LogicalCloudName: logicalcloud.MetaData.LogicalCloudName,
+		Project:          project,
+	}
 
-	ac, cid, err := lcclient.GetLogicalCloudContext(project, logicalCloudName)
+	ac, cid, err := lcclient.util.GetLogicalCloudContext(lcclient.storeName, lckey, lcclient.tagMeta, project, logicalCloudName)
 	if err != nil {
 		return pkgerrors.Wrapf(err, "Logical Cloud doesn't seem applied: %v", logicalCloudName)
 	}
@@ -642,7 +558,7 @@ func Terminate(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 	if cid != "" {
 		// Make sure rsync status for this logical cloud is Terminated,
 		// otherwise we can't re-apply logical cloud yet
-		acStatus, _ := getAppContextStatus(ac)
+		acStatus, _ := lcclient.util.GetAppContextStatus(ac)
 		switch acStatus.Status {
 		case appcontext.AppContextStatusEnum.Terminated:
 			return pkgerrors.New("The Logical Cloud has already been terminated: " + logicalCloudName)
diff --git a/src/dcm/pkg/module/cluster.go b/src/dcm/pkg/module/cluster.go
index 33de7acf..6ad46404 100644
--- a/src/dcm/pkg/module/cluster.go
+++ b/src/dcm/pkg/module/cluster.go
@@ -59,7 +59,7 @@ type KubeConfig struct {
 	Kind           string            `yaml:"kind"`
 	Clusters       []KubeCluster     `yaml:"clusters"`
 	Contexts       []KubeContext     `yaml:"contexts"`
-	CurrentContext string            `yaml:"current-context`
+	CurrentContext string            `yaml:"current-context"`
 	Preferences    map[string]string `yaml:"preferences"`
 	Users          []KubeUser        `yaml:"users"`
 }
@@ -260,7 +260,11 @@ func (v *ClusterClient) UpdateCluster(project, logicalCloud, clusterReference st
 // Get returns Cluster's kubeconfig for corresponding cluster reference
 func (v *ClusterClient) GetClusterConfig(project, logicalCloud, clusterReference string) (string, error) {
 	lcClient := NewLogicalCloudClient()
-	context, ctxVal, err := lcClient.GetLogicalCloudContext(project, logicalCloud)
+	lckey := LogicalCloudKey{
+		Project:          project,
+		LogicalCloudName: logicalCloud,
+	}
+	context, ctxVal, err := lcClient.util.GetLogicalCloudContext(lcClient.storeName, lckey, lcClient.tagMeta, project, logicalCloud)
 	if err != nil {
 		return "", pkgerrors.Wrap(err, "Error getting logical cloud context.")
 	}
@@ -268,11 +272,6 @@ func (v *ClusterClient) GetClusterConfig(project, logicalCloud, clusterReference
 		return "", pkgerrors.New("Logical Cloud hasn't been applied yet")
 	}
 
-	// private key comes from logical cloud
-	lckey := LogicalCloudKey{
-		Project:          project,
-		LogicalCloudName: logicalCloud,
-	}
 	// get logical cloud resource
 	lc, err := lcClient.Get(project, logicalCloud)
 	if err != nil {
diff --git a/src/dcm/pkg/module/logicalcloud.go b/src/dcm/pkg/module/logicalcloud.go
index 580e9022..3fe981b8 100644
--- a/src/dcm/pkg/module/logicalcloud.go
+++ b/src/dcm/pkg/module/logicalcloud.go
@@ -75,7 +75,6 @@ type LogicalCloudManager interface {
 	GetAll(project string) ([]LogicalCloud, error)
 	Delete(project, name string) error
 	Update(project, name string, c LogicalCloud) (LogicalCloud, error)
-	GetLogicalCloudContext(project string, name string) (appcontext.AppContext, string, error)
 }
 
 // Interface facilitates unit testing by mocking functions
@@ -86,6 +85,8 @@ type Utility interface {
 	DBRemove(storeName string, key db.Key) error
 	CheckProject(project string) error
 	CheckLogicalCloud(project, logicalCloud string) error
+	GetLogicalCloudContext(storeName string, key db.Key, meta string, project string, name string) (appcontext.AppContext, string, error)
+	GetAppContextStatus(ac appcontext.AppContext) (*appcontext.AppContextStatus, error)
 }
 
 // LogicalCloudClient implements the LogicalCloudManager
@@ -208,7 +209,7 @@ func (v *LogicalCloudClient) Delete(project, logicalCloudName string) error {
 		return pkgerrors.New("Logical Cloud does not exist")
 	}
 
-	context, _, err := v.GetLogicalCloudContext(project, logicalCloudName)
+	context, _, err := v.util.GetLogicalCloudContext(v.storeName, key, v.tagMeta, project, logicalCloudName)
 	// If there's no context for Logical Cloud, just go ahead and delete it now
 	if err != nil {
 		err = v.util.DBRemove(v.storeName, key)
@@ -220,7 +221,7 @@ func (v *LogicalCloudClient) Delete(project, logicalCloudName string) error {
 
 	// Make sure rsync status for this logical cloud is Terminated,
 	// otherwise we can't remove appcontext yet
-	acStatus, _ := getAppContextStatus(context)
+	acStatus, _ := v.util.GetAppContextStatus(context)
 	switch acStatus.Status {
 	case appcontext.AppContextStatusEnum.Terminated:
 		// remove the appcontext
@@ -267,14 +268,9 @@ func (v *LogicalCloudClient) Update(project, logicalCloudName string, c LogicalC
 }
 
 // GetLogicalCloudContext returns the AppContext for corresponding provider and name
-func (v *LogicalCloudClient) GetLogicalCloudContext(project string, name string) (appcontext.AppContext, string, error) {
-	//Construct key and tag to select the entry
-	key := LogicalCloudKey{
-		LogicalCloudName: name,
-		Project:          project,
-	}
+func (d DBService) GetLogicalCloudContext(storeName string, key db.Key, meta string, project string, name string) (appcontext.AppContext, string, error) {
 
-	value, err := v.util.DBFind(v.storeName, key, v.tagContext)
+	value, err := d.DBFind(storeName, key, meta)
 	if err != nil {
 		return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Get Logical Cloud Context")
 	}
@@ -353,7 +349,7 @@ func (d DBService) CheckLogicalCloud(project, logicalCloud string) error {
 	return nil
 }
 
-func getAppContextStatus(ac appcontext.AppContext) (*appcontext.AppContextStatus, error) {
+func (d DBService) GetAppContextStatus(ac appcontext.AppContext) (*appcontext.AppContextStatus, error) {
 
 	h, err := ac.GetCompositeAppHandle()
 	if err != nil {
diff --git a/src/dcm/pkg/module/logicalcloud_test.go b/src/dcm/pkg/module/logicalcloud_test.go
index efce568f..cbdc1304 100644
--- a/src/dcm/pkg/module/logicalcloud_test.go
+++ b/src/dcm/pkg/module/logicalcloud_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/mock"
@@ -55,6 +56,20 @@ func (m *mockValues) CheckLogicalCloud(project, logicalCloud string) error {
 	return args.Error(0)
 }
 
+func (m *mockValues) GetLogicalCloudContext(name string, key db.Key, meta string, project, logicalCloud string) (appcontext.AppContext, string, error) {
+	fmt.Println("Mocked Get Logical Cloud Context")
+	args := m.Called(name, key, meta, project, logicalCloud)
+
+	return appcontext.AppContext{}, "", args.Error(2)
+}
+
+func (m *mockValues) GetAppContextStatus(ac appcontext.AppContext) (*appcontext.AppContextStatus, error) {
+	fmt.Println("Mocked GetAppContextStatus")
+	args := m.Called(ac)
+
+	return &appcontext.AppContextStatus{}, args.Error(1)
+}
+
 func TestCreateLogicalCloud(t *testing.T) {
 
 	mData := MetaDataList{
@@ -110,7 +125,7 @@ func TestGetLogicalCloud(t *testing.T) {
 	}
 }
 
-func TestDeleteLogicalCloud(t *testing.T) {
+func TestDeleteLogicalCloudWithSuccess(t *testing.T) {
 
 	key := LogicalCloudKey{
 		Project:          "test_project",
@@ -128,14 +143,14 @@ func TestDeleteLogicalCloud(t *testing.T) {
 	myMocks.On("DBFind", "test_dcm", key, "test_meta").Return(data1, nil)
 	myMocks.On("DBUnmarshal", data2).Return(nil)
 	myMocks.On("DBFind", "test_dcm", key, "test_context").Return(data1, nil)
-	// TODO also test for when the logical cloud doesn't exist
-
-	// TODO: fix Etcd-related test crash
-	// lcClient := LogicalCloudClient{"test_dcm", "test_meta", "test_context", myMocks}
-	// err := lcClient.Delete("test_project", "test_asdf")
-	// if err != nil {
-	// 	t.Errorf("Some error occured!")
-	// }
+	myMocks.On("GetLogicalCloudContext", "test_dcm", key, "test_meta", "test_project", "test_asdf").Return(appcontext.AppContext{}, "", nil)
+	myMocks.On("GetAppContextStatus", appcontext.AppContext{}).Return(&appcontext.AppContextStatus{}, nil)
+
+	lcClient := LogicalCloudClient{"test_dcm", "test_meta", "test_context", myMocks}
+	err := lcClient.Delete("test_project", "test_asdf")
+	if err.Error() != "The Logical Cloud can't be deleted yet at this point." {
+		t.Errorf("Some unexpected error occurred!")
+	}
 }
 
 func TestUpdateLogicalCloud(t *testing.T) {