Reimplement Terminate to be compatible with Status

This also includes modifying Apply and Delete, since there are
strict conditions that need to be met in each, to prevent
Logical Clouds (LCs) from entering a bad state.

Summary of what's being done here:
- When applying:
  - set tag 'lccontext' in the LC to the context ID (was already done)
  - and let rsync know about the appcontext (grpc) (was already done)
  - if tag was already set, check current context /status
  - if context /status is actually Terminated,  'lccontext' is set to
    new context ID and previous AppContext deleted
- When terminating:
  - lets rsync know about the termination request (grpc)
- When deleting:
  - checks whether the current context /status is Terminated
  - if it is, then it will remove the latest LC context

This particular commit disables the TestDeleteLogicalCloud test
until a known issue behind the test is resolved.

This commit does not leverage the full capacity of the Status
framework, but is sufficient to support all operations. A future
patch will entirely migrate DCM to the Status framework. Until
then, a known issue exists where DCM will forget about context IDs
previously associated to a particular Logical Cloud (only keeps last).

Issue-ID: MULTICLOUD-1143
Change-Id: I7a6034eba543c2a27daa41b7fe6298cb2a85f9ce
Signed-off-by: Igor D.C <igor.duarte.cardoso@intel.com>

1 files changed, 57 insertions, 43 deletions

diff --git a/src/dcm/pkg/module/apply.go b/src/dcm/pkg/module/apply.go
index b5ef61c2..d911ba37 100644
--- a/src/dcm/pkg/module/apply.go
+++ b/src/dcm/pkg/module/apply.go
@@ -313,6 +313,39 @@ func Apply(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 	APP := "logical-cloud"
 	logicalCloudName := logicalcloud.MetaData.LogicalCloudName
 
+	lcclient := NewLogicalCloudClient()
+	lckey := LogicalCloudKey{
+		LogicalCloudName: logicalcloud.MetaData.LogicalCloudName,
+		Project:          project,
+	}
+
+	// Check if there was a previous context for this logical cloud
+	ac, cid, err := lcclient.GetLogicalCloudContext(project, logicalCloudName)
+	if cid != "" {
+		// Make sure rsync status for this logical cloud is Terminated,
+		// otherwise we can't re-apply logical cloud yet
+		acStatus, _ := getAppContextStatus(ac)
+		switch acStatus.Status {
+		case appcontext.AppContextStatusEnum.Terminated:
+			// We now know Logical Cloud has terminated, so let's update the entry before we process the apply
+			err = db.DBconn.RemoveTag(lcclient.storeName, lckey, lcclient.tagContext)
+			if err != nil {
+				return pkgerrors.Wrap(err, "Error removing lccontext tag from Logical Cloud")
+			}
+			// And fully delete the old AppContext
+			err := ac.DeleteCompositeApp()
+			if err != nil {
+				return pkgerrors.Wrap(err, "Error deleting AppContext CompositeApp Logical Cloud")
+			}
+		case appcontext.AppContextStatusEnum.Terminating:
+			return pkgerrors.New("The Logical Cloud can't be re-applied yet, it is being terminated.")
+		case appcontext.AppContextStatusEnum.Instantiated:
+			return pkgerrors.New("The Logical Cloud is already applied.")
+		default:
+			return pkgerrors.New("The Logical Cloud can't be applied at this point.")
+		}
+	}
+
 	//Resource Names
 	namespaceName := strings.Join([]string{logicalcloud.MetaData.LogicalCloudName, "+namespace"}, "")
 	roleName := strings.Join([]string{logicalcloud.MetaData.LogicalCloudName, "+role"}, "")
@@ -348,14 +381,13 @@ func Apply(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 
 	approval, err := createApprovalSubresource(logicalcloud)
 
+	// From this point on, we are dealing with a new context (not "ac" from above)
 	context := appcontext.AppContext{}
 	ctxVal, err := context.InitAppContext()
 	if err != nil {
 		return pkgerrors.Wrap(err, "Error creating AppContext")
 	}
 
-	fmt.Printf("%v\n", ctxVal)
-
 	handle, err := context.CreateCompositeApp()
 	if err != nil {
 		return pkgerrors.Wrap(err, "Error creating AppContext CompositeApp")
@@ -432,10 +464,6 @@ func Apply(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 		}
 
 		// Add private key to MongoDB
-		lckey := LogicalCloudKey{
-			LogicalCloudName: logicalcloud.MetaData.LogicalCloudName,
-			Project:          project,
-		}
 		err = db.DBconn.Insert("orchestrator", lckey, nil, "privatekey", key)
 		if err != nil {
 			cleanuperr := context.DeleteCompositeApp()
@@ -573,10 +601,6 @@ func Apply(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 		_, err = context.AddInstruction(handle, "app", "dependency", string(appDependency))
 	}
 	// save the context in the logicalcloud db record
-	lckey := LogicalCloudKey{
-		LogicalCloudName: logicalcloud.MetaData.LogicalCloudName,
-		Project:          project,
-	}
 	err = db.DBconn.Insert("orchestrator", lckey, nil, "lccontext", ctxVal)
 	if err != nil {
 		cleanuperr := context.DeleteCompositeApp()
@@ -605,40 +629,30 @@ func Terminate(project string, logicalcloud LogicalCloud, clusterList []Cluster,
 
 	logicalCloudName := logicalcloud.MetaData.LogicalCloudName
 
-	_, ctxVal, err := NewLogicalCloudClient().GetLogicalCloudContext(project, logicalCloudName)
-	if err != nil {
-		return pkgerrors.Wrapf(err, "Error finding AppContext for Logical Cloud: %v", logicalCloudName)
-	}
+	lcclient := NewLogicalCloudClient()
 
-	// call resource synchronizer to delete the CRs from every cluster of the logical cloud
-	err = callRsyncUninstall(ctxVal)
+	ac, cid, err := lcclient.GetLogicalCloudContext(project, logicalCloudName)
 	if err != nil {
-		return err
+		return pkgerrors.Wrapf(err, "Logical Cloud doesn't seem applied: %v", logicalCloudName)
+	}
+
+	// Check if there was a previous context for this logical cloud
+	if cid != "" {
+		// Make sure rsync status for this logical cloud is Terminated,
+		// otherwise we can't re-apply logical cloud yet
+		acStatus, _ := getAppContextStatus(ac)
+		switch acStatus.Status {
+		case appcontext.AppContextStatusEnum.Terminated:
+			return pkgerrors.New("The Logical Cloud has already been terminated: " + logicalCloudName)
+		case appcontext.AppContextStatusEnum.Terminating:
+			return pkgerrors.New("The Logical Cloud is already being terminated: " + logicalCloudName)
+		case appcontext.AppContextStatusEnum.Instantiated:
+			// call resource synchronizer to delete the CRs from every cluster of the logical cloud
+			err = callRsyncUninstall(cid)
+			return err
+		default:
+			return pkgerrors.New("The Logical Cloud can't be deleted at this point: " + logicalCloudName)
+		}
 	}
-
-	// TODO: status handling for logical cloud after terminate:
-	// rsync updates the status of the appcontext to Terminated
-	// dcm should launch thread to observe status of appcontext before concluding logical cloud is terminated
-	// dcm should somewhat mimic the status tracking of rsync
-	// logical cloud might be in a non-applied non-terminated state for a long period of time.........
-
-	// // remove the app context
-	// err = context.DeleteCompositeApp()
-	// if err != nil {
-	// 	return pkgerrors.Wrap(err, "Error deleting AppContext CompositeApp")
-	// }
-
-	// remove the app context field from the cluster db record
-	// lckey := LogicalCloudKey{
-	// 	LogicalCloudName: logicalcloud.MetaData.LogicalCloudName,
-	// 	Project:          project,
-	// }
-	// err = db.DBconn.RemoveTag("orchestrator", lckey, "lccontext")
-	// if err != nil {
-	// 	log.Warn("Error removing AppContext from Logical Cloud", log.Fields{
-	// 		"logical-cloud": logicalCloudName,
-	// 	})
-	// }
-
-	return nil
+	return pkgerrors.New("Logical Cloud is not applied: " + logicalCloudName)
 }