Add terminate support to orchestrator and ncm

Complete the basic terminate support for the
orchestrator and ncm services.

1. When terminate REST API is invoked on a deployment
   intent group, call the uninstall grpc api to rsync
   and then remove the app context.

2. When terminate REST API is invoked on a cluster,
   add the uninstall grpc api call to rsync to remove
   the network resources from the clusters.

Issue-ID: MULTICLOUD-1040
Signed-off-by: Eric Multanen <eric.w.multanen@intel.com>
Change-Id: I181e891a8c7c973970af061f9ff07d80c3bb64f9

3 files changed, 58 insertions, 19 deletions

diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index 3412a034..0decb68f 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -62,7 +62,7 @@ type OverrideValues struct {
 type DeploymentIntentGroupManager interface {
 	CreateDeploymentIntentGroup(d DeploymentIntentGroup, p string, ca string, v string) (DeploymentIntentGroup, error)
 	GetDeploymentIntentGroup(di string, p string, ca string, v string) (DeploymentIntentGroup, error)
-	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error)
+	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error)
 	DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error
 }
 
@@ -165,7 +165,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroup(di string, p stri
 }
 
 // GetDeploymentIntentGroup returns the DeploymentIntentGroup with a given name, project, compositeApp and version of compositeApp
-func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) {
+func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error) {
 
 	key := DeploymentIntentGroupKey{
 		Name:         di,
@@ -176,7 +176,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 
 	result, err := db.DBconn.Find(c.storeName, key, c.tagContext)
 	if err != nil {
-		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
+		return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
 	}
 
 	if result != nil {
@@ -184,12 +184,12 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 		var cc appcontext.AppContext
 		_, err = cc.LoadAppContext(ctxVal)
 		if err != nil {
-			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
+			return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
 		}
-		return cc, nil
+		return cc, ctxVal, nil
 	}
 
-	return appcontext.AppContext{}, pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
+	return appcontext.AppContext{}, "", pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
 }
 
 // DeleteDeploymentIntentGroup deletes a DeploymentIntentGroup
@@ -200,10 +200,14 @@ func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p s
 		CompositeApp: ca,
 		Version:      v,
 	}
+	_, _, err := c.GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup must be terminated before it can be deleted "+di)
+	}
 
-	err := db.DBconn.Remove(c.storeName, k)
+	err = db.DBconn.Remove(c.storeName, k)
 	if err != nil {
-		return pkgerrors.Wrap(err, "Delete DeploymentIntentGroup entry;")
+		return pkgerrors.Wrap(err, "Error deleting DeploymentIntentGroup entry")
 	}
 	return nil
 
diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go
index 9432e4b9..1f2e1117 100644
--- a/src/orchestrator/pkg/module/instantiation.go
+++ b/src/orchestrator/pkg/module/instantiation.go
@@ -188,6 +188,12 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	if err != nil {
 		return pkgerrors.Wrap(err, "Not finding the deploymentIntentGroup")
 	}
+
+	_, _, err = NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Errorf("DeploymentIntentGroup has already been instantiated: " + di)
+	}
+
 	rName := dIGrp.Spec.Version //rName is releaseName
 	overrideValues := dIGrp.Spec.OverrideValuesObj
 	cp := dIGrp.Spec.Profile
@@ -331,7 +337,7 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	// END: Scheduler code
 
 	// BEGIN : Rsync code
-	err = callRsync(ctxval)
+	err = callRsyncInstall(ctxval)
 	if err != nil {
 		return err
 	}
@@ -348,9 +354,9 @@ the deployment, which is made available in the appcontext.
 */
 func (c InstantiationClient) Status(p string, ca string, v string, di string) (StatusData, error) {
 
-	ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, _, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
 	if err != nil {
-		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found: "+di)
 	}
 
 	// Get all apps in this composite app
@@ -409,15 +415,32 @@ DeploymentIntentName and calls rsync to terminate.
 */
 func (c InstantiationClient) Terminate(p string, ca string, v string, di string) error {
 
-	//ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
-	_, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, ctxval, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup has no app context: "+di)
+	}
+
+	err = callRsyncUninstall(ctxval)
+	if err != nil {
+		return err
+	}
+
+	err = ac.DeleteCompositeApp()
 	if err != nil {
-		return pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return pkgerrors.Wrap(err, "Error deleting the app context for DeploymentIntentGroup: "+di)
 	}
 
-	// TODO - make call to rsync to terminate the composite app deployment
-	//        will leave the appcontext in place for clean up later
-	//        so monitoring status can be performed
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
+	}
+
+	err = db.DBconn.RemoveTag(c.db.storeName, key, c.db.tagContext)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Error removing the app context tag from DeploymentIntentGroup: "+di)
+	}
 
 	return nil
 }
diff --git a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
index 3d9d851c..184d6972 100644
--- a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
+++ b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
@@ -192,9 +192,9 @@ func callGrpcForControllerList(cl []controller.Controller, mc map[string]string,
 }
 
 /*
-callRsync method shall take in the app context id and invokes the rsync service via grpc
+callRsyncInstall method shall take in the app context id and invokes the rsync service via grpc
 */
-func callRsync(contextid interface{}) error {
+func callRsyncInstall(contextid interface{}) error {
 	appContextID := fmt.Sprintf("%v", contextid)
 	err := rsyncclient.InvokeInstallApp(appContextID)
 	if err != nil {
@@ -204,6 +204,18 @@ func callRsync(contextid interface{}) error {
 }
 
 /*
+callRsyncUninstall method shall take in the app context id and invokes the rsync service via grpc
+*/
+func callRsyncUninstall(contextid interface{}) error {
+	appContextID := fmt.Sprintf("%v", contextid)
+	err := rsyncclient.InvokeUninstallApp(appContextID)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+/*
 deleteExtraClusters method shall delete the extra cluster handles for each AnyOf cluster present in the etcd after the grpc call for context updation.
 */
 func deleteExtraClusters(apps []App, ct appcontext.AppContext) error {