Add terminate support to orchestrator and ncm

Complete the basic terminate support for the
orchestrator and ncm services.

1. When terminate REST API is invoked on a deployment
   intent group, call the uninstall grpc api to rsync
   and then remove the app context.

2. When terminate REST API is invoked on a cluster,
   add the uninstall grpc api call to rsync to remove
   the network resources from the clusters.

Issue-ID: MULTICLOUD-1040
Signed-off-by: Eric Multanen <eric.w.multanen@intel.com>
Change-Id: I181e891a8c7c973970af061f9ff07d80c3bb64f9

4 files changed, 123 insertions, 20 deletions

diff --git a/src/orchestrator/pkg/grpc/installappclient/client.go b/src/orchestrator/pkg/grpc/installappclient/client.go
index 4c652a84..0e9141a6 100644
--- a/src/orchestrator/pkg/grpc/installappclient/client.go
+++ b/src/orchestrator/pkg/grpc/installappclient/client.go
@@ -19,10 +19,32 @@ import (
 
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/rpc"
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/module/controller"
 	installpb "github.com/onap/multicloud-k8s/src/rsync/pkg/grpc/installapp"
 	pkgerrors "github.com/pkg/errors"
 )
 
+const rsyncName = "rsync"
+
+// InitRsyncClient initializes connctions to the Resource Synchronizer service
+func initRsyncClient() bool {
+	client := controller.NewControllerClient()
+
+	vals, _ := client.GetControllers()
+	found := false
+	for _, v := range vals {
+		if v.Metadata.Name == rsyncName {
+			log.Info("Initializing RPC connection to resource synchronizer", log.Fields{
+				"Controller": v.Metadata.Name,
+			})
+			rpc.UpdateRpcConn(v.Metadata.Name, v.Spec.Host, v.Spec.Port)
+			found = true
+			break
+		}
+	}
+	return found
+}
+
 // InvokeInstallApp will make the grpc call to the resource synchronizer
 // or rsync controller.
 // rsync will deply the resources in the app context to the clusters as
@@ -34,7 +56,11 @@ func InvokeInstallApp(appContextId string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	conn := rpc.GetRpcConn("rsync")
+	conn := rpc.GetRpcConn(rsyncName)
+	if conn == nil {
+		initRsyncClient()
+		conn = rpc.GetRpcConn(rsyncName)
+	}
 
 	if conn != nil {
 		rpcClient = installpb.NewInstallappClient(conn)
@@ -64,3 +90,41 @@ func InvokeInstallApp(appContextId string) error {
 	}
 	return err
 }
+
+func InvokeUninstallApp(appContextId string) error {
+	var err error
+	var rpcClient installpb.InstallappClient
+	var uninstallRes *installpb.UninstallAppResponse
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	conn := rpc.GetRpcConn("rsync")
+
+	if conn != nil {
+		rpcClient = installpb.NewInstallappClient(conn)
+		uninstallReq := new(installpb.UninstallAppRequest)
+		uninstallReq.AppContext = appContextId
+		uninstallRes, err = rpcClient.UninstallApp(ctx, uninstallReq)
+		if err == nil {
+			log.Info("Response from UninstappApp GRPC call", log.Fields{
+				"Succeeded": uninstallRes.AppContextUninstalled,
+				"Message":   uninstallRes.AppContextUninstallMessage,
+			})
+		}
+	} else {
+		return pkgerrors.Errorf("UninstallApp Failed - Could not get InstallAppClient: %v", "rsync")
+	}
+
+	if err == nil {
+		if uninstallRes.AppContextUninstalled {
+			log.Info("UninstallApp Success", log.Fields{
+				"AppContext": appContextId,
+				"Message":    uninstallRes.AppContextUninstallMessage,
+			})
+			return nil
+		} else {
+			return pkgerrors.Errorf("UninstallApp Failed: %v", uninstallRes.AppContextUninstallMessage)
+		}
+	}
+	return err
+}
diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index 3412a034..0decb68f 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -62,7 +62,7 @@ type OverrideValues struct {
 type DeploymentIntentGroupManager interface {
 	CreateDeploymentIntentGroup(d DeploymentIntentGroup, p string, ca string, v string) (DeploymentIntentGroup, error)
 	GetDeploymentIntentGroup(di string, p string, ca string, v string) (DeploymentIntentGroup, error)
-	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error)
+	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error)
 	DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error
 }
 
@@ -165,7 +165,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroup(di string, p stri
 }
 
 // GetDeploymentIntentGroup returns the DeploymentIntentGroup with a given name, project, compositeApp and version of compositeApp
-func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) {
+func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error) {
 
 	key := DeploymentIntentGroupKey{
 		Name:         di,
@@ -176,7 +176,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 
 	result, err := db.DBconn.Find(c.storeName, key, c.tagContext)
 	if err != nil {
-		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
+		return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
 	}
 
 	if result != nil {
@@ -184,12 +184,12 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 		var cc appcontext.AppContext
 		_, err = cc.LoadAppContext(ctxVal)
 		if err != nil {
-			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
+			return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
 		}
-		return cc, nil
+		return cc, ctxVal, nil
 	}
 
-	return appcontext.AppContext{}, pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
+	return appcontext.AppContext{}, "", pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
 }
 
 // DeleteDeploymentIntentGroup deletes a DeploymentIntentGroup
@@ -200,10 +200,14 @@ func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p s
 		CompositeApp: ca,
 		Version:      v,
 	}
+	_, _, err := c.GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup must be terminated before it can be deleted "+di)
+	}
 
-	err := db.DBconn.Remove(c.storeName, k)
+	err = db.DBconn.Remove(c.storeName, k)
 	if err != nil {
-		return pkgerrors.Wrap(err, "Delete DeploymentIntentGroup entry;")
+		return pkgerrors.Wrap(err, "Error deleting DeploymentIntentGroup entry")
 	}
 	return nil
 
diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go
index 9432e4b9..1f2e1117 100644
--- a/src/orchestrator/pkg/module/instantiation.go
+++ b/src/orchestrator/pkg/module/instantiation.go
@@ -188,6 +188,12 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	if err != nil {
 		return pkgerrors.Wrap(err, "Not finding the deploymentIntentGroup")
 	}
+
+	_, _, err = NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Errorf("DeploymentIntentGroup has already been instantiated: " + di)
+	}
+
 	rName := dIGrp.Spec.Version //rName is releaseName
 	overrideValues := dIGrp.Spec.OverrideValuesObj
 	cp := dIGrp.Spec.Profile
@@ -331,7 +337,7 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	// END: Scheduler code
 
 	// BEGIN : Rsync code
-	err = callRsync(ctxval)
+	err = callRsyncInstall(ctxval)
 	if err != nil {
 		return err
 	}
@@ -348,9 +354,9 @@ the deployment, which is made available in the appcontext.
 */
 func (c InstantiationClient) Status(p string, ca string, v string, di string) (StatusData, error) {
 
-	ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, _, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
 	if err != nil {
-		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found: "+di)
 	}
 
 	// Get all apps in this composite app
@@ -409,15 +415,32 @@ DeploymentIntentName and calls rsync to terminate.
 */
 func (c InstantiationClient) Terminate(p string, ca string, v string, di string) error {
 
-	//ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
-	_, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, ctxval, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup has no app context: "+di)
+	}
+
+	err = callRsyncUninstall(ctxval)
+	if err != nil {
+		return err
+	}
+
+	err = ac.DeleteCompositeApp()
 	if err != nil {
-		return pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return pkgerrors.Wrap(err, "Error deleting the app context for DeploymentIntentGroup: "+di)
 	}
 
-	// TODO - make call to rsync to terminate the composite app deployment
-	//        will leave the appcontext in place for clean up later
-	//        so monitoring status can be performed
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
+	}
+
+	err = db.DBconn.RemoveTag(c.db.storeName, key, c.db.tagContext)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Error removing the app context tag from DeploymentIntentGroup: "+di)
+	}
 
 	return nil
 }
diff --git a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
index 3d9d851c..184d6972 100644
--- a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
+++ b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
@@ -192,9 +192,9 @@ func callGrpcForControllerList(cl []controller.Controller, mc map[string]string,
 }
 
 /*
-callRsync method shall take in the app context id and invokes the rsync service via grpc
+callRsyncInstall method shall take in the app context id and invokes the rsync service via grpc
 */
-func callRsync(contextid interface{}) error {
+func callRsyncInstall(contextid interface{}) error {
 	appContextID := fmt.Sprintf("%v", contextid)
 	err := rsyncclient.InvokeInstallApp(appContextID)
 	if err != nil {
@@ -204,6 +204,18 @@ func callRsync(contextid interface{}) error {
 }
 
 /*
+callRsyncUninstall method shall take in the app context id and invokes the rsync service via grpc
+*/
+func callRsyncUninstall(contextid interface{}) error {
+	appContextID := fmt.Sprintf("%v", contextid)
+	err := rsyncclient.InvokeUninstallApp(appContextID)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+/*
 deleteExtraClusters method shall delete the extra cluster handles for each AnyOf cluster present in the etcd after the grpc call for context updation.
 */
 func deleteExtraClusters(apps []App, ct appcontext.AppContext) error {