Add terminate support to orchestrator and ncm

Complete the basic terminate support for the
orchestrator and ncm services.

1. When terminate REST API is invoked on a deployment
   intent group, call the uninstall grpc api to rsync
   and then remove the app context.

2. When terminate REST API is invoked on a cluster,
   add the uninstall grpc api call to rsync to remove
   the network resources from the clusters.

Issue-ID: MULTICLOUD-1040
Signed-off-by: Eric Multanen <eric.w.multanen@intel.com>
Change-Id: I181e891a8c7c973970af061f9ff07d80c3bb64f9

11 files changed, 174 insertions, 113 deletions

diff --git a/kud/tests/README-composite-vfw.txt b/kud/tests/README-composite-vfw.txt
index 3f334a25..d15c0352 100644
--- a/kud/tests/README-composite-vfw.txt
+++ b/kud/tests/README-composite-vfw.txt
@@ -127,18 +127,24 @@ The rest of the data needed for the test is present in the script.
     This will instantiate the vFW on the two edge clusters (as defined by the generic
     placement intent).
 
+5. Status query
 
-# Removing resources (until termination sequences are completed)
+   vfw-test.sh status
 
-1. Run the cleanup script (or equivalent) in the edge clusters.
-   (once the terminate flow via EMCO is complete, this step will not be necessary)
+6. Terminate
+   Terminate will remove the resources from the clusters and delete the internal
+   composite application information in the etcd base AppContext.
+   The script will do it for both the deployment intent group (i.e. the vfW composite
+   app) and the network intents.
 
-   bash cleanup-composite-vfw.sh
-
-2. Terminate the network intents
+   In principle, after runnin terminate, the 'apply' and 'instantiate' commands could
+   be invoked again to re-insantiate the networks and the vFW composite app.
 
    vfw-test.sh terminate
 
-3. Delete everything from the Mongo DB
+7. Delete the data
+   After running 'terminate', the 'delete' command can be invoked to remove all
+   the data created.  This should leave the system back in the starting state -
+   begin with point #1 above to start again.
 
    vfw-test.sh delete
diff --git a/kud/tests/vfw-test.sh b/kud/tests/vfw-test.sh
index 9c431ef0..b14ad95b 100755
--- a/kud/tests/vfw-test.sh
+++ b/kud/tests/vfw-test.sh
@@ -958,12 +958,23 @@ function applyNcmData {
     call_api -d "{ }" "${base_url_ncm}/cluster-providers/${clusterprovidername}/clusters/${clustername2}/apply"
 }
 
-# deletes the appcontext (eventually will terminate from resource synchronizer when that funcationality is ready)
+# deletes the network resources from the clusters and the associated appcontext entries
 function terminateNcmData {
     call_api -d "{ }" "${base_url_ncm}/cluster-providers/${clusterprovidername}/clusters/${clustername}/terminate"
     call_api -d "{ }" "${base_url_ncm}/cluster-providers/${clusterprovidername}/clusters/${clustername2}/terminate"
 }
 
+# terminates the vfw resources
+function terminateOrchData {
+    call_api -d "{ }" "${base_url_orchestrator}/projects/${projectname}/composite-apps/${vfw_compositeapp_name}/${vfw_compositeapp_version}/deployment-intent-groups/${deployment_intent_group_name}/terminate"
+}
+
+# terminates the vfw and ncm resources
+function terminateVfw {
+    terminateOrchData
+    terminateNcmData
+}
+
 function instantiateVfw {
     # call_api -d "{ }" "${base_url_orchestrator}/projects/${projectname}/composite-apps/${vfw_compositeapp_name}/${vfw_compositeapp_version}/deployment-intent-groups/${deployment_intent_group_name}/approve"
     call_api -d "{ }" "${base_url_orchestrator}/projects/${projectname}/composite-apps/${vfw_compositeapp_name}/${vfw_compositeapp_version}/deployment-intent-groups/${deployment_intent_group_name}/instantiate"
@@ -990,13 +1001,14 @@ function usage {
     echo "    apply - applys the network intents - e.g. networks created in ncm"
     echo "    instantiate - approves and instantiates the composite app via the generic deployment intent"
     echo "    status - get status of deployed resources"
-    echo "    terminate - remove the network inents created by ncm"
+    echo "    terminate - remove the vFW composite app resources and network resources create by 'instantiate' and 'apply'"
     echo ""
     echo "    a reasonable test sequence:"
     echo "    1.  create"
     echo "    2.  apply"
     echo "    3.  instantiate"
     echo "    4.  status"
+    echo "    5.  terminate"
 
     exit
 }
@@ -1053,8 +1065,8 @@ case "$1" in
     "get" )    getData ;;
     "delete" ) deleteData ;;
     "apply" ) applyNcmData ;;
-    "terminate" ) terminateNcmData ;;
     "instantiate" ) instantiateVfw ;;
+    "terminate" ) terminateVfw ;;
     "status" ) statusVfw ;;
     *) usage ;;
 esac
diff --git a/src/clm/api/clusterhandler_test.go b/src/clm/api/clusterhandler_test.go
index 7095e87c..4bbc91b1 100644
--- a/src/clm/api/clusterhandler_test.go
+++ b/src/clm/api/clusterhandler_test.go
@@ -102,12 +102,12 @@ func (m *mockClusterManager) GetClusterContent(provider, name string) (cluster.C
 	return m.ClusterContentItems[0], nil
 }
 
-func (m *mockClusterManager) GetClusterContext(provider, name string) (appcontext.AppContext, error) {
+func (m *mockClusterManager) GetClusterContext(provider, name string) (appcontext.AppContext, string, error) {
 	if m.Err != nil {
-		return appcontext.AppContext{}, m.Err
+		return appcontext.AppContext{}, "", m.Err
 	}
 
-	return m.ClusterContextItems[0], nil
+	return m.ClusterContextItems[0], "", nil
 }
 
 func (m *mockClusterManager) GetClusters(provider string) ([]cluster.Cluster, error) {
diff --git a/src/clm/pkg/cluster/cluster.go b/src/clm/pkg/cluster/cluster.go
index 06faafd2..ac7f31f7 100644
--- a/src/clm/pkg/cluster/cluster.go
+++ b/src/clm/pkg/cluster/cluster.go
@@ -101,7 +101,7 @@ type ClusterManager interface {
 	CreateCluster(provider string, pr Cluster, qr ClusterContent) (Cluster, error)
 	GetCluster(provider, name string) (Cluster, error)
 	GetClusterContent(provider, name string) (ClusterContent, error)
-	GetClusterContext(provider, name string) (appcontext.AppContext, error)
+	GetClusterContext(provider, name string) (appcontext.AppContext, string, error)
 	GetClusters(provider string) ([]Cluster, error)
 	GetClustersWithLabel(provider, label string) ([]string, error)
 	DeleteCluster(provider, name string) error
@@ -310,7 +310,7 @@ func (v *ClusterClient) GetClusterContent(provider, name string) (ClusterContent
 }
 
 // GetClusterContext returns the AppContext for corresponding provider and name
-func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.AppContext, error) {
+func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.AppContext, string, error) {
 	//Construct key and tag to select the entry
 	key := ClusterKey{
 		ClusterProviderName: provider,
@@ -319,7 +319,7 @@ func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.App
 
 	value, err := db.DBconn.Find(v.db.storeName, key, v.db.tagContext)
 	if err != nil {
-		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get Cluster Context")
+		return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Get Cluster Context")
 	}
 
 	//value is a byte array
@@ -328,12 +328,12 @@ func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.App
 		var cc appcontext.AppContext
 		_, err = cc.LoadAppContext(ctxVal)
 		if err != nil {
-			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Reinitializing Cluster AppContext")
+			return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Reinitializing Cluster AppContext")
 		}
-		return cc, nil
+		return cc, ctxVal, nil
 	}
 
-	return appcontext.AppContext{}, pkgerrors.New("Error getting Cluster AppContext")
+	return appcontext.AppContext{}, "", pkgerrors.New("Error getting Cluster AppContext")
 }
 
 // GetClusters returns all the Clusters for corresponding provider
@@ -393,7 +393,7 @@ func (v *ClusterClient) DeleteCluster(provider, name string) error {
 		ClusterProviderName: provider,
 		ClusterName:         name,
 	}
-	_, err := v.GetClusterContext(provider, name)
+	_, _, err := v.GetClusterContext(provider, name)
 	if err == nil {
 		return pkgerrors.Errorf("Cannot delete cluster until context is deleted: %v, %v", provider, name)
 	}
diff --git a/src/ncm/go.mod b/src/ncm/go.mod
index 233d8880..d3d2924a 100644
--- a/src/ncm/go.mod
+++ b/src/ncm/go.mod
@@ -5,6 +5,8 @@ require (
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.3
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200127152046-0ee521d56061
+	github.com/onap/multicloud-k8s/src/clm v0.0.0-00010101000000-000000000000
+	github.com/onap/multicloud-k8s/src/orchestrator v0.0.0-20200601021239-7959bd4c6fd4
 	github.com/pkg/errors v0.8.1
 	google.golang.org/grpc v1.27.1
 	gopkg.in/yaml.v2 v2.2.8
diff --git a/src/ncm/internal/grpc/rsyncclient.go b/src/ncm/internal/grpc/rsyncclient.go
deleted file mode 100644
index 5eb870a7..00000000
--- a/src/ncm/internal/grpc/rsyncclient.go
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
-Copyright 2020 Intel Corporation.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package grpc
-
-import (
-	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
-	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/rpc"
-	controller "github.com/onap/multicloud-k8s/src/orchestrator/pkg/module/controller"
-)
-
-const RsyncName = "rsync"
-
-// InitRsyncClient initializes connctions to the Resource Synchronizer serivice
-func InitRsyncClient() bool {
-	client := controller.NewControllerClient()
-
-	vals, _ := client.GetControllers()
-	found := false
-	for _, v := range vals {
-		if v.Metadata.Name == RsyncName {
-			log.Info("Initializing RPC connection to resource synchronizer", log.Fields{
-				"Controller": v.Metadata.Name,
-			})
-			rpc.UpdateRpcConn(v.Metadata.Name, v.Spec.Host, v.Spec.Port)
-			found = true
-			break
-		}
-	}
-	return found
-}
diff --git a/src/ncm/pkg/scheduler/scheduler.go b/src/ncm/pkg/scheduler/scheduler.go
index 29d67662..4886a67e 100644
--- a/src/ncm/pkg/scheduler/scheduler.go
+++ b/src/ncm/pkg/scheduler/scheduler.go
@@ -17,20 +17,16 @@
 package scheduler
 
 import (
-	"context"
 	"encoding/json"
-	"time"
 
 	clusterPkg "github.com/onap/multicloud-k8s/src/clm/pkg/cluster"
-	"github.com/onap/multicloud-k8s/src/ncm/internal/grpc"
 	oc "github.com/onap/multicloud-k8s/src/ncm/internal/ovncontroller"
 	ncmtypes "github.com/onap/multicloud-k8s/src/ncm/pkg/module/types"
 	nettypes "github.com/onap/multicloud-k8s/src/ncm/pkg/networkintents/types"
 	appcontext "github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/grpc/installappclient"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
-	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/rpc"
-	installpb "github.com/onap/multicloud-k8s/src/rsync/pkg/grpc/installapp"
 
 	pkgerrors "github.com/pkg/errors"
 )
@@ -63,7 +59,7 @@ func NewSchedulerClient() *SchedulerClient {
 // Apply Network Intents associated with a cluster
 func (v *SchedulerClient) ApplyNetworkIntents(clusterProvider, cluster string) error {
 
-	_, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
+	_, _, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
 	if err == nil {
 		return pkgerrors.Errorf("Cluster network intents have already been applied: %v, %v", clusterProvider, cluster)
 	}
@@ -157,30 +153,9 @@ func (v *SchedulerClient) ApplyNetworkIntents(clusterProvider, cluster string) e
 	}
 
 	// call resource synchronizer to instantiate the CRs in the cluster
-	conn := rpc.GetRpcConn(grpc.RsyncName)
-	if conn == nil {
-		grpc.InitRsyncClient()
-		conn = rpc.GetRpcConn(grpc.RsyncName)
-	}
-
-	var rpcClient installpb.InstallappClient
-	var installRes *installpb.InstallAppResponse
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	if conn != nil {
-		rpcClient = installpb.NewInstallappClient(conn)
-		installReq := new(installpb.InstallAppRequest)
-		installReq.AppContext = ctxVal.(string)
-		installRes, err = rpcClient.InstallApp(ctx, installReq)
-		if err == nil {
-			log.Info("Response from InstappApp GRPC call", log.Fields{
-				"Succeeded": installRes.AppContextInstalled,
-				"Message":   installRes.AppContextInstallMessage,
-			})
-		}
-	} else {
-		return pkgerrors.Errorf("InstallApp Failed - Could not get InstallAppClient: %v", grpc.RsyncName)
+	err = installappclient.InvokeInstallApp(ctxVal.(string))
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -188,12 +163,16 @@ func (v *SchedulerClient) ApplyNetworkIntents(clusterProvider, cluster string) e
 
 // Terminate Network Intents associated with a cluster
 func (v *SchedulerClient) TerminateNetworkIntents(clusterProvider, cluster string) error {
-	context, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
+	context, ctxVal, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
 	if err != nil {
 		return pkgerrors.Wrapf(err, "Error finding AppContext for cluster: %v, %v", clusterProvider, cluster)
 	}
 
-	// TODO: call resource synchronizer to terminate the CRs in the cluster
+	// call resource synchronizer to terminate the CRs in the cluster
+	err = installappclient.InvokeUninstallApp(ctxVal)
+	if err != nil {
+		return err
+	}
 
 	// remove the app context
 	cleanuperr := context.DeleteCompositeApp()
diff --git a/src/orchestrator/pkg/grpc/installappclient/client.go b/src/orchestrator/pkg/grpc/installappclient/client.go
index 4c652a84..0e9141a6 100644
--- a/src/orchestrator/pkg/grpc/installappclient/client.go
+++ b/src/orchestrator/pkg/grpc/installappclient/client.go
@@ -19,10 +19,32 @@ import (
 
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/rpc"
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/module/controller"
 	installpb "github.com/onap/multicloud-k8s/src/rsync/pkg/grpc/installapp"
 	pkgerrors "github.com/pkg/errors"
 )
 
+const rsyncName = "rsync"
+
+// InitRsyncClient initializes connctions to the Resource Synchronizer service
+func initRsyncClient() bool {
+	client := controller.NewControllerClient()
+
+	vals, _ := client.GetControllers()
+	found := false
+	for _, v := range vals {
+		if v.Metadata.Name == rsyncName {
+			log.Info("Initializing RPC connection to resource synchronizer", log.Fields{
+				"Controller": v.Metadata.Name,
+			})
+			rpc.UpdateRpcConn(v.Metadata.Name, v.Spec.Host, v.Spec.Port)
+			found = true
+			break
+		}
+	}
+	return found
+}
+
 // InvokeInstallApp will make the grpc call to the resource synchronizer
 // or rsync controller.
 // rsync will deply the resources in the app context to the clusters as
@@ -34,7 +56,11 @@ func InvokeInstallApp(appContextId string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	conn := rpc.GetRpcConn("rsync")
+	conn := rpc.GetRpcConn(rsyncName)
+	if conn == nil {
+		initRsyncClient()
+		conn = rpc.GetRpcConn(rsyncName)
+	}
 
 	if conn != nil {
 		rpcClient = installpb.NewInstallappClient(conn)
@@ -64,3 +90,41 @@ func InvokeInstallApp(appContextId string) error {
 	}
 	return err
 }
+
+func InvokeUninstallApp(appContextId string) error {
+	var err error
+	var rpcClient installpb.InstallappClient
+	var uninstallRes *installpb.UninstallAppResponse
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	conn := rpc.GetRpcConn("rsync")
+
+	if conn != nil {
+		rpcClient = installpb.NewInstallappClient(conn)
+		uninstallReq := new(installpb.UninstallAppRequest)
+		uninstallReq.AppContext = appContextId
+		uninstallRes, err = rpcClient.UninstallApp(ctx, uninstallReq)
+		if err == nil {
+			log.Info("Response from UninstappApp GRPC call", log.Fields{
+				"Succeeded": uninstallRes.AppContextUninstalled,
+				"Message":   uninstallRes.AppContextUninstallMessage,
+			})
+		}
+	} else {
+		return pkgerrors.Errorf("UninstallApp Failed - Could not get InstallAppClient: %v", "rsync")
+	}
+
+	if err == nil {
+		if uninstallRes.AppContextUninstalled {
+			log.Info("UninstallApp Success", log.Fields{
+				"AppContext": appContextId,
+				"Message":    uninstallRes.AppContextUninstallMessage,
+			})
+			return nil
+		} else {
+			return pkgerrors.Errorf("UninstallApp Failed: %v", uninstallRes.AppContextUninstallMessage)
+		}
+	}
+	return err
+}
diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index 3412a034..0decb68f 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -62,7 +62,7 @@ type OverrideValues struct {
 type DeploymentIntentGroupManager interface {
 	CreateDeploymentIntentGroup(d DeploymentIntentGroup, p string, ca string, v string) (DeploymentIntentGroup, error)
 	GetDeploymentIntentGroup(di string, p string, ca string, v string) (DeploymentIntentGroup, error)
-	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error)
+	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error)
 	DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error
 }
 
@@ -165,7 +165,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroup(di string, p stri
 }
 
 // GetDeploymentIntentGroup returns the DeploymentIntentGroup with a given name, project, compositeApp and version of compositeApp
-func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) {
+func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error) {
 
 	key := DeploymentIntentGroupKey{
 		Name:         di,
@@ -176,7 +176,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 
 	result, err := db.DBconn.Find(c.storeName, key, c.tagContext)
 	if err != nil {
-		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
+		return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
 	}
 
 	if result != nil {
@@ -184,12 +184,12 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 		var cc appcontext.AppContext
 		_, err = cc.LoadAppContext(ctxVal)
 		if err != nil {
-			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
+			return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
 		}
-		return cc, nil
+		return cc, ctxVal, nil
 	}
 
-	return appcontext.AppContext{}, pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
+	return appcontext.AppContext{}, "", pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
 }
 
 // DeleteDeploymentIntentGroup deletes a DeploymentIntentGroup
@@ -200,10 +200,14 @@ func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p s
 		CompositeApp: ca,
 		Version:      v,
 	}
+	_, _, err := c.GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup must be terminated before it can be deleted "+di)
+	}
 
-	err := db.DBconn.Remove(c.storeName, k)
+	err = db.DBconn.Remove(c.storeName, k)
 	if err != nil {
-		return pkgerrors.Wrap(err, "Delete DeploymentIntentGroup entry;")
+		return pkgerrors.Wrap(err, "Error deleting DeploymentIntentGroup entry")
 	}
 	return nil
 
diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go
index 9432e4b9..1f2e1117 100644
--- a/src/orchestrator/pkg/module/instantiation.go
+++ b/src/orchestrator/pkg/module/instantiation.go
@@ -188,6 +188,12 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	if err != nil {
 		return pkgerrors.Wrap(err, "Not finding the deploymentIntentGroup")
 	}
+
+	_, _, err = NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Errorf("DeploymentIntentGroup has already been instantiated: " + di)
+	}
+
 	rName := dIGrp.Spec.Version //rName is releaseName
 	overrideValues := dIGrp.Spec.OverrideValuesObj
 	cp := dIGrp.Spec.Profile
@@ -331,7 +337,7 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	// END: Scheduler code
 
 	// BEGIN : Rsync code
-	err = callRsync(ctxval)
+	err = callRsyncInstall(ctxval)
 	if err != nil {
 		return err
 	}
@@ -348,9 +354,9 @@ the deployment, which is made available in the appcontext.
 */
 func (c InstantiationClient) Status(p string, ca string, v string, di string) (StatusData, error) {
 
-	ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, _, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
 	if err != nil {
-		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found: "+di)
 	}
 
 	// Get all apps in this composite app
@@ -409,15 +415,32 @@ DeploymentIntentName and calls rsync to terminate.
 */
 func (c InstantiationClient) Terminate(p string, ca string, v string, di string) error {
 
-	//ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
-	_, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, ctxval, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup has no app context: "+di)
+	}
+
+	err = callRsyncUninstall(ctxval)
+	if err != nil {
+		return err
+	}
+
+	err = ac.DeleteCompositeApp()
 	if err != nil {
-		return pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return pkgerrors.Wrap(err, "Error deleting the app context for DeploymentIntentGroup: "+di)
 	}
 
-	// TODO - make call to rsync to terminate the composite app deployment
-	//        will leave the appcontext in place for clean up later
-	//        so monitoring status can be performed
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
+	}
+
+	err = db.DBconn.RemoveTag(c.db.storeName, key, c.db.tagContext)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Error removing the app context tag from DeploymentIntentGroup: "+di)
+	}
 
 	return nil
 }
diff --git a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
index 3d9d851c..184d6972 100644
--- a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
+++ b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
@@ -192,9 +192,9 @@ func callGrpcForControllerList(cl []controller.Controller, mc map[string]string,
 }
 
 /*
-callRsync method shall take in the app context id and invokes the rsync service via grpc
+callRsyncInstall method shall take in the app context id and invokes the rsync service via grpc
 */
-func callRsync(contextid interface{}) error {
+func callRsyncInstall(contextid interface{}) error {
 	appContextID := fmt.Sprintf("%v", contextid)
 	err := rsyncclient.InvokeInstallApp(appContextID)
 	if err != nil {
@@ -204,6 +204,18 @@ func callRsync(contextid interface{}) error {
 }
 
 /*
+callRsyncUninstall method shall take in the app context id and invokes the rsync service via grpc
+*/
+func callRsyncUninstall(contextid interface{}) error {
+	appContextID := fmt.Sprintf("%v", contextid)
+	err := rsyncclient.InvokeUninstallApp(appContextID)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+/*
 deleteExtraClusters method shall delete the extra cluster handles for each AnyOf cluster present in the etcd after the grpc call for context updation.
 */
 func deleteExtraClusters(apps []App, ct appcontext.AppContext) error {