Add terminate support to orchestrator and ncm

Complete the basic terminate support for the
orchestrator and ncm services.

1. When terminate REST API is invoked on a deployment
   intent group, call the uninstall grpc api to rsync
   and then remove the app context.

2. When terminate REST API is invoked on a cluster,
   add the uninstall grpc api call to rsync to remove
   the network resources from the clusters.

Issue-ID: MULTICLOUD-1040
Signed-off-by: Eric Multanen <eric.w.multanen@intel.com>
Change-Id: I181e891a8c7c973970af061f9ff07d80c3bb64f9

9 files changed, 146 insertions, 103 deletions

diff --git a/src/clm/api/clusterhandler_test.go b/src/clm/api/clusterhandler_test.go
index 7095e87c..4bbc91b1 100644
--- a/src/clm/api/clusterhandler_test.go
+++ b/src/clm/api/clusterhandler_test.go
@@ -102,12 +102,12 @@ func (m *mockClusterManager) GetClusterContent(provider, name string) (cluster.C
 	return m.ClusterContentItems[0], nil
 }
 
-func (m *mockClusterManager) GetClusterContext(provider, name string) (appcontext.AppContext, error) {
+func (m *mockClusterManager) GetClusterContext(provider, name string) (appcontext.AppContext, string, error) {
 	if m.Err != nil {
-		return appcontext.AppContext{}, m.Err
+		return appcontext.AppContext{}, "", m.Err
 	}
 
-	return m.ClusterContextItems[0], nil
+	return m.ClusterContextItems[0], "", nil
 }
 
 func (m *mockClusterManager) GetClusters(provider string) ([]cluster.Cluster, error) {
diff --git a/src/clm/pkg/cluster/cluster.go b/src/clm/pkg/cluster/cluster.go
index 06faafd2..ac7f31f7 100644
--- a/src/clm/pkg/cluster/cluster.go
+++ b/src/clm/pkg/cluster/cluster.go
@@ -101,7 +101,7 @@ type ClusterManager interface {
 	CreateCluster(provider string, pr Cluster, qr ClusterContent) (Cluster, error)
 	GetCluster(provider, name string) (Cluster, error)
 	GetClusterContent(provider, name string) (ClusterContent, error)
-	GetClusterContext(provider, name string) (appcontext.AppContext, error)
+	GetClusterContext(provider, name string) (appcontext.AppContext, string, error)
 	GetClusters(provider string) ([]Cluster, error)
 	GetClustersWithLabel(provider, label string) ([]string, error)
 	DeleteCluster(provider, name string) error
@@ -310,7 +310,7 @@ func (v *ClusterClient) GetClusterContent(provider, name string) (ClusterContent
 }
 
 // GetClusterContext returns the AppContext for corresponding provider and name
-func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.AppContext, error) {
+func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.AppContext, string, error) {
 	//Construct key and tag to select the entry
 	key := ClusterKey{
 		ClusterProviderName: provider,
@@ -319,7 +319,7 @@ func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.App
 
 	value, err := db.DBconn.Find(v.db.storeName, key, v.db.tagContext)
 	if err != nil {
-		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get Cluster Context")
+		return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Get Cluster Context")
 	}
 
 	//value is a byte array
@@ -328,12 +328,12 @@ func (v *ClusterClient) GetClusterContext(provider, name string) (appcontext.App
 		var cc appcontext.AppContext
 		_, err = cc.LoadAppContext(ctxVal)
 		if err != nil {
-			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Reinitializing Cluster AppContext")
+			return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Reinitializing Cluster AppContext")
 		}
-		return cc, nil
+		return cc, ctxVal, nil
 	}
 
-	return appcontext.AppContext{}, pkgerrors.New("Error getting Cluster AppContext")
+	return appcontext.AppContext{}, "", pkgerrors.New("Error getting Cluster AppContext")
 }
 
 // GetClusters returns all the Clusters for corresponding provider
@@ -393,7 +393,7 @@ func (v *ClusterClient) DeleteCluster(provider, name string) error {
 		ClusterProviderName: provider,
 		ClusterName:         name,
 	}
-	_, err := v.GetClusterContext(provider, name)
+	_, _, err := v.GetClusterContext(provider, name)
 	if err == nil {
 		return pkgerrors.Errorf("Cannot delete cluster until context is deleted: %v, %v", provider, name)
 	}
diff --git a/src/ncm/go.mod b/src/ncm/go.mod
index 233d8880..d3d2924a 100644
--- a/src/ncm/go.mod
+++ b/src/ncm/go.mod
@@ -5,6 +5,8 @@ require (
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.3
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200127152046-0ee521d56061
+	github.com/onap/multicloud-k8s/src/clm v0.0.0-00010101000000-000000000000
+	github.com/onap/multicloud-k8s/src/orchestrator v0.0.0-20200601021239-7959bd4c6fd4
 	github.com/pkg/errors v0.8.1
 	google.golang.org/grpc v1.27.1
 	gopkg.in/yaml.v2 v2.2.8
diff --git a/src/ncm/internal/grpc/rsyncclient.go b/src/ncm/internal/grpc/rsyncclient.go
deleted file mode 100644
index 5eb870a7..00000000
--- a/src/ncm/internal/grpc/rsyncclient.go
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
-Copyright 2020 Intel Corporation.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package grpc
-
-import (
-	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
-	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/rpc"
-	controller "github.com/onap/multicloud-k8s/src/orchestrator/pkg/module/controller"
-)
-
-const RsyncName = "rsync"
-
-// InitRsyncClient initializes connctions to the Resource Synchronizer serivice
-func InitRsyncClient() bool {
-	client := controller.NewControllerClient()
-
-	vals, _ := client.GetControllers()
-	found := false
-	for _, v := range vals {
-		if v.Metadata.Name == RsyncName {
-			log.Info("Initializing RPC connection to resource synchronizer", log.Fields{
-				"Controller": v.Metadata.Name,
-			})
-			rpc.UpdateRpcConn(v.Metadata.Name, v.Spec.Host, v.Spec.Port)
-			found = true
-			break
-		}
-	}
-	return found
-}
diff --git a/src/ncm/pkg/scheduler/scheduler.go b/src/ncm/pkg/scheduler/scheduler.go
index 29d67662..4886a67e 100644
--- a/src/ncm/pkg/scheduler/scheduler.go
+++ b/src/ncm/pkg/scheduler/scheduler.go
@@ -17,20 +17,16 @@
 package scheduler
 
 import (
-	"context"
 	"encoding/json"
-	"time"
 
 	clusterPkg "github.com/onap/multicloud-k8s/src/clm/pkg/cluster"
-	"github.com/onap/multicloud-k8s/src/ncm/internal/grpc"
 	oc "github.com/onap/multicloud-k8s/src/ncm/internal/ovncontroller"
 	ncmtypes "github.com/onap/multicloud-k8s/src/ncm/pkg/module/types"
 	nettypes "github.com/onap/multicloud-k8s/src/ncm/pkg/networkintents/types"
 	appcontext "github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/grpc/installappclient"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
-	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/rpc"
-	installpb "github.com/onap/multicloud-k8s/src/rsync/pkg/grpc/installapp"
 
 	pkgerrors "github.com/pkg/errors"
 )
@@ -63,7 +59,7 @@ func NewSchedulerClient() *SchedulerClient {
 // Apply Network Intents associated with a cluster
 func (v *SchedulerClient) ApplyNetworkIntents(clusterProvider, cluster string) error {
 
-	_, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
+	_, _, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
 	if err == nil {
 		return pkgerrors.Errorf("Cluster network intents have already been applied: %v, %v", clusterProvider, cluster)
 	}
@@ -157,30 +153,9 @@ func (v *SchedulerClient) ApplyNetworkIntents(clusterProvider, cluster string) e
 	}
 
 	// call resource synchronizer to instantiate the CRs in the cluster
-	conn := rpc.GetRpcConn(grpc.RsyncName)
-	if conn == nil {
-		grpc.InitRsyncClient()
-		conn = rpc.GetRpcConn(grpc.RsyncName)
-	}
-
-	var rpcClient installpb.InstallappClient
-	var installRes *installpb.InstallAppResponse
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	if conn != nil {
-		rpcClient = installpb.NewInstallappClient(conn)
-		installReq := new(installpb.InstallAppRequest)
-		installReq.AppContext = ctxVal.(string)
-		installRes, err = rpcClient.InstallApp(ctx, installReq)
-		if err == nil {
-			log.Info("Response from InstappApp GRPC call", log.Fields{
-				"Succeeded": installRes.AppContextInstalled,
-				"Message":   installRes.AppContextInstallMessage,
-			})
-		}
-	} else {
-		return pkgerrors.Errorf("InstallApp Failed - Could not get InstallAppClient: %v", grpc.RsyncName)
+	err = installappclient.InvokeInstallApp(ctxVal.(string))
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -188,12 +163,16 @@ func (v *SchedulerClient) ApplyNetworkIntents(clusterProvider, cluster string) e
 
 // Terminate Network Intents associated with a cluster
 func (v *SchedulerClient) TerminateNetworkIntents(clusterProvider, cluster string) error {
-	context, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
+	context, ctxVal, err := clusterPkg.NewClusterClient().GetClusterContext(clusterProvider, cluster)
 	if err != nil {
 		return pkgerrors.Wrapf(err, "Error finding AppContext for cluster: %v, %v", clusterProvider, cluster)
 	}
 
-	// TODO: call resource synchronizer to terminate the CRs in the cluster
+	// call resource synchronizer to terminate the CRs in the cluster
+	err = installappclient.InvokeUninstallApp(ctxVal)
+	if err != nil {
+		return err
+	}
 
 	// remove the app context
 	cleanuperr := context.DeleteCompositeApp()
diff --git a/src/orchestrator/pkg/grpc/installappclient/client.go b/src/orchestrator/pkg/grpc/installappclient/client.go
index 4c652a84..0e9141a6 100644
--- a/src/orchestrator/pkg/grpc/installappclient/client.go
+++ b/src/orchestrator/pkg/grpc/installappclient/client.go
@@ -19,10 +19,32 @@ import (
 
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/rpc"
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/module/controller"
 	installpb "github.com/onap/multicloud-k8s/src/rsync/pkg/grpc/installapp"
 	pkgerrors "github.com/pkg/errors"
 )
 
+const rsyncName = "rsync"
+
+// InitRsyncClient initializes connctions to the Resource Synchronizer service
+func initRsyncClient() bool {
+	client := controller.NewControllerClient()
+
+	vals, _ := client.GetControllers()
+	found := false
+	for _, v := range vals {
+		if v.Metadata.Name == rsyncName {
+			log.Info("Initializing RPC connection to resource synchronizer", log.Fields{
+				"Controller": v.Metadata.Name,
+			})
+			rpc.UpdateRpcConn(v.Metadata.Name, v.Spec.Host, v.Spec.Port)
+			found = true
+			break
+		}
+	}
+	return found
+}
+
 // InvokeInstallApp will make the grpc call to the resource synchronizer
 // or rsync controller.
 // rsync will deply the resources in the app context to the clusters as
@@ -34,7 +56,11 @@ func InvokeInstallApp(appContextId string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	conn := rpc.GetRpcConn("rsync")
+	conn := rpc.GetRpcConn(rsyncName)
+	if conn == nil {
+		initRsyncClient()
+		conn = rpc.GetRpcConn(rsyncName)
+	}
 
 	if conn != nil {
 		rpcClient = installpb.NewInstallappClient(conn)
@@ -64,3 +90,41 @@ func InvokeInstallApp(appContextId string) error {
 	}
 	return err
 }
+
+func InvokeUninstallApp(appContextId string) error {
+	var err error
+	var rpcClient installpb.InstallappClient
+	var uninstallRes *installpb.UninstallAppResponse
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	conn := rpc.GetRpcConn("rsync")
+
+	if conn != nil {
+		rpcClient = installpb.NewInstallappClient(conn)
+		uninstallReq := new(installpb.UninstallAppRequest)
+		uninstallReq.AppContext = appContextId
+		uninstallRes, err = rpcClient.UninstallApp(ctx, uninstallReq)
+		if err == nil {
+			log.Info("Response from UninstappApp GRPC call", log.Fields{
+				"Succeeded": uninstallRes.AppContextUninstalled,
+				"Message":   uninstallRes.AppContextUninstallMessage,
+			})
+		}
+	} else {
+		return pkgerrors.Errorf("UninstallApp Failed - Could not get InstallAppClient: %v", "rsync")
+	}
+
+	if err == nil {
+		if uninstallRes.AppContextUninstalled {
+			log.Info("UninstallApp Success", log.Fields{
+				"AppContext": appContextId,
+				"Message":    uninstallRes.AppContextUninstallMessage,
+			})
+			return nil
+		} else {
+			return pkgerrors.Errorf("UninstallApp Failed: %v", uninstallRes.AppContextUninstallMessage)
+		}
+	}
+	return err
+}
diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index 3412a034..0decb68f 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -62,7 +62,7 @@ type OverrideValues struct {
 type DeploymentIntentGroupManager interface {
 	CreateDeploymentIntentGroup(d DeploymentIntentGroup, p string, ca string, v string) (DeploymentIntentGroup, error)
 	GetDeploymentIntentGroup(di string, p string, ca string, v string) (DeploymentIntentGroup, error)
-	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error)
+	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error)
 	DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error
 }
 
@@ -165,7 +165,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroup(di string, p stri
 }
 
 // GetDeploymentIntentGroup returns the DeploymentIntentGroup with a given name, project, compositeApp and version of compositeApp
-func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) {
+func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, string, error) {
 
 	key := DeploymentIntentGroupKey{
 		Name:         di,
@@ -176,7 +176,7 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 
 	result, err := db.DBconn.Find(c.storeName, key, c.tagContext)
 	if err != nil {
-		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
+		return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
 	}
 
 	if result != nil {
@@ -184,12 +184,12 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string,
 		var cc appcontext.AppContext
 		_, err = cc.LoadAppContext(ctxVal)
 		if err != nil {
-			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
+			return appcontext.AppContext{}, "", pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
 		}
-		return cc, nil
+		return cc, ctxVal, nil
 	}
 
-	return appcontext.AppContext{}, pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
+	return appcontext.AppContext{}, "", pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
 }
 
 // DeleteDeploymentIntentGroup deletes a DeploymentIntentGroup
@@ -200,10 +200,14 @@ func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p s
 		CompositeApp: ca,
 		Version:      v,
 	}
+	_, _, err := c.GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup must be terminated before it can be deleted "+di)
+	}
 
-	err := db.DBconn.Remove(c.storeName, k)
+	err = db.DBconn.Remove(c.storeName, k)
 	if err != nil {
-		return pkgerrors.Wrap(err, "Delete DeploymentIntentGroup entry;")
+		return pkgerrors.Wrap(err, "Error deleting DeploymentIntentGroup entry")
 	}
 	return nil
 
diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go
index 9432e4b9..1f2e1117 100644
--- a/src/orchestrator/pkg/module/instantiation.go
+++ b/src/orchestrator/pkg/module/instantiation.go
@@ -188,6 +188,12 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	if err != nil {
 		return pkgerrors.Wrap(err, "Not finding the deploymentIntentGroup")
 	}
+
+	_, _, err = NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err == nil {
+		return pkgerrors.Errorf("DeploymentIntentGroup has already been instantiated: " + di)
+	}
+
 	rName := dIGrp.Spec.Version //rName is releaseName
 	overrideValues := dIGrp.Spec.OverrideValuesObj
 	cp := dIGrp.Spec.Profile
@@ -331,7 +337,7 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	// END: Scheduler code
 
 	// BEGIN : Rsync code
-	err = callRsync(ctxval)
+	err = callRsyncInstall(ctxval)
 	if err != nil {
 		return err
 	}
@@ -348,9 +354,9 @@ the deployment, which is made available in the appcontext.
 */
 func (c InstantiationClient) Status(p string, ca string, v string, di string) (StatusData, error) {
 
-	ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, _, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
 	if err != nil {
-		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found: "+di)
 	}
 
 	// Get all apps in this composite app
@@ -409,15 +415,32 @@ DeploymentIntentName and calls rsync to terminate.
 */
 func (c InstantiationClient) Terminate(p string, ca string, v string, di string) error {
 
-	//ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
-	_, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	ac, ctxval, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return pkgerrors.Wrap(err, "DeploymentIntentGroup has no app context: "+di)
+	}
+
+	err = callRsyncUninstall(ctxval)
+	if err != nil {
+		return err
+	}
+
+	err = ac.DeleteCompositeApp()
 	if err != nil {
-		return pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+		return pkgerrors.Wrap(err, "Error deleting the app context for DeploymentIntentGroup: "+di)
 	}
 
-	// TODO - make call to rsync to terminate the composite app deployment
-	//        will leave the appcontext in place for clean up later
-	//        so monitoring status can be performed
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
+	}
+
+	err = db.DBconn.RemoveTag(c.db.storeName, key, c.db.tagContext)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Error removing the app context tag from DeploymentIntentGroup: "+di)
+	}
 
 	return nil
 }
diff --git a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
index 3d9d851c..184d6972 100644
--- a/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
+++ b/src/orchestrator/pkg/module/instantiation_scheduler_helper.go
@@ -192,9 +192,9 @@ func callGrpcForControllerList(cl []controller.Controller, mc map[string]string,
 }
 
 /*
-callRsync method shall take in the app context id and invokes the rsync service via grpc
+callRsyncInstall method shall take in the app context id and invokes the rsync service via grpc
 */
-func callRsync(contextid interface{}) error {
+func callRsyncInstall(contextid interface{}) error {
 	appContextID := fmt.Sprintf("%v", contextid)
 	err := rsyncclient.InvokeInstallApp(appContextID)
 	if err != nil {
@@ -204,6 +204,18 @@ func callRsync(contextid interface{}) error {
 }
 
 /*
+callRsyncUninstall method shall take in the app context id and invokes the rsync service via grpc
+*/
+func callRsyncUninstall(contextid interface{}) error {
+	appContextID := fmt.Sprintf("%v", contextid)
+	err := rsyncclient.InvokeUninstallApp(appContextID)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+/*
 deleteExtraClusters method shall delete the extra cluster handles for each AnyOf cluster present in the etcd after the grpc call for context updation.
 */
 func deleteExtraClusters(apps []App, ct appcontext.AppContext) error {