Adds composite app status update and query

This patch provides the basic framework for supporting
monitoring of composite application resources in clusters.

1. Updates to the monitor files for use with v2.
2. Invokes the Watcher process per cluster/app when the
   app is instantiated.
3. Adds a ResourceBundleState CR resource to the cluster/app
   so that monitor will be able to update status to it.
4. Watcher updates appropriate appcontext status object
   when updates are made in clusters by monitor
5. Update appcontext library to define a status handle
   and object at the app/cluster level
6. Labels resources with an appropriate tracking label
   to coordinate with the ResourceBundleState CR

Issue-ID: MULTICLOUD-1042
Signed-off-by: Eric Multanen <eric.w.multanen@intel.com>
Change-Id: If007c1fd86ca7a65bb941d1776cfd2d3afed766b

6 files changed, 263 insertions, 11 deletions

diff --git a/src/orchestrator/pkg/appcontext/appcontext.go b/src/orchestrator/pkg/appcontext/appcontext.go
index a847ae32..cdf23bfa 100644
--- a/src/orchestrator/pkg/appcontext/appcontext.go
+++ b/src/orchestrator/pkg/appcontext/appcontext.go
@@ -18,10 +18,11 @@ package appcontext
 
 import (
 	"fmt"
+	"strings"
+
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/rtcontext"
 	pkgerrors "github.com/pkg/errors"
-	"strings"
 )
 
 // metaPrefix used for denoting clusterMeta level
@@ -413,6 +414,59 @@ func (ac *AppContext) GetResourceInstruction(appname string, clustername string,
 	return v, nil
 }
 
+//AddStatus for holding status of all resources under app and cluster
+// handle should be a cluster handle
+func (ac *AppContext) AddStatus(handle interface{}, value interface{}) (interface{}, error) {
+	h, err := ac.rtc.RtcAddStatus(handle, value)
+	if err != nil {
+		return nil, err
+	}
+	log.Info(":: Added status handle ::", log.Fields{"StatusHandler": h})
+
+	return h, nil
+}
+
+//DeleteStatus for the given the handle
+func (ac *AppContext) DeleteStatus(handle interface{}) error {
+	err := ac.rtc.RtcDeletePair(handle)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+//Return the handle for status for a given app and cluster
+func (ac *AppContext) GetStatusHandle(appname string, clustername string) (interface{}, error) {
+	if appname == "" {
+		return nil, pkgerrors.Errorf("Not a valid run time context app name")
+	}
+	if clustername == "" {
+		return nil, pkgerrors.Errorf("Not a valid run time context cluster name")
+	}
+
+	rh, err := ac.rtc.RtcGet()
+	if err != nil {
+		return nil, err
+	}
+
+	acrh := fmt.Sprintf("%v", rh) + "app/" + appname + "/cluster/" + clustername + "/status/"
+	hs, err := ac.rtc.RtcGetHandles(acrh)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range hs {
+		if v == acrh {
+			return v, nil
+		}
+	}
+	return nil, pkgerrors.Errorf("No handle was found for the given resource")
+}
+
+//UpdateStatusValue updates the status value with the given handle
+func (ac *AppContext) UpdateStatusValue(handle interface{}, value interface{}) error {
+	return ac.rtc.RtcUpdateValue(handle, value)
+}
+
 //Return all the handles under the composite app
 func (ac *AppContext) GetAllHandles(handle interface{}) ([]interface{}, error) {
 	hs, err := ac.rtc.RtcGetHandles(handle)
diff --git a/src/orchestrator/pkg/appcontext/appcontext_test.go b/src/orchestrator/pkg/appcontext/appcontext_test.go
index 05c73703..92c43113 100644
--- a/src/orchestrator/pkg/appcontext/appcontext_test.go
+++ b/src/orchestrator/pkg/appcontext/appcontext_test.go
@@ -145,6 +145,10 @@ func (c *MockRunTimeContext) RtcUpdateValue(handle interface{}, value interface{
 	return c.Err
 }
 
+func (rtc *MockRunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) {
+	return nil, nil
+}
+
 func TestCreateCompositeApp(t *testing.T) {
 	var ac = AppContext{}
 	testCases := []struct {
diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index 16a14c7b..3412a034 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"reflect"
 
+	appcontext "github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 
 	pkgerrors "github.com/pkg/errors"
@@ -61,6 +62,7 @@ type OverrideValues struct {
 type DeploymentIntentGroupManager interface {
 	CreateDeploymentIntentGroup(d DeploymentIntentGroup, p string, ca string, v string) (DeploymentIntentGroup, error)
 	GetDeploymentIntentGroup(di string, p string, ca string, v string) (DeploymentIntentGroup, error)
+	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error)
 	DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error
 }
 
@@ -86,6 +88,7 @@ func (dk DeploymentIntentGroupKey) String() string {
 type DeploymentIntentGroupClient struct {
 	storeName   string
 	tagMetaData string
+	tagContext  string
 }
 
 // NewDeploymentIntentGroupClient return an instance of DeploymentIntentGroupClient which implements DeploymentIntentGroupManager
@@ -93,6 +96,7 @@ func NewDeploymentIntentGroupClient() *DeploymentIntentGroupClient {
 	return &DeploymentIntentGroupClient{
 		storeName:   "orchestrator",
 		tagMetaData: "deploymentintentgroupmetadata",
+		tagContext:  "contextid",
 	}
 }
 
@@ -160,6 +164,34 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroup(di string, p stri
 
 }
 
+// GetDeploymentIntentGroup returns the DeploymentIntentGroup with a given name, project, compositeApp and version of compositeApp
+func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) {
+
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
+	}
+
+	result, err := db.DBconn.Find(c.storeName, key, c.tagContext)
+	if err != nil {
+		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
+	}
+
+	if result != nil {
+		ctxVal := string(result[0])
+		var cc appcontext.AppContext
+		_, err = cc.LoadAppContext(ctxVal)
+		if err != nil {
+			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
+		}
+		return cc, nil
+	}
+
+	return appcontext.AppContext{}, pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
+}
+
 // DeleteDeploymentIntentGroup deletes a DeploymentIntentGroup
 func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error {
 	k := DeploymentIntentGroupKey{
diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go
index 043b80f2..9432e4b9 100644
--- a/src/orchestrator/pkg/module/instantiation.go
+++ b/src/orchestrator/pkg/module/instantiation.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
 	gpic "github.com/onap/multicloud-k8s/src/orchestrator/pkg/gpic"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
@@ -42,17 +43,25 @@ type InstantiationClient struct {
 	db InstantiationClientDbInfo
 }
 
+type ClusterAppStatus struct {
+	Cluster string
+	App     string
+	Status  rb.ResourceBundleStatus
+}
+
+type StatusData struct {
+	Data []ClusterAppStatus
+}
+
 /*
 InstantiationKey used in storing the contextid in the momgodb
 It consists of
-GenericPlacementIntentName,
 ProjectName,
 CompositeAppName,
 CompositeAppVersion,
 DeploymentIntentGroup
 */
 type InstantiationKey struct {
-	IntentName            string
 	Project               string
 	CompositeApp          string
 	Version               string
@@ -64,6 +73,8 @@ type InstantiationKey struct {
 type InstantiationManager interface {
 	//ApproveInstantiation(p string, ca string, v string, di string) (error)
 	Instantiate(p string, ca string, v string, di string) error
+	Status(p string, ca string, v string, di string) (StatusData, error)
+	Terminate(p string, ca string, v string, di string) error
 }
 
 // InstantiationClientDbInfo consists of storeName and tagContext
@@ -229,6 +240,12 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 			return pkgerrors.Wrapf(err, "Unable to get the resources for app :: %s", eachApp.Metadata.Name)
 		}
 
+		statusResource, err := getStatusResource(ctxval.(string), eachApp.Metadata.Name)
+		if err != nil {
+			return pkgerrors.Wrapf(err, "Unable to generate the status resource for app :: %s", eachApp.Metadata.Name)
+		}
+		resources = append(resources, statusResource)
+
 		specData, err := NewAppIntentClient().GetAllIntentsByApp(eachApp.Metadata.Name, p, ca, v, gIntent)
 		if err != nil {
 			return pkgerrors.Wrap(err, "Unable to get the intents for app")
@@ -269,12 +286,11 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	//END: storing into etcd
 
 	// BEGIN:: save the context in the orchestrator db record
-	key := InstantiationKey{
-		IntentName:            gIntent,
-		Project:               p,
-		CompositeApp:          ca,
-		Version:               v,
-		DeploymentIntentGroup: di,
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
 	}
 
 	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagContext, ctxval)
@@ -324,3 +340,84 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	log.Info(":: Done with instantiation... ::", log.Fields{"CompositeAppName": ca})
 	return err
 }
+
+/*
+Status takes in projectName, compositeAppName, compositeAppVersion,
+DeploymentIntentName. This method is responsible obtaining the status of
+the deployment, which is made available in the appcontext.
+*/
+func (c InstantiationClient) Status(p string, ca string, v string, di string) (StatusData, error) {
+
+	ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+	}
+
+	// Get all apps in this composite app
+	allApps, err := NewAppClient().GetApps(p, ca, v)
+	if err != nil {
+		return StatusData{}, pkgerrors.Wrap(err, "Not finding the apps")
+	}
+
+	var diStatus StatusData
+	diStatus.Data = make([]ClusterAppStatus, 0)
+
+	// Loop through each app and get the status data for each cluster in the app
+	for _, app := range allApps {
+		// Get the clusters in the appcontext for this app
+		clusters, err := ac.GetClusterNames(app.Metadata.Name)
+		if err != nil {
+			log.Info(":: No clusters for app ::", log.Fields{"AppName": app.Metadata.Name})
+			continue
+		}
+
+		for _, cluster := range clusters {
+			handle, err := ac.GetStatusHandle(app.Metadata.Name, cluster)
+			if err != nil {
+				log.Info(":: No status handle for cluster, app ::",
+					log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err})
+				continue
+			}
+			statusValue, err := ac.GetValue(handle)
+			if err != nil {
+				log.Info(":: No status value for cluster, app ::",
+					log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err})
+				continue
+			}
+			log.Info(":: STATUS VALUE ::", log.Fields{"statusValue": statusValue})
+			var statusData ClusterAppStatus
+			err = json.Unmarshal([]byte(statusValue.(string)), &statusData.Status)
+			if err != nil {
+				log.Info(":: Error unmarshaling status value for cluster, app ::",
+					log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err})
+				continue
+			}
+			statusData.Cluster = cluster
+			statusData.App = app.Metadata.Name
+			log.Info(":: STATUS DATA ::", log.Fields{"status": statusData})
+
+			diStatus.Data = append(diStatus.Data, statusData)
+		}
+	}
+
+	return diStatus, nil
+}
+
+/*
+Terminate takes in projectName, compositeAppName, compositeAppVersion,
+DeploymentIntentName and calls rsync to terminate.
+*/
+func (c InstantiationClient) Terminate(p string, ca string, v string, di string) error {
+
+	//ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	_, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+	}
+
+	// TODO - make call to rsync to terminate the composite app deployment
+	//        will leave the appcontext in place for clean up later
+	//        so monitoring status can be performed
+
+	return nil
+}
diff --git a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
index 43ddd6df..e6e2bf30 100644
--- a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
+++ b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
@@ -25,12 +25,16 @@ import (
 	"encoding/json"
 	"io/ioutil"
 
+	jyaml "github.com/ghodss/yaml"
+
+	rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
 	gpic "github.com/onap/multicloud-k8s/src/orchestrator/pkg/gpic"
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/utils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/utils/helm"
 	pkgerrors "github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // resource consists of name of reource
@@ -90,6 +94,45 @@ func getResources(st []helm.KubernetesResourceTemplate) ([]resource, error) {
 	return resources, nil
 }
 
+// addStatusResource adds a status monitoring resource to the app
+// which consists of name(name+kind) and content
+func getStatusResource(id, app string) (resource, error) {
+
+	var statusCr rb.ResourceBundleState
+
+	label := id + "-" + app
+	name := app + "-" + id
+
+	statusCr.TypeMeta.APIVersion = "k8splugin.io/v1alpha1"
+	statusCr.TypeMeta.Kind = "ResourceBundleState"
+	statusCr.SetName(name)
+
+	labels := make(map[string]string)
+	labels["emco/deployment-id"] = label
+	statusCr.SetLabels(labels)
+
+	labelSelector, err := metav1.ParseToLabelSelector("emco/deployment-id = " + label)
+	if err != nil {
+		log.Info(":: ERROR Parsing Label Selector ::", log.Fields{"Error": err})
+	} else {
+		statusCr.Spec.Selector = labelSelector
+	}
+
+	// Marshaling to json then convert to yaml works better than marshaling to yaml
+	// The 'apiVersion' attribute was marshaling to 'apiversion'
+	//	y, _ := yaml.Marshal(&statusCr)
+	j, _ := json.Marshal(&statusCr)
+	y, _ := jyaml.JSONToYAML(j)
+	log.Info(":: RESULTING STATUS CR ::", log.Fields{"StatusCR": y})
+
+	statusResource := resource{
+		name:        name + "+" + "ResourceBundleState",
+		filecontent: string(y),
+	}
+
+	return statusResource, nil
+}
+
 func addResourcesToCluster(ct appcontext.AppContext, ch interface{}, resources []resource) error {
 
 	var resOrderInstr struct {
diff --git a/src/orchestrator/pkg/rtcontext/rtcontext.go b/src/orchestrator/pkg/rtcontext/rtcontext.go
index 432c5d87..f3905eb0 100644
--- a/src/orchestrator/pkg/rtcontext/rtcontext.go
+++ b/src/orchestrator/pkg/rtcontext/rtcontext.go
@@ -18,11 +18,12 @@ package rtcontext
 
 import (
 	"fmt"
-	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/contextdb"
-	pkgerrors "github.com/pkg/errors"
 	"math/rand"
 	"strings"
 	"time"
+
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/contextdb"
+	pkgerrors "github.com/pkg/errors"
 )
 
 const maxrand = 0x7fffffffffffffff
@@ -40,6 +41,7 @@ type Rtcontext interface {
 	RtcAddMeta(meta interface{}) error
 	RtcGet() (interface{}, error)
 	RtcAddLevel(handle interface{}, level string, value string) (interface{}, error)
+	RtcAddStatus(handle interface{}, value interface{}) (interface{}, error)
 	RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error)
 	RtcAddInstruction(handle interface{}, level string, insttype string, value interface{}) (interface{}, error)
 	RtcDeletePair(handle interface{}) error
@@ -201,6 +203,26 @@ func (rtc *RunTimeContext) RtcAddOneLevel(pl interface{}, level string, value in
 	return (interface{})(key), nil
 }
 
+// Add status under the given level and return new handle
+func (rtc *RunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) {
+
+	str := fmt.Sprintf("%v", handle)
+	sid := fmt.Sprintf("%v", rtc.cid)
+	if !strings.HasPrefix(str, sid) {
+		return nil, pkgerrors.Errorf("Not a valid run time context handle")
+	}
+	if value == nil {
+		return nil, pkgerrors.Errorf("Not a valid run time context resource value")
+	}
+
+	k := str + "status" + "/"
+	err := contextdb.Db.Put(k, value)
+	if err != nil {
+		return nil, pkgerrors.Errorf("Error adding run time context status: %s", err.Error())
+	}
+	return (interface{})(k), nil
+}
+
 // Add a resource under the given level and return new handle
 func (rtc *RunTimeContext) RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error) {