Adds composite app status update and query

This patch provides the basic framework for supporting
monitoring of composite application resources in clusters.

1. Updates to the monitor files for use with v2.
2. Invokes the Watcher process per cluster/app when the
   app is instantiated.
3. Adds a ResourceBundleState CR resource to the cluster/app
   so that monitor will be able to update status to it.
4. Watcher updates appropriate appcontext status object
   when updates are made in clusters by monitor
5. Update appcontext library to define a status handle
   and object at the app/cluster level
6. Labels resources with an appropriate tracking label
   to coordinate with the ResourceBundleState CR

Issue-ID: MULTICLOUD-1042
Signed-off-by: Eric Multanen <eric.w.multanen@intel.com>
Change-Id: If007c1fd86ca7a65bb941d1776cfd2d3afed766b

9 files changed, 314 insertions, 23 deletions

diff --git a/src/orchestrator/api/api.go b/src/orchestrator/api/api.go
index 2470a1be..5abbb96d 100644
--- a/src/orchestrator/api/api.go
+++ b/src/orchestrator/api/api.go
@@ -180,6 +180,8 @@ func NewRouter(projectClient moduleLib.ProjectManager,
 	}
 
 	router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/deployment-intent-groups/{deployment-intent-group-name}/instantiate", instantiationHandler.instantiateHandler).Methods("POST")
+	router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/deployment-intent-groups/{deployment-intent-group-name}/terminate", instantiationHandler.terminateHandler).Methods("POST")
+	router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/deployment-intent-groups/{deployment-intent-group-name}/status", instantiationHandler.statusHandler).Methods("GET")
 
 	return router
 }
diff --git a/src/orchestrator/api/instantiation_handler.go b/src/orchestrator/api/instantiation_handler.go
index c95785f2..ce50e5b8 100644
--- a/src/orchestrator/api/instantiation_handler.go
+++ b/src/orchestrator/api/instantiation_handler.go
@@ -17,9 +17,11 @@
 package api
 
 import (
+	"encoding/json"
+	"net/http"
+
 	"github.com/gorilla/mux"
 	moduleLib "github.com/onap/multicloud-k8s/src/orchestrator/pkg/module"
-	"net/http"
 )
 
 /* Used to store backend implementation objects
@@ -45,3 +47,45 @@ func (h instantiationHandler) instantiateHandler(w http.ResponseWriter, r *http.
 	w.WriteHeader(http.StatusAccepted)
 
 }
+
+func (h instantiationHandler) terminateHandler(w http.ResponseWriter, r *http.Request) {
+
+	vars := mux.Vars(r)
+	p := vars["project-name"]
+	ca := vars["composite-app-name"]
+	v := vars["composite-app-version"]
+	di := vars["deployment-intent-group-name"]
+
+	iErr := h.client.Terminate(p, ca, v, di)
+	if iErr != nil {
+		http.Error(w, iErr.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(http.StatusAccepted)
+
+}
+
+func (h instantiationHandler) statusHandler(w http.ResponseWriter, r *http.Request) {
+
+	vars := mux.Vars(r)
+	p := vars["project-name"]
+	ca := vars["composite-app-name"]
+	v := vars["composite-app-version"]
+	di := vars["deployment-intent-group-name"]
+
+	status, iErr := h.client.Status(p, ca, v, di)
+	if iErr != nil {
+		http.Error(w, iErr.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	iErr = json.NewEncoder(w).Encode(status)
+	if iErr != nil {
+		http.Error(w, iErr.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(http.StatusAccepted)
+
+}
diff --git a/src/orchestrator/go.mod b/src/orchestrator/go.mod
index 223dc068..3f14f00b 100644
--- a/src/orchestrator/go.mod
+++ b/src/orchestrator/go.mod
@@ -2,17 +2,12 @@ module github.com/onap/multicloud-k8s/src/orchestrator
 
 require (
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
-	github.com/Masterminds/goutils v1.1.0 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1
 	github.com/coreos/etcd v3.3.12+incompatible
-	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
 	github.com/docker/docker v1.13.1 // indirect
-	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
-	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/ghodss/yaml v1.0.0
-	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/golang/protobuf v1.4.1
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.3
@@ -21,34 +16,32 @@ require (
 	github.com/lib/pq v1.6.0 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/onap/multicloud-k8s/src/clm v0.0.0-00010101000000-000000000000
+	github.com/onap/multicloud-k8s/src/monitor v0.0.0-20200630152613-7c20f73e7c5d
 	github.com/onap/multicloud-k8s/src/ncm v0.0.0-20200515060444-c77850a75eee
+	github.com/onap/multicloud-k8s/src/rsync v0.0.0-20200630152613-7c20f73e7c5d
 	github.com/pkg/errors v0.8.1
 	github.com/rubenv/sql-migrate v0.0.0-20200429072036-ae26b214fa43 // indirect
 	github.com/russross/blackfriday v1.5.2
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v1.0.0 // indirect
-	github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 // indirect
 	go.etcd.io/etcd v3.3.12+incompatible
 	go.mongodb.org/mongo-driver v1.0.0
 	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
 	google.golang.org/grpc v1.27.1
 	google.golang.org/protobuf v1.24.0
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
+	gopkg.in/yaml.v2 v2.2.8
 	gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86
-	k8s.io/apiextensions-apiserver v0.0.0-00010101000000-000000000000 // indirect
 	k8s.io/apimachinery v0.0.0-20190831074630-461753078381
-	k8s.io/apiserver v0.0.0-00010101000000-000000000000 // indirect
-	k8s.io/cli-runtime v0.0.0-00010101000000-000000000000 // indirect
 	k8s.io/cloud-provider v0.0.0-00010101000000-000000000000 // indirect
 	k8s.io/helm v2.14.3+incompatible
 	sigs.k8s.io/kustomize v2.0.3+incompatible // indirect
-	vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect
 )
 
 replace (
 	github.com/onap/multicloud-k8s/src/clm => ../clm
+	github.com/onap/multicloud-k8s/src/monitor => ../monitor
 	k8s.io/api => k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
diff --git a/src/orchestrator/pkg/appcontext/appcontext.go b/src/orchestrator/pkg/appcontext/appcontext.go
index a847ae32..cdf23bfa 100644
--- a/src/orchestrator/pkg/appcontext/appcontext.go
+++ b/src/orchestrator/pkg/appcontext/appcontext.go
@@ -18,10 +18,11 @@ package appcontext
 
 import (
 	"fmt"
+	"strings"
+
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/rtcontext"
 	pkgerrors "github.com/pkg/errors"
-	"strings"
 )
 
 // metaPrefix used for denoting clusterMeta level
@@ -413,6 +414,59 @@ func (ac *AppContext) GetResourceInstruction(appname string, clustername string,
 	return v, nil
 }
 
+//AddStatus for holding status of all resources under app and cluster
+// handle should be a cluster handle
+func (ac *AppContext) AddStatus(handle interface{}, value interface{}) (interface{}, error) {
+	h, err := ac.rtc.RtcAddStatus(handle, value)
+	if err != nil {
+		return nil, err
+	}
+	log.Info(":: Added status handle ::", log.Fields{"StatusHandler": h})
+
+	return h, nil
+}
+
+//DeleteStatus for the given the handle
+func (ac *AppContext) DeleteStatus(handle interface{}) error {
+	err := ac.rtc.RtcDeletePair(handle)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+//Return the handle for status for a given app and cluster
+func (ac *AppContext) GetStatusHandle(appname string, clustername string) (interface{}, error) {
+	if appname == "" {
+		return nil, pkgerrors.Errorf("Not a valid run time context app name")
+	}
+	if clustername == "" {
+		return nil, pkgerrors.Errorf("Not a valid run time context cluster name")
+	}
+
+	rh, err := ac.rtc.RtcGet()
+	if err != nil {
+		return nil, err
+	}
+
+	acrh := fmt.Sprintf("%v", rh) + "app/" + appname + "/cluster/" + clustername + "/status/"
+	hs, err := ac.rtc.RtcGetHandles(acrh)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range hs {
+		if v == acrh {
+			return v, nil
+		}
+	}
+	return nil, pkgerrors.Errorf("No handle was found for the given resource")
+}
+
+//UpdateStatusValue updates the status value with the given handle
+func (ac *AppContext) UpdateStatusValue(handle interface{}, value interface{}) error {
+	return ac.rtc.RtcUpdateValue(handle, value)
+}
+
 //Return all the handles under the composite app
 func (ac *AppContext) GetAllHandles(handle interface{}) ([]interface{}, error) {
 	hs, err := ac.rtc.RtcGetHandles(handle)
diff --git a/src/orchestrator/pkg/appcontext/appcontext_test.go b/src/orchestrator/pkg/appcontext/appcontext_test.go
index 05c73703..92c43113 100644
--- a/src/orchestrator/pkg/appcontext/appcontext_test.go
+++ b/src/orchestrator/pkg/appcontext/appcontext_test.go
@@ -145,6 +145,10 @@ func (c *MockRunTimeContext) RtcUpdateValue(handle interface{}, value interface{
 	return c.Err
 }
 
+func (rtc *MockRunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) {
+	return nil, nil
+}
+
 func TestCreateCompositeApp(t *testing.T) {
 	var ac = AppContext{}
 	testCases := []struct {
diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index 16a14c7b..3412a034 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"reflect"
 
+	appcontext "github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 
 	pkgerrors "github.com/pkg/errors"
@@ -61,6 +62,7 @@ type OverrideValues struct {
 type DeploymentIntentGroupManager interface {
 	CreateDeploymentIntentGroup(d DeploymentIntentGroup, p string, ca string, v string) (DeploymentIntentGroup, error)
 	GetDeploymentIntentGroup(di string, p string, ca string, v string) (DeploymentIntentGroup, error)
+	GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error)
 	DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error
 }
 
@@ -86,6 +88,7 @@ func (dk DeploymentIntentGroupKey) String() string {
 type DeploymentIntentGroupClient struct {
 	storeName   string
 	tagMetaData string
+	tagContext  string
 }
 
 // NewDeploymentIntentGroupClient return an instance of DeploymentIntentGroupClient which implements DeploymentIntentGroupManager
@@ -93,6 +96,7 @@ func NewDeploymentIntentGroupClient() *DeploymentIntentGroupClient {
 	return &DeploymentIntentGroupClient{
 		storeName:   "orchestrator",
 		tagMetaData: "deploymentintentgroupmetadata",
+		tagContext:  "contextid",
 	}
 }
 
@@ -160,6 +164,34 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroup(di string, p stri
 
 }
 
+// GetDeploymentIntentGroup returns the DeploymentIntentGroup with a given name, project, compositeApp and version of compositeApp
+func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) {
+
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
+	}
+
+	result, err := db.DBconn.Find(c.storeName, key, c.tagContext)
+	if err != nil {
+		return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error")
+	}
+
+	if result != nil {
+		ctxVal := string(result[0])
+		var cc appcontext.AppContext
+		_, err = cc.LoadAppContext(ctxVal)
+		if err != nil {
+			return appcontext.AppContext{}, pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext")
+		}
+		return cc, nil
+	}
+
+	return appcontext.AppContext{}, pkgerrors.New("Error getting DeploymentIntentGroup AppContext")
+}
+
 // DeleteDeploymentIntentGroup deletes a DeploymentIntentGroup
 func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error {
 	k := DeploymentIntentGroupKey{
diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go
index 043b80f2..9432e4b9 100644
--- a/src/orchestrator/pkg/module/instantiation.go
+++ b/src/orchestrator/pkg/module/instantiation.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
 	gpic "github.com/onap/multicloud-k8s/src/orchestrator/pkg/gpic"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
@@ -42,17 +43,25 @@ type InstantiationClient struct {
 	db InstantiationClientDbInfo
 }
 
+type ClusterAppStatus struct {
+	Cluster string
+	App     string
+	Status  rb.ResourceBundleStatus
+}
+
+type StatusData struct {
+	Data []ClusterAppStatus
+}
+
 /*
 InstantiationKey used in storing the contextid in the momgodb
 It consists of
-GenericPlacementIntentName,
 ProjectName,
 CompositeAppName,
 CompositeAppVersion,
 DeploymentIntentGroup
 */
 type InstantiationKey struct {
-	IntentName            string
 	Project               string
 	CompositeApp          string
 	Version               string
@@ -64,6 +73,8 @@ type InstantiationKey struct {
 type InstantiationManager interface {
 	//ApproveInstantiation(p string, ca string, v string, di string) (error)
 	Instantiate(p string, ca string, v string, di string) error
+	Status(p string, ca string, v string, di string) (StatusData, error)
+	Terminate(p string, ca string, v string, di string) error
 }
 
 // InstantiationClientDbInfo consists of storeName and tagContext
@@ -229,6 +240,12 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 			return pkgerrors.Wrapf(err, "Unable to get the resources for app :: %s", eachApp.Metadata.Name)
 		}
 
+		statusResource, err := getStatusResource(ctxval.(string), eachApp.Metadata.Name)
+		if err != nil {
+			return pkgerrors.Wrapf(err, "Unable to generate the status resource for app :: %s", eachApp.Metadata.Name)
+		}
+		resources = append(resources, statusResource)
+
 		specData, err := NewAppIntentClient().GetAllIntentsByApp(eachApp.Metadata.Name, p, ca, v, gIntent)
 		if err != nil {
 			return pkgerrors.Wrap(err, "Unable to get the intents for app")
@@ -269,12 +286,11 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	//END: storing into etcd
 
 	// BEGIN:: save the context in the orchestrator db record
-	key := InstantiationKey{
-		IntentName:            gIntent,
-		Project:               p,
-		CompositeApp:          ca,
-		Version:               v,
-		DeploymentIntentGroup: di,
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
 	}
 
 	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagContext, ctxval)
@@ -324,3 +340,84 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	log.Info(":: Done with instantiation... ::", log.Fields{"CompositeAppName": ca})
 	return err
 }
+
+/*
+Status takes in projectName, compositeAppName, compositeAppVersion,
+DeploymentIntentName. This method is responsible obtaining the status of
+the deployment, which is made available in the appcontext.
+*/
+func (c InstantiationClient) Status(p string, ca string, v string, di string) (StatusData, error) {
+
+	ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+	}
+
+	// Get all apps in this composite app
+	allApps, err := NewAppClient().GetApps(p, ca, v)
+	if err != nil {
+		return StatusData{}, pkgerrors.Wrap(err, "Not finding the apps")
+	}
+
+	var diStatus StatusData
+	diStatus.Data = make([]ClusterAppStatus, 0)
+
+	// Loop through each app and get the status data for each cluster in the app
+	for _, app := range allApps {
+		// Get the clusters in the appcontext for this app
+		clusters, err := ac.GetClusterNames(app.Metadata.Name)
+		if err != nil {
+			log.Info(":: No clusters for app ::", log.Fields{"AppName": app.Metadata.Name})
+			continue
+		}
+
+		for _, cluster := range clusters {
+			handle, err := ac.GetStatusHandle(app.Metadata.Name, cluster)
+			if err != nil {
+				log.Info(":: No status handle for cluster, app ::",
+					log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err})
+				continue
+			}
+			statusValue, err := ac.GetValue(handle)
+			if err != nil {
+				log.Info(":: No status value for cluster, app ::",
+					log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err})
+				continue
+			}
+			log.Info(":: STATUS VALUE ::", log.Fields{"statusValue": statusValue})
+			var statusData ClusterAppStatus
+			err = json.Unmarshal([]byte(statusValue.(string)), &statusData.Status)
+			if err != nil {
+				log.Info(":: Error unmarshaling status value for cluster, app ::",
+					log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err})
+				continue
+			}
+			statusData.Cluster = cluster
+			statusData.App = app.Metadata.Name
+			log.Info(":: STATUS DATA ::", log.Fields{"status": statusData})
+
+			diStatus.Data = append(diStatus.Data, statusData)
+		}
+	}
+
+	return diStatus, nil
+}
+
+/*
+Terminate takes in projectName, compositeAppName, compositeAppVersion,
+DeploymentIntentName and calls rsync to terminate.
+*/
+func (c InstantiationClient) Terminate(p string, ca string, v string, di string) error {
+
+	//ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	_, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v)
+	if err != nil {
+		return pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di)
+	}
+
+	// TODO - make call to rsync to terminate the composite app deployment
+	//        will leave the appcontext in place for clean up later
+	//        so monitoring status can be performed
+
+	return nil
+}
diff --git a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
index 43ddd6df..e6e2bf30 100644
--- a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
+++ b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
@@ -25,12 +25,16 @@ import (
 	"encoding/json"
 	"io/ioutil"
 
+	jyaml "github.com/ghodss/yaml"
+
+	rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
 	gpic "github.com/onap/multicloud-k8s/src/orchestrator/pkg/gpic"
 	log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/utils"
 	"github.com/onap/multicloud-k8s/src/orchestrator/utils/helm"
 	pkgerrors "github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // resource consists of name of reource
@@ -90,6 +94,45 @@ func getResources(st []helm.KubernetesResourceTemplate) ([]resource, error) {
 	return resources, nil
 }
 
+// addStatusResource adds a status monitoring resource to the app
+// which consists of name(name+kind) and content
+func getStatusResource(id, app string) (resource, error) {
+
+	var statusCr rb.ResourceBundleState
+
+	label := id + "-" + app
+	name := app + "-" + id
+
+	statusCr.TypeMeta.APIVersion = "k8splugin.io/v1alpha1"
+	statusCr.TypeMeta.Kind = "ResourceBundleState"
+	statusCr.SetName(name)
+
+	labels := make(map[string]string)
+	labels["emco/deployment-id"] = label
+	statusCr.SetLabels(labels)
+
+	labelSelector, err := metav1.ParseToLabelSelector("emco/deployment-id = " + label)
+	if err != nil {
+		log.Info(":: ERROR Parsing Label Selector ::", log.Fields{"Error": err})
+	} else {
+		statusCr.Spec.Selector = labelSelector
+	}
+
+	// Marshaling to json then convert to yaml works better than marshaling to yaml
+	// The 'apiVersion' attribute was marshaling to 'apiversion'
+	//	y, _ := yaml.Marshal(&statusCr)
+	j, _ := json.Marshal(&statusCr)
+	y, _ := jyaml.JSONToYAML(j)
+	log.Info(":: RESULTING STATUS CR ::", log.Fields{"StatusCR": y})
+
+	statusResource := resource{
+		name:        name + "+" + "ResourceBundleState",
+		filecontent: string(y),
+	}
+
+	return statusResource, nil
+}
+
 func addResourcesToCluster(ct appcontext.AppContext, ch interface{}, resources []resource) error {
 
 	var resOrderInstr struct {
diff --git a/src/orchestrator/pkg/rtcontext/rtcontext.go b/src/orchestrator/pkg/rtcontext/rtcontext.go
index 432c5d87..f3905eb0 100644
--- a/src/orchestrator/pkg/rtcontext/rtcontext.go
+++ b/src/orchestrator/pkg/rtcontext/rtcontext.go
@@ -18,11 +18,12 @@ package rtcontext
 
 import (
 	"fmt"
-	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/contextdb"
-	pkgerrors "github.com/pkg/errors"
 	"math/rand"
 	"strings"
 	"time"
+
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/contextdb"
+	pkgerrors "github.com/pkg/errors"
 )
 
 const maxrand = 0x7fffffffffffffff
@@ -40,6 +41,7 @@ type Rtcontext interface {
 	RtcAddMeta(meta interface{}) error
 	RtcGet() (interface{}, error)
 	RtcAddLevel(handle interface{}, level string, value string) (interface{}, error)
+	RtcAddStatus(handle interface{}, value interface{}) (interface{}, error)
 	RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error)
 	RtcAddInstruction(handle interface{}, level string, insttype string, value interface{}) (interface{}, error)
 	RtcDeletePair(handle interface{}) error
@@ -201,6 +203,26 @@ func (rtc *RunTimeContext) RtcAddOneLevel(pl interface{}, level string, value in
 	return (interface{})(key), nil
 }
 
+// Add status under the given level and return new handle
+func (rtc *RunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) {
+
+	str := fmt.Sprintf("%v", handle)
+	sid := fmt.Sprintf("%v", rtc.cid)
+	if !strings.HasPrefix(str, sid) {
+		return nil, pkgerrors.Errorf("Not a valid run time context handle")
+	}
+	if value == nil {
+		return nil, pkgerrors.Errorf("Not a valid run time context resource value")
+	}
+
+	k := str + "status" + "/"
+	err := contextdb.Db.Put(k, value)
+	if err != nil {
+		return nil, pkgerrors.Errorf("Error adding run time context status: %s", err.Error())
+	}
+	return (interface{})(k), nil
+}
+
 // Add a resource under the given level and return new handle
 func (rtc *RunTimeContext) RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error) {