Add appcontext state, status and resource status

Add support in the AppContext for managing an AppContext
(composite app level) status value.
Also adds support for tracking rsync status at the resource
level.
A mechanism for tracking history at the controlling resource
level (i.e. DeploymentGroupIntnt or Cluster) is added, in part,
so that all AppContexts associated can be deleted when
the resource is eventually deleted.

Issue-ID: MULTICLOUD-1042
Change-Id: I3d0a9a97ea45ca11f9f873104476e4b67521e56a
Signed-off-by: Eric Multanen <eric.w.multanen@intel.com>

9 files changed, 321 insertions, 120 deletions

diff --git a/src/orchestrator/pkg/appcontext/appcontext.go b/src/orchestrator/pkg/appcontext/appcontext.go
index cdf23bfa..db2ba432 100644
--- a/src/orchestrator/pkg/appcontext/appcontext.go
+++ b/src/orchestrator/pkg/appcontext/appcontext.go
@@ -34,6 +34,35 @@ type AppContext struct {
 	rtc      rtcontext.Rtcontext
 }
 
+// AppContextStatus represents the current status of the appcontext
+//	Instantiating - instantiate has been invoked and is still in progress
+//	Instantiated - instantiate has completed
+//	PreTerminate - terminate has been invoked when in Instantiating status - need to clean up first
+//	Terminating - terminate has been invoked and is still in progress
+//	Terminated - terminate has completed
+//	Failed - the instantiate or terminate action has failed
+type AppContextStatus struct {
+	Status StatusValue
+}
+type StatusValue string
+type statuses struct {
+	Instantiating StatusValue
+	Instantiated  StatusValue
+	PreTerminate  StatusValue
+	Terminating   StatusValue
+	Terminated    StatusValue
+	Failed        StatusValue
+}
+
+var AppContextStatusEnum = &statuses{
+	Instantiating: "Instantiating",
+	Instantiated:  "Instantiated",
+	PreTerminate:  "PreTerminate",
+	Terminating:   "Terminating",
+	Terminated:    "Terminated",
+	Failed:        "Failed",
+}
+
 // CompositeAppMeta consists of projectName, CompositeAppName,
 // CompositeAppVersion, ReleaseName. This shall be used for
 // instantiation of a compositeApp
@@ -99,6 +128,22 @@ func (ac *AppContext) GetCompositeAppHandle() (interface{}, error) {
 	return h, nil
 }
 
+// GetLevelHandle returns the handle for the supplied level at the given handle.
+// For example, to get the handle of the 'status' level at a given handle.
+func (ac *AppContext) GetLevelHandle(handle interface{}, level string) (interface{}, error) {
+	ach := fmt.Sprintf("%v%v/", handle, level)
+	hs, err := ac.rtc.RtcGetHandles(ach)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range hs {
+		if v == ach {
+			return v, nil
+		}
+	}
+	return nil, pkgerrors.Errorf("No handle was found for level %v", level)
+}
+
 //Add app to the context under composite app
 func (ac *AppContext) AddApp(handle interface{}, appname string) (interface{}, error) {
 	h, err := ac.rtc.RtcAddLevel(handle, "app", appname)
@@ -307,16 +352,7 @@ func (ac *AppContext) AddResource(handle interface{}, resname string, value inte
 	return h, nil
 }
 
-//Delete resource given the handle
-func (ac *AppContext) DeleteResource(handle interface{}) error {
-	err := ac.rtc.RtcDeletePair(handle)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-//Return the hanlde for given app, cluster and resource name
+//Return the handle for given app, cluster and resource name
 func (ac *AppContext) GetResourceHandle(appname string, clustername string, resname string) (interface{}, error) {
 	if appname == "" {
 		return nil, pkgerrors.Errorf("Not a valid run time context app name")
@@ -343,11 +379,41 @@ func (ac *AppContext) GetResourceHandle(appname string, clustername string, resn
 	return nil, pkgerrors.Errorf("No handle was found for the given resource")
 }
 
-//Update the resource value usign the given handle
+//Update the resource value using the given handle
 func (ac *AppContext) UpdateResourceValue(handle interface{}, value interface{}) error {
 	return ac.rtc.RtcUpdateValue(handle, value)
 }
 
+//Return the handle for given app, cluster and resource name
+func (ac *AppContext) GetResourceStatusHandle(appname string, clustername string, resname string) (interface{}, error) {
+	if appname == "" {
+		return nil, pkgerrors.Errorf("Not a valid run time context app name")
+	}
+	if clustername == "" {
+		return nil, pkgerrors.Errorf("Not a valid run time context cluster name")
+	}
+	if resname == "" {
+		return nil, pkgerrors.Errorf("Not a valid run time context resource name")
+	}
+
+	rh, err := ac.rtc.RtcGet()
+	if err != nil {
+		return nil, err
+	}
+
+	acrh := fmt.Sprintf("%v", rh) + "app/" + appname + "/cluster/" + clustername + "/resource/" + resname + "/status/"
+	hs, err := ac.rtc.RtcGetHandles(acrh)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range hs {
+		if v == acrh {
+			return v, nil
+		}
+	}
+	return nil, pkgerrors.Errorf("No handle was found for the given resource")
+}
+
 //Add instruction under given handle and type
 func (ac *AppContext) AddInstruction(handle interface{}, level string, insttype string, value interface{}) (interface{}, error) {
 	if !(insttype == "order" || insttype == "dependency") {
@@ -364,7 +430,7 @@ func (ac *AppContext) AddInstruction(handle interface{}, level string, insttype
 	return h, nil
 }
 
-//Delete instruction under gievn handle
+//Delete instruction under given handle
 func (ac *AppContext) DeleteInstruction(handle interface{}) error {
 	err := ac.rtc.RtcDeletePair(handle)
 	if err != nil {
@@ -414,29 +480,20 @@ func (ac *AppContext) GetResourceInstruction(appname string, clustername string,
 	return v, nil
 }
 
-//AddStatus for holding status of all resources under app and cluster
-// handle should be a cluster handle
-func (ac *AppContext) AddStatus(handle interface{}, value interface{}) (interface{}, error) {
-	h, err := ac.rtc.RtcAddStatus(handle, value)
+// AddLevelValue for holding a state object at a given level
+// will make a handle with an appended "<level>/" to the key
+func (ac *AppContext) AddLevelValue(handle interface{}, level string, value interface{}) (interface{}, error) {
+	h, err := ac.rtc.RtcAddOneLevel(handle, level, value)
 	if err != nil {
 		return nil, err
 	}
-	log.Info(":: Added status handle ::", log.Fields{"StatusHandler": h})
+	log.Info(":: Added handle ::", log.Fields{"Handle": h})
 
 	return h, nil
 }
 
-//DeleteStatus for the given the handle
-func (ac *AppContext) DeleteStatus(handle interface{}) error {
-	err := ac.rtc.RtcDeletePair(handle)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-//Return the handle for status for a given app and cluster
-func (ac *AppContext) GetStatusHandle(appname string, clustername string) (interface{}, error) {
+// GetClusterStatusHandle returns the handle for cluster status for a given app and cluster
+func (ac *AppContext) GetClusterStatusHandle(appname string, clustername string) (interface{}, error) {
 	if appname == "" {
 		return nil, pkgerrors.Errorf("Not a valid run time context app name")
 	}
@@ -467,6 +524,11 @@ func (ac *AppContext) UpdateStatusValue(handle interface{}, value interface{}) e
 	return ac.rtc.RtcUpdateValue(handle, value)
 }
 
+//UpdateValue updates the state value with the given handle
+func (ac *AppContext) UpdateValue(handle interface{}, value interface{}) error {
+	return ac.rtc.RtcUpdateValue(handle, value)
+}
+
 //Return all the handles under the composite app
 func (ac *AppContext) GetAllHandles(handle interface{}) ([]interface{}, error) {
 	hs, err := ac.rtc.RtcGetHandles(handle)
@@ -478,7 +540,7 @@ func (ac *AppContext) GetAllHandles(handle interface{}) ([]interface{}, error) {
 
 //Returns the value for a given handle
 func (ac *AppContext) GetValue(handle interface{}) (interface{}, error) {
-	var v string
+	var v interface{}
 	err := ac.rtc.RtcGetValue(handle, &v)
 	if err != nil {
 		return nil, err
diff --git a/src/orchestrator/pkg/appcontext/appcontext_test.go b/src/orchestrator/pkg/appcontext/appcontext_test.go
index 92c43113..4d66b559 100644
--- a/src/orchestrator/pkg/appcontext/appcontext_test.go
+++ b/src/orchestrator/pkg/appcontext/appcontext_test.go
@@ -18,9 +18,10 @@ package appcontext
 
 import (
 	"fmt"
-	pkgerrors "github.com/pkg/errors"
 	"strings"
 	"testing"
+
+	pkgerrors "github.com/pkg/errors"
 )
 
 // Mock run time context
@@ -145,10 +146,6 @@ func (c *MockRunTimeContext) RtcUpdateValue(handle interface{}, value interface{
 	return c.Err
 }
 
-func (rtc *MockRunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) {
-	return nil, nil
-}
-
 func TestCreateCompositeApp(t *testing.T) {
 	var ac = AppContext{}
 	testCases := []struct {
diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index d017c1e9..f9829853 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -19,6 +19,7 @@ package module
 import (
 	"encoding/json"
 	"reflect"
+	"time"
 
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/state"
@@ -135,12 +136,15 @@ func (c *DeploymentIntentGroupClient) CreateDeploymentIntentGroup(d DeploymentIn
 	}
 
 	// Add the stateInfo record
-	stateInfo := state.StateInfo{
+	s := state.StateInfo{}
+	a := state.ActionEntry{
 		State:     state.StateEnum.Created,
 		ContextId: "",
+		TimeStamp: time.Now(),
 	}
+	s.Actions = append(s.Actions, a)
 
-	err = db.DBconn.Insert(c.storeName, gkey, nil, c.tagState, stateInfo)
+	err = db.DBconn.Insert(c.storeName, gkey, nil, c.tagState, s)
 	if err != nil {
 		return DeploymentIntentGroup{}, pkgerrors.Wrap(err, "Error updating the stateInfo of the DeploymentIntentGroup: "+d.MetaData.Name)
 	}
@@ -252,10 +256,33 @@ func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p s
 		Version:      v,
 	}
 	s, err := c.GetDeploymentIntentGroupState(di, p, ca, v)
-	if err == nil && s.State == state.StateEnum.Instantiated {
+	if err != nil {
+		return pkgerrors.Errorf("Error getting stateInfo from DeploymentIntentGroup: " + di)
+	}
+
+	stateVal, err := state.GetCurrentStateFromStateInfo(s)
+	if err != nil {
+		return pkgerrors.Errorf("Error getting current state from DeploymentIntentGroup stateInfo: " + di)
+	}
+
+	if stateVal == state.StateEnum.Instantiated {
 		return pkgerrors.Errorf("DeploymentIntentGroup must be terminated before it can be deleted " + di)
 	}
 
+	// remove the app contexts associated with thie Deployment Intent Group
+	if stateVal == state.StateEnum.Terminated {
+		for _, id := range state.GetContextIdsFromStateInfo(s) {
+			context, err := state.GetAppContextFromId(id)
+			if err != nil {
+				return pkgerrors.Wrap(err, "Error getting appcontext from Deployment Intent Group StateInfo")
+			}
+			err = context.DeleteCompositeApp()
+			if err != nil {
+				return pkgerrors.Wrap(err, "Error deleting appcontext for Deployment Intent Group")
+			}
+		}
+	}
+
 	err = db.DBconn.Remove(c.storeName, k)
 	if err != nil {
 		return pkgerrors.Wrap(err, "Error deleting DeploymentIntentGroup entry")
diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go
index 9c0c9e31..08250d16 100644
--- a/src/orchestrator/pkg/module/instantiation.go
+++ b/src/orchestrator/pkg/module/instantiation.go
@@ -20,6 +20,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"time"
 
 	rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
 	gpic "github.com/onap/multicloud-k8s/src/orchestrator/pkg/gpic"
@@ -100,7 +101,11 @@ func (c InstantiationClient) Approve(p string, ca string, v string, di string) e
 	if err != nil {
 		return pkgerrors.Wrap(err, "DeploymentIntentGroup has no state info: "+di)
 	}
-	switch s.State {
+	stateVal, err := state.GetCurrentStateFromStateInfo(s)
+	if err != nil {
+		return pkgerrors.Errorf("Error getting current state from DeploymentIntentGroup stateInfo: " + di)
+	}
+	switch stateVal {
 	case state.StateEnum.Approved:
 		return nil
 	case state.StateEnum.Terminated:
@@ -108,11 +113,11 @@ func (c InstantiationClient) Approve(p string, ca string, v string, di string) e
 	case state.StateEnum.Created:
 		break
 	case state.StateEnum.Applied:
-		return pkgerrors.Errorf("DeploymentIntentGroup is in an invalid state" + s.State)
+		return pkgerrors.Errorf("DeploymentIntentGroup is in an invalid state" + stateVal)
 	case state.StateEnum.Instantiated:
 		return pkgerrors.Errorf("DeploymentIntentGroup has already been instantiated" + di)
 	default:
-		return pkgerrors.Errorf("DeploymentIntentGroup is in an unknown state" + s.State)
+		return pkgerrors.Errorf("DeploymentIntentGroup is in an unknown state" + stateVal)
 	}
 
 	key := DeploymentIntentGroupKey{
@@ -121,12 +126,14 @@ func (c InstantiationClient) Approve(p string, ca string, v string, di string) e
 		CompositeApp: ca,
 		Version:      v,
 	}
-	stateInfo := state.StateInfo{
+	a := state.ActionEntry{
 		State:     state.StateEnum.Approved,
 		ContextId: "",
+		TimeStamp: time.Now(),
 	}
+	s.Actions = append(s.Actions, a)
 
-	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagState, stateInfo)
+	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagState, s)
 	if err != nil {
 		return pkgerrors.Wrap(err, "Error updating the stateInfo of the DeploymentIntentGroup: "+di)
 	}
@@ -229,7 +236,11 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	if err != nil {
 		return pkgerrors.Errorf("Error retrieving DeploymentIntentGroup stateInfo: " + di)
 	}
-	switch s.State {
+	stateVal, err := state.GetCurrentStateFromStateInfo(s)
+	if err != nil {
+		return pkgerrors.Errorf("Error getting current state from DeploymentIntentGroup stateInfo: " + di)
+	}
+	switch stateVal {
 	case state.StateEnum.Approved:
 		break
 	case state.StateEnum.Terminated:
@@ -241,7 +252,7 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 	case state.StateEnum.Instantiated:
 		return pkgerrors.Errorf("DeploymentIntentGroup has already been instantiated" + di)
 	default:
-		return pkgerrors.Errorf("DeploymentIntentGroup is in an unknown state" + s.State)
+		return pkgerrors.Errorf("DeploymentIntentGroup is in an unknown state" + stateVal)
 	}
 
 	rName := dIGrp.Spec.Version //rName is releaseName
@@ -286,6 +297,7 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 		sortedTemplates, err := GetSortedTemplateForApp(eachApp.Metadata.Name, p, ca, v, rName, cp, overrideValues)
 
 		if err != nil {
+			deleteAppContext(context)
 			return pkgerrors.Wrap(err, "Unable to get the sorted templates for app")
 		}
 
@@ -293,16 +305,19 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 
 		resources, err := getResources(sortedTemplates)
 		if err != nil {
+			deleteAppContext(context)
 			return pkgerrors.Wrapf(err, "Unable to get the resources for app :: %s", eachApp.Metadata.Name)
 		}
 
 		specData, err := NewAppIntentClient().GetAllIntentsByApp(eachApp.Metadata.Name, p, ca, v, gIntent)
 		if err != nil {
+			deleteAppContext(context)
 			return pkgerrors.Wrap(err, "Unable to get the intents for app")
 		}
 		// listOfClusters shall have both mandatoryClusters and optionalClusters where the app needs to be installed.
 		listOfClusters, err := gpic.IntentResolver(specData.Intent)
 		if err != nil {
+			deleteAppContext(context)
 			return pkgerrors.Wrap(err, "Unable to get the intents resolved for app")
 		}
 
@@ -312,85 +327,100 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin
 		// Add an app to the app context
 		apphandle, err := context.AddApp(compositeHandle, eachApp.Metadata.Name)
 		if err != nil {
-			cleanuperr := context.DeleteCompositeApp()
-			if cleanuperr != nil {
-				log.Info(":: Error Cleaning up AppContext compositeApp failure ::", log.Fields{"Error": cleanuperr.Error(), "AppName": eachApp.Metadata.Name})
-			}
+			deleteAppContext(context)
 			return pkgerrors.Wrap(err, "Error adding App to AppContext")
 		}
 		err = addClustersToAppContext(listOfClusters, context, apphandle, resources)
 		if err != nil {
-			log.Info(":: Error while adding cluster and resources to app ::", log.Fields{"Error": err.Error(), "AppName": eachApp.Metadata.Name})
+			deleteAppContext(context)
+			return pkgerrors.Wrap(err, "Error while adding cluster and resources to app")
 		}
 		err = verifyResources(listOfClusters, context, resources, eachApp.Metadata.Name)
 		if err != nil {
-			log.Info(":: Error while verifying resources in app ::", log.Fields{"Error": err.Error(), "AppName": eachApp.Metadata.Name})
+			deleteAppContext(context)
+			return pkgerrors.Wrap(err, "Error while verifying resources in app: ")
 		}
-
 	}
-	jappOrderInstr, _ := json.Marshal(appOrderInstr)
+	jappOrderInstr, err := json.Marshal(appOrderInstr)
+	if err != nil {
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error marshalling app order instruction")
+	}
 	appDepInstr.Appdep = appdep
-	jappDepInstr, _ := json.Marshal(appDepInstr)
-	context.AddInstruction(compositeHandle, "app", "order", string(jappOrderInstr))
-	context.AddInstruction(compositeHandle, "app", "dependency", string(jappDepInstr))
-	//END: storing into etcd
-
-	// BEGIN:: save the context in the orchestrator db record
-	key := DeploymentIntentGroupKey{
-		Name:         di,
-		Project:      p,
-		CompositeApp: ca,
-		Version:      v,
+	jappDepInstr, err := json.Marshal(appDepInstr)
+	if err != nil {
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error marshalling app dependency instruction")
 	}
-	stateInfo := state.StateInfo{
-		State:     state.StateEnum.Instantiated,
-		ContextId: ctxval.(string),
+	_, err = context.AddInstruction(compositeHandle, "app", "order", string(jappOrderInstr))
+	if err != nil {
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error adding app dependency instruction")
 	}
-	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagState, stateInfo)
+	_, err = context.AddInstruction(compositeHandle, "app", "dependency", string(jappDepInstr))
 	if err != nil {
-		cleanuperr := context.DeleteCompositeApp()
-		if cleanuperr != nil {
-
-			log.Info(":: Error Cleaning up AppContext while saving context in the db for GPIntent ::", log.Fields{"Error": cleanuperr.Error(), "GPIntent": gIntent, "DeploymentIntentGroup": di, "CompositeApp": ca, "CompositeAppVersion": v, "Project": p})
-		}
-		return pkgerrors.Wrap(err, "Error adding AppContext to DB")
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error adding app dependency instruction")
 	}
-	// END:: save the context in the orchestrator db record
+	//END: storing into etcd
 
 	// BEGIN: scheduler code
 
 	pl, mapOfControllers, err := getPrioritizedControllerList(p, ca, v, di)
 	if err != nil {
-		return err
+		return pkgerrors.Wrap(err, "Error adding getting prioritized controller list")
 	}
 	log.Info("Priority Based List ", log.Fields{"PlacementControllers::": pl.pPlaCont,
 		"ActionControllers::": pl.pActCont, "mapOfControllers::": mapOfControllers})
 
 	err = callGrpcForControllerList(pl.pPlaCont, mapOfControllers, ctxval)
 	if err != nil {
-		return err
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error calling gRPC for placement controller list")
 	}
 
 	err = deleteExtraClusters(allApps, context)
 	if err != nil {
-		return err
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error deleting extra clusters")
 	}
 
 	err = callGrpcForControllerList(pl.pActCont, mapOfControllers, ctxval)
 	if err != nil {
-		return err
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error calling gRPC for action controller list")
 	}
-
 	// END: Scheduler code
 
 	// BEGIN : Rsync code
 	err = callRsyncInstall(ctxval)
 	if err != nil {
-		return err
+		deleteAppContext(context)
+		return pkgerrors.Wrap(err, "Error calling rsync")
 	}
 	// END : Rsyc code
 
-	log.Info(":: Done with instantiation... ::", log.Fields{"CompositeAppName": ca})
+	// BEGIN:: save the context in the orchestrator db record
+	key := DeploymentIntentGroupKey{
+		Name:         di,
+		Project:      p,
+		CompositeApp: ca,
+		Version:      v,
+	}
+	a := state.ActionEntry{
+		State:     state.StateEnum.Instantiated,
+		ContextId: ctxval.(string),
+		TimeStamp: time.Now(),
+	}
+	s.Actions = append(s.Actions, a)
+	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagState, s)
+	if err != nil {
+		log.Warn(":: Error updating DeploymentIntentGroup state in DB ::", log.Fields{"Error": err.Error(), "GPIntent": gIntent, "DeploymentIntentGroup": di, "CompositeApp": ca, "CompositeAppVersion": v, "Project": p, "AppContext": ctxval.(string)})
+		return pkgerrors.Wrap(err, "Error adding DeploymentIntentGroup state to DB")
+	}
+	// END:: save the context in the orchestrator db record
+
+	log.Info(":: Done with instantiation call to rsync... ::", log.Fields{"CompositeAppName": ca})
 	return err
 }
 
@@ -406,7 +436,8 @@ func (c InstantiationClient) Status(p string, ca string, v string, di string) (S
 		return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found: "+di)
 	}
 
-	ac, err := state.GetAppContextFromStateInfo(s)
+	currentCtxId := state.GetLastContextIdFromStateInfo(s)
+	ac, err := state.GetAppContextFromId(currentCtxId)
 	if err != nil {
 		return StatusData{}, pkgerrors.Wrap(err, "AppContext for deploymentIntentGroup not found: "+di)
 	}
@@ -430,7 +461,7 @@ func (c InstantiationClient) Status(p string, ca string, v string, di string) (S
 		}
 
 		for _, cluster := range clusters {
-			handle, err := ac.GetStatusHandle(app.Metadata.Name, cluster)
+			handle, err := ac.GetClusterStatusHandle(app.Metadata.Name, cluster)
 			if err != nil {
 				log.Info(":: No status handle for cluster, app ::",
 					log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err})
@@ -470,23 +501,21 @@ func (c InstantiationClient) Terminate(p string, ca string, v string, di string)
 	s, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupState(di, p, ca, v)
 	if err != nil {
 		return pkgerrors.Wrap(err, "DeploymentIntentGroup has no state info: "+di)
-	} else if s.State != state.StateEnum.Instantiated {
-		return pkgerrors.Errorf("DeploymentIntentGroup is not instantiated" + di)
 	}
 
-	ac, err := state.GetAppContextFromStateInfo(s)
+	stateVal, err := state.GetCurrentStateFromStateInfo(s)
 	if err != nil {
-		return pkgerrors.Wrap(err, "AppContext for deploymentIntentGroup not found: "+di)
+		return pkgerrors.Errorf("Error getting current state from DeploymentIntentGroup stateInfo: " + di)
 	}
 
-	err = callRsyncUninstall(s.ContextId)
-	if err != nil {
-		return err
+	if stateVal != state.StateEnum.Instantiated {
+		return pkgerrors.Errorf("DeploymentIntentGroup is not instantiated" + di)
 	}
 
-	err = ac.DeleteCompositeApp()
+	currentCtxId := state.GetLastContextIdFromStateInfo(s)
+	err = callRsyncUninstall(currentCtxId)
 	if err != nil {
-		return pkgerrors.Wrap(err, "Error deleting the app context for DeploymentIntentGroup: "+di)
+		return err
 	}
 
 	key := DeploymentIntentGroupKey{
@@ -495,12 +524,14 @@ func (c InstantiationClient) Terminate(p string, ca string, v string, di string)
 		CompositeApp: ca,
 		Version:      v,
 	}
-	stateInfo := state.StateInfo{
+	a := state.ActionEntry{
 		State:     state.StateEnum.Terminated,
-		ContextId: "",
+		ContextId: currentCtxId,
+		TimeStamp: time.Now(),
 	}
+	s.Actions = append(s.Actions, a)
 
-	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagState, stateInfo)
+	err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagState, s)
 	if err != nil {
 		return pkgerrors.Wrap(err, "Error updating the stateInfo of the DeploymentIntentGroup: "+di)
 	}
diff --git a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
index 9ace81b6..692cdf1e 100644
--- a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
+++ b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go
@@ -71,6 +71,16 @@ func makeAppContextForCompositeApp(p, ca, v, rName string) (contextForCompositeA
 
 }
 
+// deleteAppContext removes an appcontext
+func deleteAppContext(ct appcontext.AppContext) error {
+	err := ct.DeleteCompositeApp()
+	if err != nil {
+		log.Warn(":: Error deleting AppContext ::", log.Fields{"Error": err})
+		return pkgerrors.Wrapf(err, "Error Deleteing AppContext")
+	}
+	return nil
+}
+
 // getResources shall take in the sorted templates and output the resources
 // which consists of name(name+kind) and filecontent
 func getResources(st []helm.KubernetesResourceTemplate) ([]resource, error) {
@@ -208,7 +218,7 @@ func verifyResources(l gpic.ClusterList, ct appcontext.AppContext, resources []r
 			for _, res := range resources {
 				rh, err := ct.GetResourceHandle(appName, cn, res.name)
 				if err != nil {
-					return pkgerrors.Wrapf(err, "Error getting resoure handle for resource :: %s, app:: %s, cluster :: %s, groupName :: %s", appName, res.name, cn, gn)
+					return pkgerrors.Wrapf(err, "Error getting resource handle for resource :: %s, app:: %s, cluster :: %s, groupName :: %s", appName, res.name, cn, gn)
 				}
 				log.Info(":: GetResourceHandle ::", log.Fields{"ResourceHandler": rh, "appName": appName, "Cluster": cn, "Resource": res.name})
 			}
diff --git a/src/orchestrator/pkg/resourcestatus/resourcestatus.go b/src/orchestrator/pkg/resourcestatus/resourcestatus.go
new file mode 100644
index 00000000..f399e29e
--- /dev/null
+++ b/src/orchestrator/pkg/resourcestatus/resourcestatus.go
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package resourcestatus
+
+// ResourceStatus struct is used to maintain the rsync status for resources in the appcontext
+// that rsync is synchronizing to clusters
+type ResourceStatus struct {
+	Status RsyncStatus
+}
+
+type RsyncStatus = string
+
+type statusValues struct {
+	Pending  RsyncStatus
+	Applied  RsyncStatus
+	Failed   RsyncStatus
+	Retrying RsyncStatus
+	Deleted  RsyncStatus
+}
+
+var RsyncStatusEnum = &statusValues{
+	Pending:  "Pending",
+	Applied:  "Applied",
+	Failed:   "Failed",
+	Retrying: "Retrying",
+	Deleted:  "Deleted",
+}
diff --git a/src/orchestrator/pkg/rtcontext/rtcontext.go b/src/orchestrator/pkg/rtcontext/rtcontext.go
index f3905eb0..f77fb329 100644
--- a/src/orchestrator/pkg/rtcontext/rtcontext.go
+++ b/src/orchestrator/pkg/rtcontext/rtcontext.go
@@ -41,7 +41,6 @@ type Rtcontext interface {
 	RtcAddMeta(meta interface{}) error
 	RtcGet() (interface{}, error)
 	RtcAddLevel(handle interface{}, level string, value string) (interface{}, error)
-	RtcAddStatus(handle interface{}, value interface{}) (interface{}, error)
 	RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error)
 	RtcAddInstruction(handle interface{}, level string, insttype string, value interface{}) (interface{}, error)
 	RtcDeletePair(handle interface{}) error
@@ -203,26 +202,6 @@ func (rtc *RunTimeContext) RtcAddOneLevel(pl interface{}, level string, value in
 	return (interface{})(key), nil
 }
 
-// Add status under the given level and return new handle
-func (rtc *RunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) {
-
-	str := fmt.Sprintf("%v", handle)
-	sid := fmt.Sprintf("%v", rtc.cid)
-	if !strings.HasPrefix(str, sid) {
-		return nil, pkgerrors.Errorf("Not a valid run time context handle")
-	}
-	if value == nil {
-		return nil, pkgerrors.Errorf("Not a valid run time context resource value")
-	}
-
-	k := str + "status" + "/"
-	err := contextdb.Db.Put(k, value)
-	if err != nil {
-		return nil, pkgerrors.Errorf("Error adding run time context status: %s", err.Error())
-	}
-	return (interface{})(k), nil
-}
-
 // Add a resource under the given level and return new handle
 func (rtc *RunTimeContext) RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error) {
 
diff --git a/src/orchestrator/pkg/state/state_helper.go b/src/orchestrator/pkg/state/state_helper.go
index a65cea8d..9d59fb75 100644
--- a/src/orchestrator/pkg/state/state_helper.go
+++ b/src/orchestrator/pkg/state/state_helper.go
@@ -16,14 +16,56 @@
 
 package state
 
-import "github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
+import (
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
+	pkgerrors "github.com/pkg/errors"
+)
 
 // GetAppContextFromStateInfo loads the appcontext present in the StateInfo input
-func GetAppContextFromStateInfo(s StateInfo) (appcontext.AppContext, error) {
+func GetAppContextFromId(ctxid string) (appcontext.AppContext, error) {
 	var cc appcontext.AppContext
-	_, err := cc.LoadAppContext(s.ContextId)
+	_, err := cc.LoadAppContext(ctxid)
 	if err != nil {
 		return appcontext.AppContext{}, err
 	}
 	return cc, nil
 }
+
+// GetCurrentStateFromStatInfo gets the last (current) state from StateInfo
+func GetCurrentStateFromStateInfo(s StateInfo) (StateValue, error) {
+	alen := len(s.Actions)
+	if alen == 0 {
+		return StateEnum.Undefined, pkgerrors.Errorf("No state information")
+	}
+	return s.Actions[alen-1].State, nil
+}
+
+// GetLastContextFromStatInfo gets the last (most recent) context id from StateInfo
+func GetLastContextIdFromStateInfo(s StateInfo) string {
+	alen := len(s.Actions)
+	if alen > 0 {
+		return s.Actions[alen-1].ContextId
+	} else {
+		return ""
+	}
+}
+
+// GetContextIdsFromStatInfo return a list of the unique AppContext Ids in the StateInfo
+func GetContextIdsFromStateInfo(s StateInfo) []string {
+	m := make(map[string]string)
+
+	for _, a := range s.Actions {
+		if a.ContextId != "" {
+			m[a.ContextId] = ""
+		}
+	}
+
+	ids := make([]string, len(m))
+	i := 0
+	for k := range m {
+		ids[i] = k
+		i++
+	}
+
+	return ids
+}
diff --git a/src/orchestrator/pkg/state/types.go b/src/orchestrator/pkg/state/types.go
index 25fb60d2..665a1be4 100644
--- a/src/orchestrator/pkg/state/types.go
+++ b/src/orchestrator/pkg/state/types.go
@@ -16,16 +16,27 @@
 
 package state
 
+import "time"
+
 // StateInfo struct is used to maintain the values for state, contextid, (and other)
 // information about resources which can be instantiated via rsync.
+// The last Actions entry holds the current state of the container object.
 type StateInfo struct {
+	Actions []ActionEntry
+}
+
+// ActionEntry is used to keep track of the time an action (e.g. Created, Instantiate, Terminate) was invoked
+// For actions where an AppContext is relevent, the ContextId field will be non-zero length
+type ActionEntry struct {
 	State     StateValue
 	ContextId string
+	TimeStamp time.Time
 }
 
 type StateValue = string
 
 type states struct {
+	Undefined    StateValue
 	Created      StateValue
 	Approved     StateValue
 	Applied      StateValue
@@ -34,6 +45,7 @@ type states struct {
 }
 
 var StateEnum = &states{
+	Undefined:    "Undefined",
 	Created:      "Created",
 	Approved:     "Approved",
 	Applied:      "Applied",