Changes to add state and retry logic to rsync

- Adds retry watcher and related functionality.
- Adds code to update, get the status from appcontext.
- Adds logic to handle state transition during terminate.

Issue-ID: MULTICLOUD-1005
Signed-off-by: Manjunath Ranganathaiah <manjunath.ranganathaiah@intel.com>
Change-Id: I2ed76efd9d8b6f40fec547bbe8b7d8a86f69ce07

3 files changed, 52 insertions, 14 deletions

diff --git a/src/orchestrator/pkg/appcontext/appcontext.go b/src/orchestrator/pkg/appcontext/appcontext.go
index db2ba432..5d757940 100644
--- a/src/orchestrator/pkg/appcontext/appcontext.go
+++ b/src/orchestrator/pkg/appcontext/appcontext.go
@@ -37,30 +37,30 @@ type AppContext struct {
 // AppContextStatus represents the current status of the appcontext
 //	Instantiating - instantiate has been invoked and is still in progress
 //	Instantiated - instantiate has completed
-//	PreTerminate - terminate has been invoked when in Instantiating status - need to clean up first
 //	Terminating - terminate has been invoked and is still in progress
 //	Terminated - terminate has completed
-//	Failed - the instantiate or terminate action has failed
+//	InstantiateFailed - the instantiate action has failed
+//	TerminateFailed - the terminate action has failed
 type AppContextStatus struct {
 	Status StatusValue
 }
 type StatusValue string
 type statuses struct {
-	Instantiating StatusValue
-	Instantiated  StatusValue
-	PreTerminate  StatusValue
-	Terminating   StatusValue
-	Terminated    StatusValue
-	Failed        StatusValue
+	Instantiating     StatusValue
+	Instantiated      StatusValue
+	Terminating       StatusValue
+	Terminated        StatusValue
+	InstantiateFailed StatusValue
+	TerminateFailed   StatusValue
 }
 
 var AppContextStatusEnum = &statuses{
-	Instantiating: "Instantiating",
-	Instantiated:  "Instantiated",
-	PreTerminate:  "PreTerminate",
-	Terminating:   "Terminating",
-	Terminated:    "Terminated",
-	Failed:        "Failed",
+	Instantiating:     "Instantiating",
+	Instantiated:      "Instantiated",
+	Terminating:       "Terminating",
+	Terminated:        "Terminated",
+	InstantiateFailed: "InstantiateFailed",
+	TerminateFailed:   "TerminateFailed",
 }
 
 // CompositeAppMeta consists of projectName, CompositeAppName,
diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go
index f9829853..dec6391f 100644
--- a/src/orchestrator/pkg/module/deployment_intent_groups.go
+++ b/src/orchestrator/pkg/module/deployment_intent_groups.go
@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db"
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/state"
 
@@ -271,6 +272,14 @@ func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p s
 
 	// remove the app contexts associated with thie Deployment Intent Group
 	if stateVal == state.StateEnum.Terminated {
+		// Verify that the appcontext has completed terminating
+		ctxid := state.GetLastContextIdFromStateInfo(s)
+		acStatus, err := state.GetAppContextStatus(ctxid)
+		if err == nil &&
+			!(acStatus.Status == appcontext.AppContextStatusEnum.Terminated || acStatus.Status == appcontext.AppContextStatusEnum.TerminateFailed) {
+			return pkgerrors.Errorf("DeploymentIntentGroup has not completed terminating: " + di)
+		}
+
 		for _, id := range state.GetContextIdsFromStateInfo(s) {
 			context, err := state.GetAppContextFromId(id)
 			if err != nil {
diff --git a/src/orchestrator/pkg/state/state_helper.go b/src/orchestrator/pkg/state/state_helper.go
index 9d59fb75..1f926f8f 100644
--- a/src/orchestrator/pkg/state/state_helper.go
+++ b/src/orchestrator/pkg/state/state_helper.go
@@ -17,6 +17,8 @@
 package state
 
 import (
+	"encoding/json"
+
 	"github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext"
 	pkgerrors "github.com/pkg/errors"
 )
@@ -69,3 +71,30 @@ func GetContextIdsFromStateInfo(s StateInfo) []string {
 
 	return ids
 }
+
+func GetAppContextStatus(ctxid string) (appcontext.AppContextStatus, error) {
+
+	ac, err := GetAppContextFromId(ctxid)
+	if err != nil {
+		return appcontext.AppContextStatus{}, err
+	}
+
+	h, err := ac.GetCompositeAppHandle()
+	if err != nil {
+		return appcontext.AppContextStatus{}, err
+	}
+	sh, err := ac.GetLevelHandle(h, "status")
+	if err != nil {
+		return appcontext.AppContextStatus{}, err
+	}
+	s, err := ac.GetValue(sh)
+	if err != nil {
+		return appcontext.AppContextStatus{}, err
+	}
+	acStatus := appcontext.AppContextStatus{}
+	js, _ := json.Marshal(s)
+	json.Unmarshal(js, &acStatus)
+
+	return acStatus, nil
+
+}