diff options
Diffstat (limited to 'src/orchestrator')
-rw-r--r-- | src/orchestrator/api/api.go | 2 | ||||
-rw-r--r-- | src/orchestrator/api/instantiation_handler.go | 46 | ||||
-rw-r--r-- | src/orchestrator/go.mod | 15 | ||||
-rw-r--r-- | src/orchestrator/pkg/appcontext/appcontext.go | 56 | ||||
-rw-r--r-- | src/orchestrator/pkg/appcontext/appcontext_test.go | 4 | ||||
-rw-r--r-- | src/orchestrator/pkg/module/deployment_intent_groups.go | 32 | ||||
-rw-r--r-- | src/orchestrator/pkg/module/instantiation.go | 113 | ||||
-rw-r--r-- | src/orchestrator/pkg/module/instantiation_appcontext_helper.go | 43 | ||||
-rw-r--r-- | src/orchestrator/pkg/rtcontext/rtcontext.go | 26 |
9 files changed, 314 insertions, 23 deletions
diff --git a/src/orchestrator/api/api.go b/src/orchestrator/api/api.go index 2470a1be..5abbb96d 100644 --- a/src/orchestrator/api/api.go +++ b/src/orchestrator/api/api.go @@ -180,6 +180,8 @@ func NewRouter(projectClient moduleLib.ProjectManager, } router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/deployment-intent-groups/{deployment-intent-group-name}/instantiate", instantiationHandler.instantiateHandler).Methods("POST") + router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/deployment-intent-groups/{deployment-intent-group-name}/terminate", instantiationHandler.terminateHandler).Methods("POST") + router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/deployment-intent-groups/{deployment-intent-group-name}/status", instantiationHandler.statusHandler).Methods("GET") return router } diff --git a/src/orchestrator/api/instantiation_handler.go b/src/orchestrator/api/instantiation_handler.go index c95785f2..ce50e5b8 100644 --- a/src/orchestrator/api/instantiation_handler.go +++ b/src/orchestrator/api/instantiation_handler.go @@ -17,9 +17,11 @@ package api import ( + "encoding/json" + "net/http" + "github.com/gorilla/mux" moduleLib "github.com/onap/multicloud-k8s/src/orchestrator/pkg/module" - "net/http" ) /* Used to store backend implementation objects @@ -45,3 +47,45 @@ func (h instantiationHandler) instantiateHandler(w http.ResponseWriter, r *http. w.WriteHeader(http.StatusAccepted) } + +func (h instantiationHandler) terminateHandler(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + p := vars["project-name"] + ca := vars["composite-app-name"] + v := vars["composite-app-version"] + di := vars["deployment-intent-group-name"] + + iErr := h.client.Terminate(p, ca, v, di) + if iErr != nil { + http.Error(w, iErr.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusAccepted) + +} + +func (h instantiationHandler) statusHandler(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + p := vars["project-name"] + ca := vars["composite-app-name"] + v := vars["composite-app-version"] + di := vars["deployment-intent-group-name"] + + status, iErr := h.client.Status(p, ca, v, di) + if iErr != nil { + http.Error(w, iErr.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + iErr = json.NewEncoder(w).Encode(status) + if iErr != nil { + http.Error(w, iErr.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusAccepted) + +} diff --git a/src/orchestrator/go.mod b/src/orchestrator/go.mod index 223dc068..3f14f00b 100644 --- a/src/orchestrator/go.mod +++ b/src/orchestrator/go.mod @@ -2,17 +2,12 @@ module github.com/onap/multicloud-k8s/src/orchestrator require ( github.com/MakeNowJust/heredoc v1.0.0 // indirect - github.com/Masterminds/goutils v1.1.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 github.com/coreos/etcd v3.3.12+incompatible - github.com/cyphar/filepath-securejoin v0.2.2 // indirect github.com/docker/docker v1.13.1 // indirect - github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect - github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/ghodss/yaml v1.0.0 - github.com/gobwas/glob v0.2.3 // indirect github.com/golang/protobuf v1.4.1 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.3 @@ -21,34 +16,32 @@ require ( github.com/lib/pq v1.6.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mitchellh/copystructure v1.0.0 // indirect - github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/onap/multicloud-k8s/src/clm v0.0.0-00010101000000-000000000000 + github.com/onap/multicloud-k8s/src/monitor v0.0.0-20200630152613-7c20f73e7c5d github.com/onap/multicloud-k8s/src/ncm v0.0.0-20200515060444-c77850a75eee + github.com/onap/multicloud-k8s/src/rsync v0.0.0-20200630152613-7c20f73e7c5d github.com/pkg/errors v0.8.1 github.com/rubenv/sql-migrate v0.0.0-20200429072036-ae26b214fa43 // indirect github.com/russross/blackfriday v1.5.2 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v1.0.0 // indirect - github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 // indirect go.etcd.io/etcd v3.3.12+incompatible go.mongodb.org/mongo-driver v1.0.0 golang.org/x/net v0.0.0-20200301022130-244492dfa37a google.golang.org/grpc v1.27.1 google.golang.org/protobuf v1.24.0 gopkg.in/square/go-jose.v2 v2.5.1 // indirect + gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 - k8s.io/apiextensions-apiserver v0.0.0-00010101000000-000000000000 // indirect k8s.io/apimachinery v0.0.0-20190831074630-461753078381 - k8s.io/apiserver v0.0.0-00010101000000-000000000000 // indirect - k8s.io/cli-runtime v0.0.0-00010101000000-000000000000 // indirect k8s.io/cloud-provider v0.0.0-00010101000000-000000000000 // indirect k8s.io/helm v2.14.3+incompatible sigs.k8s.io/kustomize v2.0.3+incompatible // indirect - vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect ) replace ( github.com/onap/multicloud-k8s/src/clm => ../clm + github.com/onap/multicloud-k8s/src/monitor => ../monitor k8s.io/api => k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8 k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d diff --git a/src/orchestrator/pkg/appcontext/appcontext.go b/src/orchestrator/pkg/appcontext/appcontext.go index a847ae32..cdf23bfa 100644 --- a/src/orchestrator/pkg/appcontext/appcontext.go +++ b/src/orchestrator/pkg/appcontext/appcontext.go @@ -18,10 +18,11 @@ package appcontext import ( "fmt" + "strings" + log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils" "github.com/onap/multicloud-k8s/src/orchestrator/pkg/rtcontext" pkgerrors "github.com/pkg/errors" - "strings" ) // metaPrefix used for denoting clusterMeta level @@ -413,6 +414,59 @@ func (ac *AppContext) GetResourceInstruction(appname string, clustername string, return v, nil } +//AddStatus for holding status of all resources under app and cluster +// handle should be a cluster handle +func (ac *AppContext) AddStatus(handle interface{}, value interface{}) (interface{}, error) { + h, err := ac.rtc.RtcAddStatus(handle, value) + if err != nil { + return nil, err + } + log.Info(":: Added status handle ::", log.Fields{"StatusHandler": h}) + + return h, nil +} + +//DeleteStatus for the given the handle +func (ac *AppContext) DeleteStatus(handle interface{}) error { + err := ac.rtc.RtcDeletePair(handle) + if err != nil { + return err + } + return nil +} + +//Return the handle for status for a given app and cluster +func (ac *AppContext) GetStatusHandle(appname string, clustername string) (interface{}, error) { + if appname == "" { + return nil, pkgerrors.Errorf("Not a valid run time context app name") + } + if clustername == "" { + return nil, pkgerrors.Errorf("Not a valid run time context cluster name") + } + + rh, err := ac.rtc.RtcGet() + if err != nil { + return nil, err + } + + acrh := fmt.Sprintf("%v", rh) + "app/" + appname + "/cluster/" + clustername + "/status/" + hs, err := ac.rtc.RtcGetHandles(acrh) + if err != nil { + return nil, err + } + for _, v := range hs { + if v == acrh { + return v, nil + } + } + return nil, pkgerrors.Errorf("No handle was found for the given resource") +} + +//UpdateStatusValue updates the status value with the given handle +func (ac *AppContext) UpdateStatusValue(handle interface{}, value interface{}) error { + return ac.rtc.RtcUpdateValue(handle, value) +} + //Return all the handles under the composite app func (ac *AppContext) GetAllHandles(handle interface{}) ([]interface{}, error) { hs, err := ac.rtc.RtcGetHandles(handle) diff --git a/src/orchestrator/pkg/appcontext/appcontext_test.go b/src/orchestrator/pkg/appcontext/appcontext_test.go index 05c73703..92c43113 100644 --- a/src/orchestrator/pkg/appcontext/appcontext_test.go +++ b/src/orchestrator/pkg/appcontext/appcontext_test.go @@ -145,6 +145,10 @@ func (c *MockRunTimeContext) RtcUpdateValue(handle interface{}, value interface{ return c.Err } +func (rtc *MockRunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) { + return nil, nil +} + func TestCreateCompositeApp(t *testing.T) { var ac = AppContext{} testCases := []struct { diff --git a/src/orchestrator/pkg/module/deployment_intent_groups.go b/src/orchestrator/pkg/module/deployment_intent_groups.go index 16a14c7b..3412a034 100644 --- a/src/orchestrator/pkg/module/deployment_intent_groups.go +++ b/src/orchestrator/pkg/module/deployment_intent_groups.go @@ -20,6 +20,7 @@ import ( "encoding/json" "reflect" + appcontext "github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext" "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db" pkgerrors "github.com/pkg/errors" @@ -61,6 +62,7 @@ type OverrideValues struct { type DeploymentIntentGroupManager interface { CreateDeploymentIntentGroup(d DeploymentIntentGroup, p string, ca string, v string) (DeploymentIntentGroup, error) GetDeploymentIntentGroup(di string, p string, ca string, v string) (DeploymentIntentGroup, error) + GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error } @@ -86,6 +88,7 @@ func (dk DeploymentIntentGroupKey) String() string { type DeploymentIntentGroupClient struct { storeName string tagMetaData string + tagContext string } // NewDeploymentIntentGroupClient return an instance of DeploymentIntentGroupClient which implements DeploymentIntentGroupManager @@ -93,6 +96,7 @@ func NewDeploymentIntentGroupClient() *DeploymentIntentGroupClient { return &DeploymentIntentGroupClient{ storeName: "orchestrator", tagMetaData: "deploymentintentgroupmetadata", + tagContext: "contextid", } } @@ -160,6 +164,34 @@ func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroup(di string, p stri } +// GetDeploymentIntentGroup returns the DeploymentIntentGroup with a given name, project, compositeApp and version of compositeApp +func (c *DeploymentIntentGroupClient) GetDeploymentIntentGroupContext(di string, p string, ca string, v string) (appcontext.AppContext, error) { + + key := DeploymentIntentGroupKey{ + Name: di, + Project: p, + CompositeApp: ca, + Version: v, + } + + result, err := db.DBconn.Find(c.storeName, key, c.tagContext) + if err != nil { + return appcontext.AppContext{}, pkgerrors.Wrap(err, "Get DeploymentIntentGroup Context error") + } + + if result != nil { + ctxVal := string(result[0]) + var cc appcontext.AppContext + _, err = cc.LoadAppContext(ctxVal) + if err != nil { + return appcontext.AppContext{}, pkgerrors.Wrap(err, "Error loading DeploymentIntentGroup Appcontext") + } + return cc, nil + } + + return appcontext.AppContext{}, pkgerrors.New("Error getting DeploymentIntentGroup AppContext") +} + // DeleteDeploymentIntentGroup deletes a DeploymentIntentGroup func (c *DeploymentIntentGroupClient) DeleteDeploymentIntentGroup(di string, p string, ca string, v string) error { k := DeploymentIntentGroupKey{ diff --git a/src/orchestrator/pkg/module/instantiation.go b/src/orchestrator/pkg/module/instantiation.go index 043b80f2..9432e4b9 100644 --- a/src/orchestrator/pkg/module/instantiation.go +++ b/src/orchestrator/pkg/module/instantiation.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" + rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1" gpic "github.com/onap/multicloud-k8s/src/orchestrator/pkg/gpic" "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/db" log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils" @@ -42,17 +43,25 @@ type InstantiationClient struct { db InstantiationClientDbInfo } +type ClusterAppStatus struct { + Cluster string + App string + Status rb.ResourceBundleStatus +} + +type StatusData struct { + Data []ClusterAppStatus +} + /* InstantiationKey used in storing the contextid in the momgodb It consists of -GenericPlacementIntentName, ProjectName, CompositeAppName, CompositeAppVersion, DeploymentIntentGroup */ type InstantiationKey struct { - IntentName string Project string CompositeApp string Version string @@ -64,6 +73,8 @@ type InstantiationKey struct { type InstantiationManager interface { //ApproveInstantiation(p string, ca string, v string, di string) (error) Instantiate(p string, ca string, v string, di string) error + Status(p string, ca string, v string, di string) (StatusData, error) + Terminate(p string, ca string, v string, di string) error } // InstantiationClientDbInfo consists of storeName and tagContext @@ -229,6 +240,12 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin return pkgerrors.Wrapf(err, "Unable to get the resources for app :: %s", eachApp.Metadata.Name) } + statusResource, err := getStatusResource(ctxval.(string), eachApp.Metadata.Name) + if err != nil { + return pkgerrors.Wrapf(err, "Unable to generate the status resource for app :: %s", eachApp.Metadata.Name) + } + resources = append(resources, statusResource) + specData, err := NewAppIntentClient().GetAllIntentsByApp(eachApp.Metadata.Name, p, ca, v, gIntent) if err != nil { return pkgerrors.Wrap(err, "Unable to get the intents for app") @@ -269,12 +286,11 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin //END: storing into etcd // BEGIN:: save the context in the orchestrator db record - key := InstantiationKey{ - IntentName: gIntent, - Project: p, - CompositeApp: ca, - Version: v, - DeploymentIntentGroup: di, + key := DeploymentIntentGroupKey{ + Name: di, + Project: p, + CompositeApp: ca, + Version: v, } err = db.DBconn.Insert(c.db.storeName, key, nil, c.db.tagContext, ctxval) @@ -324,3 +340,84 @@ func (c InstantiationClient) Instantiate(p string, ca string, v string, di strin log.Info(":: Done with instantiation... ::", log.Fields{"CompositeAppName": ca}) return err } + +/* +Status takes in projectName, compositeAppName, compositeAppVersion, +DeploymentIntentName. This method is responsible obtaining the status of +the deployment, which is made available in the appcontext. +*/ +func (c InstantiationClient) Status(p string, ca string, v string, di string) (StatusData, error) { + + ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v) + if err != nil { + return StatusData{}, pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di) + } + + // Get all apps in this composite app + allApps, err := NewAppClient().GetApps(p, ca, v) + if err != nil { + return StatusData{}, pkgerrors.Wrap(err, "Not finding the apps") + } + + var diStatus StatusData + diStatus.Data = make([]ClusterAppStatus, 0) + + // Loop through each app and get the status data for each cluster in the app + for _, app := range allApps { + // Get the clusters in the appcontext for this app + clusters, err := ac.GetClusterNames(app.Metadata.Name) + if err != nil { + log.Info(":: No clusters for app ::", log.Fields{"AppName": app.Metadata.Name}) + continue + } + + for _, cluster := range clusters { + handle, err := ac.GetStatusHandle(app.Metadata.Name, cluster) + if err != nil { + log.Info(":: No status handle for cluster, app ::", + log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err}) + continue + } + statusValue, err := ac.GetValue(handle) + if err != nil { + log.Info(":: No status value for cluster, app ::", + log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err}) + continue + } + log.Info(":: STATUS VALUE ::", log.Fields{"statusValue": statusValue}) + var statusData ClusterAppStatus + err = json.Unmarshal([]byte(statusValue.(string)), &statusData.Status) + if err != nil { + log.Info(":: Error unmarshaling status value for cluster, app ::", + log.Fields{"Cluster": cluster, "AppName": app.Metadata.Name, "Error": err}) + continue + } + statusData.Cluster = cluster + statusData.App = app.Metadata.Name + log.Info(":: STATUS DATA ::", log.Fields{"status": statusData}) + + diStatus.Data = append(diStatus.Data, statusData) + } + } + + return diStatus, nil +} + +/* +Terminate takes in projectName, compositeAppName, compositeAppVersion, +DeploymentIntentName and calls rsync to terminate. +*/ +func (c InstantiationClient) Terminate(p string, ca string, v string, di string) error { + + //ac, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v) + _, err := NewDeploymentIntentGroupClient().GetDeploymentIntentGroupContext(di, p, ca, v) + if err != nil { + return pkgerrors.Wrap(err, "deploymentIntentGroup not found "+di) + } + + // TODO - make call to rsync to terminate the composite app deployment + // will leave the appcontext in place for clean up later + // so monitoring status can be performed + + return nil +} diff --git a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go index 43ddd6df..e6e2bf30 100644 --- a/src/orchestrator/pkg/module/instantiation_appcontext_helper.go +++ b/src/orchestrator/pkg/module/instantiation_appcontext_helper.go @@ -25,12 +25,16 @@ import ( "encoding/json" "io/ioutil" + jyaml "github.com/ghodss/yaml" + + rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1" "github.com/onap/multicloud-k8s/src/orchestrator/pkg/appcontext" gpic "github.com/onap/multicloud-k8s/src/orchestrator/pkg/gpic" log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils" "github.com/onap/multicloud-k8s/src/orchestrator/utils" "github.com/onap/multicloud-k8s/src/orchestrator/utils/helm" pkgerrors "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // resource consists of name of reource @@ -90,6 +94,45 @@ func getResources(st []helm.KubernetesResourceTemplate) ([]resource, error) { return resources, nil } +// addStatusResource adds a status monitoring resource to the app +// which consists of name(name+kind) and content +func getStatusResource(id, app string) (resource, error) { + + var statusCr rb.ResourceBundleState + + label := id + "-" + app + name := app + "-" + id + + statusCr.TypeMeta.APIVersion = "k8splugin.io/v1alpha1" + statusCr.TypeMeta.Kind = "ResourceBundleState" + statusCr.SetName(name) + + labels := make(map[string]string) + labels["emco/deployment-id"] = label + statusCr.SetLabels(labels) + + labelSelector, err := metav1.ParseToLabelSelector("emco/deployment-id = " + label) + if err != nil { + log.Info(":: ERROR Parsing Label Selector ::", log.Fields{"Error": err}) + } else { + statusCr.Spec.Selector = labelSelector + } + + // Marshaling to json then convert to yaml works better than marshaling to yaml + // The 'apiVersion' attribute was marshaling to 'apiversion' + // y, _ := yaml.Marshal(&statusCr) + j, _ := json.Marshal(&statusCr) + y, _ := jyaml.JSONToYAML(j) + log.Info(":: RESULTING STATUS CR ::", log.Fields{"StatusCR": y}) + + statusResource := resource{ + name: name + "+" + "ResourceBundleState", + filecontent: string(y), + } + + return statusResource, nil +} + func addResourcesToCluster(ct appcontext.AppContext, ch interface{}, resources []resource) error { var resOrderInstr struct { diff --git a/src/orchestrator/pkg/rtcontext/rtcontext.go b/src/orchestrator/pkg/rtcontext/rtcontext.go index 432c5d87..f3905eb0 100644 --- a/src/orchestrator/pkg/rtcontext/rtcontext.go +++ b/src/orchestrator/pkg/rtcontext/rtcontext.go @@ -18,11 +18,12 @@ package rtcontext import ( "fmt" - "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/contextdb" - pkgerrors "github.com/pkg/errors" "math/rand" "strings" "time" + + "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/contextdb" + pkgerrors "github.com/pkg/errors" ) const maxrand = 0x7fffffffffffffff @@ -40,6 +41,7 @@ type Rtcontext interface { RtcAddMeta(meta interface{}) error RtcGet() (interface{}, error) RtcAddLevel(handle interface{}, level string, value string) (interface{}, error) + RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error) RtcAddInstruction(handle interface{}, level string, insttype string, value interface{}) (interface{}, error) RtcDeletePair(handle interface{}) error @@ -201,6 +203,26 @@ func (rtc *RunTimeContext) RtcAddOneLevel(pl interface{}, level string, value in return (interface{})(key), nil } +// Add status under the given level and return new handle +func (rtc *RunTimeContext) RtcAddStatus(handle interface{}, value interface{}) (interface{}, error) { + + str := fmt.Sprintf("%v", handle) + sid := fmt.Sprintf("%v", rtc.cid) + if !strings.HasPrefix(str, sid) { + return nil, pkgerrors.Errorf("Not a valid run time context handle") + } + if value == nil { + return nil, pkgerrors.Errorf("Not a valid run time context resource value") + } + + k := str + "status" + "/" + err := contextdb.Db.Put(k, value) + if err != nil { + return nil, pkgerrors.Errorf("Error adding run time context status: %s", err.Error()) + } + return (interface{})(k), nil +} + // Add a resource under the given level and return new handle func (rtc *RunTimeContext) RtcAddResource(handle interface{}, resname string, value interface{}) (interface{}, error) { |