From f65daf54a4ab24be9e2c82236a511cedc3bdf230 Mon Sep 17 00:00:00 2001 From: Rajamohan Raj Date: Thu, 9 Apr 2020 16:54:55 +0000 Subject: Adding query APIs for AppIntents In this patch the following tasks where performed: 1. Added APIs for query Intents for each App in the compositeApp. For example, you can see all intents of collectd or prometheus seperately if both formed a compositeApp Collection. 2. Added 'provider-name' parameter to denote the cluster objects used in all intents. Earlier we had only clusterName and clusterlabel to denote a cluster. Modified the tests also for this. 3. Fixed bugs in the plugin_collection_v2.sh. 4. Fixed some minor logging and formatting bugs. Issue-ID: MULTICLOUD-1048 Signed-off-by: Rajamohan Raj Change-Id: Ic452d7ba5d98bb265eb301de84d679d1abe0e34e --- src/orchestrator/api/api.go | 3 +- src/orchestrator/api/app_intent_handler.go | 76 +++++++++++++- src/orchestrator/api/app_profilehandler.go | 8 +- src/orchestrator/pkg/appcontext/appcontext_test.go | 6 +- src/orchestrator/pkg/infra/db/mongo.go | 3 +- src/orchestrator/pkg/infra/db/store.go | 2 +- .../pkg/infra/validation/validation.go | 20 ++++ src/orchestrator/pkg/module/add_intents.go | 5 + src/orchestrator/pkg/module/app_intent.go | 112 ++++++++++++++++++++- src/orchestrator/pkg/module/app_intent_test.go | 46 ++++++--- src/orchestrator/pkg/module/app_profile.go | 2 +- 11 files changed, 252 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/orchestrator/api/api.go b/src/orchestrator/api/api.go index 1d38f106..d2673328 100644 --- a/src/orchestrator/api/api.go +++ b/src/orchestrator/api/api.go @@ -136,8 +136,9 @@ func NewRouter(projectClient moduleLib.ProjectManager, router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/generic-placement-intents/{intent-name}/app-intents", appIntentHandler.createAppIntentHandler).Methods("POST") router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/generic-placement-intents/{intent-name}/app-intents/{app-intent-name}", appIntentHandler.getAppIntentHandler).Methods("GET") + router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/generic-placement-intents/{intent-name}/app-intents", appIntentHandler.getAllAppIntentsHandler).Methods("GET") + router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/generic-placement-intents/{intent-name}/app-intents/", appIntentHandler.getAllIntentsByAppHandler).Queries("app-name", "{app-name}") router.HandleFunc("/projects/{project-name}/composite-apps/{composite-app-name}/{composite-app-version}/generic-placement-intents/{intent-name}/app-intents/{app-intent-name}", appIntentHandler.deleteAppIntentHandler).Methods("DELETE") - //setting routes for deploymentIntentGroup if deploymentIntentGrpClient == nil { deploymentIntentGrpClient = moduleClient.DeploymentIntentGroup diff --git a/src/orchestrator/api/app_intent_handler.go b/src/orchestrator/api/app_intent_handler.go index ab510c8e..4b3282ef 100644 --- a/src/orchestrator/api/app_intent_handler.go +++ b/src/orchestrator/api/app_intent_handler.go @@ -18,10 +18,12 @@ package api import ( "encoding/json" - "github.com/gorilla/mux" - moduleLib "github.com/onap/multicloud-k8s/src/orchestrator/pkg/module" "io" "net/http" + + "github.com/gorilla/mux" + "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/validation" + moduleLib "github.com/onap/multicloud-k8s/src/orchestrator/pkg/module" ) /* Used to store backend implementation objects @@ -120,6 +122,76 @@ func (h appIntentHandler) getAppIntentHandler(w http.ResponseWriter, r *http.Req } +/* +getAllIntentsByAppHandler handles the URL: +/v2/project/{project-name}/composite-apps/{composite-app-name}/{version}/generic-placement-intent/{intent-name}/app-intents?app-name= +*/ +func (h appIntentHandler) getAllIntentsByAppHandler(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + pList := []string{"project-name", "composite-app-name", "composite-app-version", "intent-name"} + err := validation.IsValidParameterPresent(vars, pList) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + p := vars["project-name"] + ca := vars["composite-app-name"] + v := vars["composite-app-version"] + i := vars["intent-name"] + aN := r.URL.Query().Get("app-name") + if aN == "" { + http.Error(w, "Missing appName in GET request", http.StatusBadRequest) + return + } + + specData, err := h.client.GetAllIntentsByApp(aN, p, ca, v, i) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + err = json.NewEncoder(w).Encode(specData) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + return + +} + +/* getAllAppIntentsHandler handles the URL: +/v2/project/{project-name}/composite-apps/{composite-app-name}/{version}/generic-placement-intent/{intent-name}/app-intents +*/ +func (h appIntentHandler) getAllAppIntentsHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + pList := []string{"project-name", "composite-app-name", "composite-app-version", "intent-name"} + err := validation.IsValidParameterPresent(vars, pList) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + p := vars["project-name"] + ca := vars["composite-app-name"] + v := vars["composite-app-version"] + i := vars["intent-name"] + + applicationsAndClusterInfo, err := h.client.GetAllAppIntents(p, ca, v, i) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + err = json.NewEncoder(w).Encode(applicationsAndClusterInfo) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + return + +} + func (h appIntentHandler) deleteAppIntentHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) diff --git a/src/orchestrator/api/app_profilehandler.go b/src/orchestrator/api/app_profilehandler.go index ef7833de..f2475e23 100644 --- a/src/orchestrator/api/app_profilehandler.go +++ b/src/orchestrator/api/app_profilehandler.go @@ -210,12 +210,12 @@ func (h appProfileHandler) getAppProfileHandler(w http.ResponseWriter, r *http.R http.Error(w, err.Error(), http.StatusInternalServerError) return } - kc_bytes, err := base64.StdEncoding.DecodeString(retAppProfileContent.Profile) + kcBytes, err := base64.StdEncoding.DecodeString(retAppProfileContent.Profile) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - _, err = pw.Write(kc_bytes) + _, err = pw.Write(kcBytes) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -231,12 +231,12 @@ func (h appProfileHandler) getAppProfileHandler(w http.ResponseWriter, r *http.R case "application/octet-stream": w.Header().Set("Content-Type", "application/octet-stream") w.WriteHeader(http.StatusOK) - kc_bytes, err := base64.StdEncoding.DecodeString(retAppProfileContent.Profile) + kcBytes, err := base64.StdEncoding.DecodeString(retAppProfileContent.Profile) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - _, err = w.Write(kc_bytes) + _, err = w.Write(kcBytes) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/src/orchestrator/pkg/appcontext/appcontext_test.go b/src/orchestrator/pkg/appcontext/appcontext_test.go index 3fde3a9e..e539ed7a 100644 --- a/src/orchestrator/pkg/appcontext/appcontext_test.go +++ b/src/orchestrator/pkg/appcontext/appcontext_test.go @@ -82,7 +82,7 @@ func (c *MockRunTimeContext) RtcDeletePair(handle interface{}) error { } func (c *MockRunTimeContext) RtcDeletePrefix(handle interface{}) error { - for k, _ := range c.Items { + for k := range c.Items { delete(c.Items, k) } return c.Err @@ -91,7 +91,7 @@ func (c *MockRunTimeContext) RtcDeletePrefix(handle interface{}) error { func (c *MockRunTimeContext) RtcGetHandles(handle interface{}) ([]interface{}, error) { var keys []interface{} - for k, _ := range c.Items { + for k := range c.Items { keys = append(keys, string(k)) } return keys, c.Err @@ -226,7 +226,7 @@ func TestAddApp(t *testing.T) { key: "/context/9345674458787728/", }, { - label: "Delete returns error case", + label: "Error case for adding app", mockRtcontext: &MockRunTimeContext{Err: pkgerrors.Errorf("Error adding app to run time context:")}, key: "/context/9345674458787728/", expectedError: "Error adding app to run time context:", diff --git a/src/orchestrator/pkg/infra/db/mongo.go b/src/orchestrator/pkg/infra/db/mongo.go index a344aa1c..b33d6c65 100644 --- a/src/orchestrator/pkg/infra/db/mongo.go +++ b/src/orchestrator/pkg/infra/db/mongo.go @@ -49,7 +49,7 @@ type MongoCollection interface { opts ...*options.FindOptions) (*mongo.Cursor, error) UpdateOne(ctx context.Context, filter interface{}, update interface{}, opts ...*options.UpdateOptions) (*mongo.UpdateResult, error) - CountDocuments(ctx context.Context, filter interface{}, + CountDocuments(ctx context.Context, filter interface{}, opts ...*options.CountOptions) (int64, error) } @@ -587,4 +587,3 @@ func (m *MongoStore) Remove(coll string, key Key) error { } return nil } - diff --git a/src/orchestrator/pkg/infra/db/store.go b/src/orchestrator/pkg/infra/db/store.go index e87722cd..9c6532f1 100644 --- a/src/orchestrator/pkg/infra/db/store.go +++ b/src/orchestrator/pkg/infra/db/store.go @@ -59,7 +59,7 @@ type Store interface { // Find the document(s) with key and get the tag values from the document(s) Find(coll string, key Key, tag string) ([][]byte, error) - // Removes the document(s) matching the key if no child reference in collection + // Removes the document(s) matching the key if no child reference in collection Remove(coll string, key Key) error // Remove all the document(s) matching the key diff --git a/src/orchestrator/pkg/infra/validation/validation.go b/src/orchestrator/pkg/infra/validation/validation.go index d744dc3d..0b44a8ba 100644 --- a/src/orchestrator/pkg/infra/validation/validation.go +++ b/src/orchestrator/pkg/infra/validation/validation.go @@ -19,6 +19,7 @@ package validation import ( "archive/tar" "compress/gzip" + "fmt" "io" "net" "regexp" @@ -262,3 +263,22 @@ func IsValidNumber(value, min, max int) []string { } return errs } + +/* +IsValidParameterPresent method takes in a vars map and a array of string parameters +that you expect to be present in the GET request. +Returns Nil if all the parameters are present or else shall return error message. +*/ +func IsValidParameterPresent(vars map[string]string, sp []string) error { + + for i := range sp { + v := vars[sp[i]] + if v == "" { + errMessage := fmt.Sprintf("Missing %v in GET request", sp[i]) + return fmt.Errorf(errMessage) + } + + } + return nil + +} diff --git a/src/orchestrator/pkg/module/add_intents.go b/src/orchestrator/pkg/module/add_intents.go index 20fba189..609f65ab 100644 --- a/src/orchestrator/pkg/module/add_intents.go +++ b/src/orchestrator/pkg/module/add_intents.go @@ -16,6 +16,11 @@ package module +/* +This files deals with the backend implementation of adding +genericPlacementIntents to deployementIntentGroup +*/ + import ( "encoding/json" "reflect" diff --git a/src/orchestrator/pkg/module/app_intent.go b/src/orchestrator/pkg/module/app_intent.go index a3f4b832..2b2c0fa5 100644 --- a/src/orchestrator/pkg/module/app_intent.go +++ b/src/orchestrator/pkg/module/app_intent.go @@ -16,6 +16,11 @@ package module +/* +This file deals with the backend implementation of +Adding/Querying AppIntents for each application in the composite-app +*/ + import ( "encoding/json" "reflect" @@ -41,13 +46,15 @@ type MetaData struct { // AllOf consists of AnyOfArray and ClusterNames array type AllOf struct { + ProviderName string `json:"provider-name,omitempty"` ClusterName string `json:"cluster-name,omitempty"` ClusterLabelName string `json:"cluster-label-name,omitempty"` AnyOfArray []AnyOf `json:"anyOf,omitempty"` } -// AnyOf consists of Array of ClusterLabelNames +// AnyOf consists of Array of ProviderName & ClusterLabelNames type AnyOf struct { + ProviderName string `json:"provider-name,omitempty"` ClusterName string `json:"cluster-name,omitempty"` ClusterLabelName string `json:"cluster-label-name,omitempty"` } @@ -69,9 +76,16 @@ type SpecData struct { type AppIntentManager interface { CreateAppIntent(a AppIntent, p string, ca string, v string, i string) (AppIntent, error) GetAppIntent(ai string, p string, ca string, v string, i string) (AppIntent, error) + GetAllIntentsByApp(aN, p, ca, v, i string) (SpecData, error) + GetAllAppIntents(p, ca, v, i string) (ApplicationsAndClusterInfo, error) DeleteAppIntent(ai string, p string, ca string, v string, i string) error } +//AppIntentQueryKey required for query +type AppIntentQueryKey struct { + AppName string `json:"app-name"` +} + // AppIntentKey is used as primary key type AppIntentKey struct { Name string `json:"appintent"` @@ -81,6 +95,28 @@ type AppIntentKey struct { Intent string `json:"genericplacement"` } +// AppIntentFindByAppKey required for query +type AppIntentFindByAppKey struct { + Project string `json:"project"` + CompositeApp string `json:"compositeapp"` + CompositeAppVersion string `json:"compositeappversion"` + Intent string `json:"genericplacement"` + AppName string `json:"app-name"` +} + +// ApplicationsAndClusterInfo type represents the list of +type ApplicationsAndClusterInfo struct { + ArrayOfAppClusterInfo []AppClusterInfo `json:"applications"` +} + +// AppClusterInfo is a type linking the app and the clusters +// on which they need to be installed. +type AppClusterInfo struct { + Name string `json:"name"` + AllOfArray []AllOf `json:"allOf,omitempty"` + AnyOfArray []AnyOf `json:"anyOf,omitempty"` +} + // We will use json marshalling to convert to string to // preserve the underlying structure. func (ak AppIntentKey) String() string { @@ -105,7 +141,8 @@ func NewAppIntentClient() *AppIntentClient { } } -// CreateAppIntent creates an entry for AppIntent in the db. Other input parameters for it - projectName, compositeAppName, version, intentName. +// CreateAppIntent creates an entry for AppIntent in the db. +// Other input parameters for it - projectName, compositeAppName, version, intentName. func (c *AppIntentClient) CreateAppIntent(a AppIntent, p string, ca string, v string, i string) (AppIntent, error) { //Check for the AppIntent already exists here. @@ -140,7 +177,11 @@ func (c *AppIntentClient) CreateAppIntent(a AppIntent, p string, ca string, v st Intent: i, } - err = db.DBconn.Insert(c.storeName, akey, nil, c.tagMetaData, a) + qkey := AppIntentQueryKey{ + AppName: a.Spec.AppName, + } + + err = db.DBconn.Insert(c.storeName, akey, qkey, c.tagMetaData, a) if err != nil { return AppIntent{}, pkgerrors.Wrap(err, "Create DB entry error") } @@ -176,6 +217,71 @@ func (c *AppIntentClient) GetAppIntent(ai string, p string, ca string, v string, return AppIntent{}, pkgerrors.New("Error getting AppIntent") } +/* +GetAllIntentsByApp takes in parameters AppName, CompositeAppName, CompositeNameVersion +and GenericPlacementIntentName. Returns SpecData which contains +all the intents for the app. +*/ +func (c *AppIntentClient) GetAllIntentsByApp(aN, p, ca, v, i string) (SpecData, error) { + k := AppIntentFindByAppKey{ + Project: p, + CompositeApp: ca, + CompositeAppVersion: v, + Intent: i, + AppName: aN, + } + result, err := db.DBconn.Find(c.storeName, k, c.tagMetaData) + if err != nil { + return SpecData{}, pkgerrors.Wrap(err, "Get SpecData error") + } + var a AppIntent + err = db.DBconn.Unmarshal(result[0], &a) + if err != nil { + return SpecData{}, pkgerrors.Wrap(err, "Unmarshalling SpecData") + } + return a.Spec, nil + +} + +/* +GetAllAppIntents takes in paramaters ProjectName, CompositeAppName, CompositeNameVersion +and GenericPlacementIntentName. Returns the ApplicationsAndClusterInfo Object - an array of AppClusterInfo +*/ +func (c *AppIntentClient) GetAllAppIntents(p, ca, v, i string) (ApplicationsAndClusterInfo, error) { + k := AppIntentKey{ + Name: "", + Project: p, + CompositeApp: ca, + Version: v, + Intent: i, + } + result, err := db.DBconn.Find(c.storeName, k, c.tagMetaData) + if err != nil { + return ApplicationsAndClusterInfo{}, pkgerrors.Wrap(err, "Get AppClusterInfo error") + } + + var a AppIntent + var appClusterInfoArray []AppClusterInfo + + if len(result) != 0 { + for i := range result { + a = AppIntent{} + err = db.DBconn.Unmarshal(result[i], &a) + if err != nil { + return ApplicationsAndClusterInfo{}, pkgerrors.Wrap(err, "Unmarshalling AppIntent") + } + appName := a.Spec.AppName + allOfArray := a.Spec.Intent.AllOfArray + anyOfArray := a.Spec.Intent.AnyOfArray + appClusterInfo := AppClusterInfo{appName, allOfArray, + anyOfArray} + appClusterInfoArray = append(appClusterInfoArray, appClusterInfo) + } + } + applicationsAndClusterInfo := ApplicationsAndClusterInfo{appClusterInfoArray} + return applicationsAndClusterInfo, err +} + // DeleteAppIntent delete an AppIntent func (c *AppIntentClient) DeleteAppIntent(ai string, p string, ca string, v string, i string) error { k := AppIntentKey{ diff --git a/src/orchestrator/pkg/module/app_intent_test.go b/src/orchestrator/pkg/module/app_intent_test.go index 6cbdf15f..726ce0a3 100644 --- a/src/orchestrator/pkg/module/app_intent_test.go +++ b/src/orchestrator/pkg/module/app_intent_test.go @@ -50,17 +50,21 @@ func TestCreateAppIntent(t *testing.T) { Intent: IntentStruc{ AllOfArray: []AllOf{ { - ClusterName: "edge1", + ProviderName: "aws", + ClusterName: "edge1", //ClusterLabelName: "edge1", }, { - ClusterName: "edge2", + ProviderName: "aws", + ClusterName: "edge2", //ClusterLabelName: "edge2", }, { AnyOfArray: []AnyOf{ - {ClusterLabelName: "east-us1"}, - {ClusterLabelName: "east-us2"}, + {ProviderName: "aws", + ClusterLabelName: "east-us1"}, + {ProviderName: "aws", + ClusterLabelName: "east-us2"}, //{ClusterName: "east-us1"}, //{ClusterName: "east-us2"}, }, @@ -88,17 +92,21 @@ func TestCreateAppIntent(t *testing.T) { Intent: IntentStruc{ AllOfArray: []AllOf{ { - ClusterName: "edge1", + ProviderName: "aws", + ClusterName: "edge1", //ClusterLabelName: "edge1", }, { - ClusterName: "edge2", + ProviderName: "aws", + ClusterName: "edge2", //ClusterLabelName: "edge2", }, { AnyOfArray: []AnyOf{ - {ClusterLabelName: "east-us1"}, - {ClusterLabelName: "east-us2"}, + {ProviderName: "aws", + ClusterLabelName: "east-us1"}, + {ProviderName: "aws", + ClusterLabelName: "east-us2"}, //{ClusterName: "east-us1"}, //{ClusterName: "east-us2"}, }, @@ -197,15 +205,19 @@ func TestGetAppIntent(t *testing.T) { Intent: IntentStruc{ AllOfArray: []AllOf{ { - ClusterName: "edge1", + ProviderName: "aws", + ClusterName: "edge1", }, { - ClusterName: "edge2", + ProviderName: "aws", + ClusterName: "edge2", }, { AnyOfArray: []AnyOf{ - {ClusterLabelName: "east-us1"}, - {ClusterLabelName: "east-us2"}, + {ProviderName: "aws", + ClusterLabelName: "east-us1"}, + {ProviderName: "aws", + ClusterLabelName: "east-us2"}, }, }, }, @@ -230,13 +242,19 @@ func TestGetAppIntent(t *testing.T) { "\"spec\":{\"app-name\": \"SampleApp\"," + "\"intent\": {" + "\"allOf\":[" + - "{\"cluster-name\":\"edge1\"}," + - "{\"cluster-name\":\"edge2\"}," + + "{" + + "\"provider-name\":\"aws\"," + + "\"cluster-name\":\"edge1\"}," + + "{" + + "\"provider-name\":\"aws\"," + + "\"cluster-name\":\"edge2\"}," + "{" + "\"anyOf\":[" + "{" + + "\"provider-name\":\"aws\"," + "\"cluster-label-name\":\"east-us1\"}," + "{" + + "\"provider-name\":\"aws\"," + "\"cluster-label-name\":\"east-us2\"}" + "]}]" + "}}}"), diff --git a/src/orchestrator/pkg/module/app_profile.go b/src/orchestrator/pkg/module/app_profile.go index 77835fb4..6600ed24 100644 --- a/src/orchestrator/pkg/module/app_profile.go +++ b/src/orchestrator/pkg/module/app_profile.go @@ -122,7 +122,7 @@ func (c *AppProfileClient) CreateAppProfile(project, compositeApp, compositeAppV //Check if composite profile exists (success assumes existance of all higher level 'parent' objects) _, err = NewCompositeProfileClient().GetCompositeProfile(compositeProfile, project, compositeApp, compositeAppVersion) if err != nil { - return AppProfile{}, pkgerrors.New("Unable to find the project") + return AppProfile{}, pkgerrors.New("Unable to find the compositeProfile") } // TODO: (after app api is ready) check that the app Spec.AppName exists as part of the composite app -- cgit 1.2.3-korg