From d203ccf9174e450b6f5a6a9aabea95b73c9973ff Mon Sep 17 00:00:00 2001 From: Kiran Kamineni Date: Mon, 10 Dec 2018 13:34:16 -0800 Subject: Migrating from consul to mongodb for backend Migrating to mongodb from consul. The main reason being the value size limitation of 512kb in consul. See https://jira.onap.org/browse/MULTICLOUD-426 for details. This requires a little bit of hierarchy management and data management. We are no longer converting structs to json encoded strings. The underlying db supports structs without any modifications. Also, since Mongo has the concept of collections, each submodule can use its own collection for storage as needed. Definition uses a collection called rbdef right now. P10: Enabling unit tests for mongo.go. This requires the usage of aliased functions. P11: Expanded unit tests for all functions in mongo.go P12: Refactored parameter validation. Removed TestHealthCheck as we are not mocking any of the db commands right now Checking return value of read with an expected value P13: Adding back consul support. Fixing functional test Full consul implementation check and modifications is being tracked by MULTICLOUD-427 P15: Fix ReadAll unit test and corresponding code ReadAll now returns error when no objects are found Issue-ID: MULTICLOUD-426 Change-Id: I42d239b324025fc4ef4e561790aceeff794001ef Signed-off-by: Kiran Kamineni --- src/k8splugin/api/handler.go | 55 +++++++-------------- src/k8splugin/api/handler_test.go | 101 +++++++++++--------------------------- 2 files changed, 46 insertions(+), 110 deletions(-) (limited to 'src/k8splugin/api') diff --git a/src/k8splugin/api/handler.go b/src/k8splugin/api/handler.go index 53fa2317..4c49ba78 100644 --- a/src/k8splugin/api/handler.go +++ b/src/k8splugin/api/handler.go @@ -30,6 +30,10 @@ import ( "k8splugin/krd" ) +//TODO: Separate the http handler code and backend code out +var storeName = "rbinst" +var tagData = "data" + // GetVNFClient retrieves the client used to communicate with a Kubernetes Cluster var GetVNFClient = func(kubeConfigPath string) (kubernetes.Clientset, error) { client, err := krd.GetKubeClient(kubeConfigPath) @@ -117,17 +121,9 @@ func CreateHandler(w http.ResponseWriter, r *http.Request) { // TODO: Uncomment when annotations are done // krd.AddNetworkAnnotationsToPod(kubeData, resource.Networks) - // "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.Serialize(resourceNameMap) - if err != nil { - werr := pkgerrors.Wrap(err, "Create VNF deployment JSON Marshalling error") - http.Error(w, werr.Error(), http.StatusInternalServerError) - return - } - // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - err = db.DBconn.Create(internalVNFID, serializedResourceNameMap) + err = db.DBconn.Create(storeName, internalVNFID, tagData, resourceNameMap) if err != nil { werr := pkgerrors.Wrap(err, "Create VNF deployment DB error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -154,27 +150,22 @@ func ListHandler(w http.ResponseWriter, r *http.Request) { namespace := vars["namespace"] prefix := cloudRegionID + "-" + namespace - internalVNFIDs, err := db.DBconn.ReadAll(prefix) + res, err := db.DBconn.ReadAll(storeName, tagData) if err != nil { http.Error(w, pkgerrors.Wrap(err, "Get VNF list error").Error(), http.StatusInternalServerError) return } - if len(internalVNFIDs) == 0 { - w.WriteHeader(http.StatusNotFound) - return - } - // TODO: There is an edge case where if namespace is passed but is missing some characters // trailing, it will print the result with those excluding characters. This is because of // the way I am trimming the Prefix. This fix is needed. var editedList []string - for _, id := range internalVNFIDs { - if len(id) > 0 { - editedList = append(editedList, strings.TrimPrefix(id, prefix)[1:]) + for key, value := range res { + if len(value) > 0 { + editedList = append(editedList, strings.TrimPrefix(key, prefix)[1:]) } } @@ -204,25 +195,20 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) { // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.DBconn.Read(internalVNFID) + res, err := db.DBconn.Read(storeName, internalVNFID, tagData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if serializedResourceNameMap == "" { - w.WriteHeader(http.StatusNotFound) - return - } - /* { "deployment": ["cloud1-default-uuid-sisedeploy1", "cloud1-default-uuid-sisedeploy2", ... ] "service": ["cloud1-default-uuid-sisesvc1", "cloud1-default-uuid-sisesvc2", ... ] }, */ - deserializedResourceNameMap := make(map[string][]string) - err = db.DeSerialize(serializedResourceNameMap, &deserializedResourceNameMap) + data := make(map[string][]string) + err = db.DBconn.Unmarshal(res, &data) if err != nil { werr := pkgerrors.Wrap(err, "Unmarshal VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -237,14 +223,14 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - err = csar.DestroyVNF(deserializedResourceNameMap, namespace, &kubeclient) + err = csar.DestroyVNF(data, namespace, &kubeclient) if err != nil { werr := pkgerrors.Wrap(err, "Delete VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) return } - err = db.DBconn.Delete(internalVNFID) + err = db.DBconn.Delete(storeName, internalVNFID, tagData) if err != nil { werr := pkgerrors.Wrap(err, "Delete VNF db record error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -337,25 +323,20 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.DBconn.Read(internalVNFID) + res, err := db.DBconn.Read(storeName, internalVNFID, tagData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if serializedResourceNameMap == "" { - w.WriteHeader(http.StatusNotFound) - return - } - /* { "deployment": ["cloud1-default-uuid-sisedeploy1", "cloud1-default-uuid-sisedeploy2", ... ] "service": ["cloud1-default-uuid-sisesvc1", "cloud1-default-uuid-sisesvc2", ... ] }, */ - deserializedResourceNameMap := make(map[string][]string) - err = db.DeSerialize(serializedResourceNameMap, &deserializedResourceNameMap) + data := make(map[string][]string) + err = db.DBconn.Unmarshal(res, &data) if err != nil { werr := pkgerrors.Wrap(err, "Unmarshal VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -366,7 +347,7 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { VNFID: externalVNFID, CloudRegionID: cloudRegionID, Namespace: namespace, - VNFComponents: deserializedResourceNameMap, + VNFComponents: data, } w.Header().Set("Content-Type", "application/json") diff --git a/src/k8splugin/api/handler_test.go b/src/k8splugin/api/handler_test.go index 3336bbc2..a3aeff7a 100644 --- a/src/k8splugin/api/handler_test.go +++ b/src/k8splugin/api/handler_test.go @@ -24,7 +24,6 @@ import ( "reflect" "testing" - "github.com/hashicorp/consul/api" pkgerrors "github.com/pkg/errors" "k8s.io/client-go/kubernetes" @@ -193,38 +192,19 @@ func TestListHandler(t *testing.T) { Err: pkgerrors.New("Internal error"), }, }, - { - label: "Get result from DB non-records", - expectedCode: http.StatusNotFound, - mockStore: &db.MockDB{}, - }, { label: "Get empty list", expectedCode: http.StatusOK, expectedResponse: []string{""}, - mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "", - Value: []byte("{}"), - }, - }, - }, + mockStore: &db.MockDB{}, }, { label: "Succesful get a list of VNF", expectedCode: http.StatusOK, - expectedResponse: []string{"uid1", "uid2"}, + expectedResponse: []string{"uid1"}, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "uuid1", - Value: []byte("{}"), - }, - &api.KVPair{ - Key: "uuid2", - Value: []byte("{}"), - }, + Items: map[string][]byte{ + "uuid1": []byte("{}"), }, }, }, @@ -275,20 +255,17 @@ func TestDeleteHandler(t *testing.T) { }, { label: "Fail to find VNF record be deleted", - expectedCode: http.StatusNotFound, + expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{}, + Items: map[string][]byte{}, }, }, { label: "Fail to unmarshal the DB record", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{invalid format}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte("{invalid format}"), }, }, }, @@ -297,14 +274,10 @@ func TestDeleteHandler(t *testing.T) { expectedCode: http.StatusInternalServerError, mockGetVNFClientErr: pkgerrors.New("Get VNF client error"), mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, }, @@ -312,14 +285,10 @@ func TestDeleteHandler(t *testing.T) { label: "Fail to destroy VNF", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, mockDeleteVNF: &mockCSAR{ @@ -330,14 +299,10 @@ func TestDeleteHandler(t *testing.T) { label: "Succesful delete a VNF", expectedCode: http.StatusAccepted, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, mockDeleteVNF: &mockCSAR{}, @@ -440,18 +405,15 @@ func TestGetHandler(t *testing.T) { }, { label: "Not found DB record", - expectedCode: http.StatusNotFound, + expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{}, }, { label: "Fail to unmarshal the DB record", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloud1-default-1", - Value: []byte("{invalid-format}"), - }, + Items: map[string][]byte{ + "cloud1-default-1": []byte("{invalid-format}"), }, }, }, @@ -468,18 +430,11 @@ func TestGetHandler(t *testing.T) { }, }, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloud1-default-1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, - &api.KVPair{ - Key: "cloud1-default-2", - Value: []byte("{}"), - }, + Items: map[string][]byte{ + "cloud1-default-1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), + "cloud1-default-2": []byte("{}"), }, }, }, -- cgit 1.2.3-korg