diff options
author | Kiran Kamineni <kiran.k.kamineni@intel.com> | 2018-12-10 13:34:16 -0800 |
---|---|---|
committer | Kiran Kamineni <kiran.k.kamineni@intel.com> | 2018-12-14 14:26:26 -0800 |
commit | d203ccf9174e450b6f5a6a9aabea95b73c9973ff (patch) | |
tree | 702034eafbd28b47f33023607ee213cf92519a99 | |
parent | aa56022e0fba3c358e46e8671d9a0cd36094ebaa (diff) |
Migrating from consul to mongodb for backend
Migrating to mongodb from consul.
The main reason being the value size limitation of 512kb in consul.
See https://jira.onap.org/browse/MULTICLOUD-426 for details.
This requires a little bit of hierarchy management
and data management.
We are no longer converting structs to json encoded
strings. The underlying db supports structs without any modifications.
Also, since Mongo has the concept of collections, each submodule can
use its own collection for storage as needed.
Definition uses a collection called rbdef right now.
P10: Enabling unit tests for mongo.go. This requires the usage
of aliased functions.
P11: Expanded unit tests for all functions in mongo.go
P12: Refactored parameter validation.
Removed TestHealthCheck as we are not mocking any of the
db commands right now
Checking return value of read with an expected value
P13: Adding back consul support.
Fixing functional test
Full consul implementation check and modifications is
being tracked by MULTICLOUD-427
P15: Fix ReadAll unit test and corresponding code
ReadAll now returns error when no objects are found
Issue-ID: MULTICLOUD-426
Change-Id: I42d239b324025fc4ef4e561790aceeff794001ef
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>
-rw-r--r-- | deployments/docker-compose.yml | 15 | ||||
-rwxr-xr-x | deployments/start.sh | 8 | ||||
-rw-r--r-- | src/k8splugin/api/handler.go | 55 | ||||
-rw-r--r-- | src/k8splugin/api/handler_test.go | 101 | ||||
-rw-r--r-- | src/k8splugin/db/consul.go | 38 | ||||
-rw-r--r-- | src/k8splugin/db/consul_test.go | 63 | ||||
-rw-r--r-- | src/k8splugin/db/mongo.go | 323 | ||||
-rw-r--r-- | src/k8splugin/db/mongo_test.go | 530 | ||||
-rw-r--r-- | src/k8splugin/db/store.go | 27 | ||||
-rw-r--r-- | src/k8splugin/db/store_test.go | 4 | ||||
-rw-r--r-- | src/k8splugin/db/testing.go | 42 | ||||
-rw-r--r-- | src/k8splugin/go.mod | 60 | ||||
-rw-r--r-- | src/k8splugin/go.sum | 46 | ||||
-rw-r--r-- | src/k8splugin/rb/definition.go | 65 | ||||
-rw-r--r-- | src/k8splugin/rb/definition_test.go | 47 | ||||
-rwxr-xr-x | vagrant/tests/plugin.sh | 2 |
16 files changed, 1136 insertions, 290 deletions
diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml index 73d5651c..a72bd096 100644 --- a/deployments/docker-compose.yml +++ b/deployments/docker-compose.yml @@ -28,33 +28,28 @@ services: environment: - CSAR_DIR=/opt/csar - KUBE_CONFIG_DIR=/opt/kubeconfig - - DATABASE_TYPE=consul + - DATABASE_TYPE=mongo - DATABASE_IP=172.19.0.2 - PLUGINS_DIR=/opt/multicloud/k8s - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY,172.19.0.2 depends_on: - - consul + - mongo links: - - consul + - mongo volumes: - /opt/csar:/opt/csar - /opt/kubeconfig:/opt/kubeconfig - consul: - image: consul + mongo: + image: mongo networks: multicloud_net: ipv4_address: 172.19.0.2 environment: - CONSUL_CLIENT_INTERFACE: 'eth0' - CONSUL_BIND_INTERFACE: 'eth0' HTTP_PROXY: $HTTP_PROXY HTTPS_PROXY: $HTTPS_PROXY NO_PROXY: $NO_PROXY - command: ["agent", "-server", "-bootstrap-expect=1"] - volumes: - - /opt/consul/config:/consul/config networks: multicloud_net: diff --git a/deployments/start.sh b/deployments/start.sh index da2eacee..d1b9f68a 100755 --- a/deployments/start.sh +++ b/deployments/start.sh @@ -19,13 +19,13 @@ export IMAGE_NAME="nexus3.onap.org:10003/onap/multicloud/k8s" export CSAR_DIR=/opt/csar export KUBE_CONFIG_DIR=/opt/kubeconfig -export DATABASE_TYPE=consul +export DATABASE_TYPE=mongo export PLUGINS_DIR=$k8s_path/src/k8splugin/plugins -echo "Starting consul services" +echo "Starting mongo services" docker-compose kill -docker-compose up -d consul -export DATABASE_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=consul")) +docker-compose up -d mongo +export DATABASE_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=mongo")) export no_proxy=$no_proxy,$DATABASE_IP export NO_PROXY=$NO_PROXY,$DATABASE_IP diff --git a/src/k8splugin/api/handler.go b/src/k8splugin/api/handler.go index 53fa2317..4c49ba78 100644 --- a/src/k8splugin/api/handler.go +++ b/src/k8splugin/api/handler.go @@ -30,6 +30,10 @@ import ( "k8splugin/krd" ) +//TODO: Separate the http handler code and backend code out +var storeName = "rbinst" +var tagData = "data" + // GetVNFClient retrieves the client used to communicate with a Kubernetes Cluster var GetVNFClient = func(kubeConfigPath string) (kubernetes.Clientset, error) { client, err := krd.GetKubeClient(kubeConfigPath) @@ -117,17 +121,9 @@ func CreateHandler(w http.ResponseWriter, r *http.Request) { // TODO: Uncomment when annotations are done // krd.AddNetworkAnnotationsToPod(kubeData, resource.Networks) - // "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.Serialize(resourceNameMap) - if err != nil { - werr := pkgerrors.Wrap(err, "Create VNF deployment JSON Marshalling error") - http.Error(w, werr.Error(), http.StatusInternalServerError) - return - } - // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - err = db.DBconn.Create(internalVNFID, serializedResourceNameMap) + err = db.DBconn.Create(storeName, internalVNFID, tagData, resourceNameMap) if err != nil { werr := pkgerrors.Wrap(err, "Create VNF deployment DB error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -154,27 +150,22 @@ func ListHandler(w http.ResponseWriter, r *http.Request) { namespace := vars["namespace"] prefix := cloudRegionID + "-" + namespace - internalVNFIDs, err := db.DBconn.ReadAll(prefix) + res, err := db.DBconn.ReadAll(storeName, tagData) if err != nil { http.Error(w, pkgerrors.Wrap(err, "Get VNF list error").Error(), http.StatusInternalServerError) return } - if len(internalVNFIDs) == 0 { - w.WriteHeader(http.StatusNotFound) - return - } - // TODO: There is an edge case where if namespace is passed but is missing some characters // trailing, it will print the result with those excluding characters. This is because of // the way I am trimming the Prefix. This fix is needed. var editedList []string - for _, id := range internalVNFIDs { - if len(id) > 0 { - editedList = append(editedList, strings.TrimPrefix(id, prefix)[1:]) + for key, value := range res { + if len(value) > 0 { + editedList = append(editedList, strings.TrimPrefix(key, prefix)[1:]) } } @@ -204,25 +195,20 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) { // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.DBconn.Read(internalVNFID) + res, err := db.DBconn.Read(storeName, internalVNFID, tagData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if serializedResourceNameMap == "" { - w.WriteHeader(http.StatusNotFound) - return - } - /* { "deployment": ["cloud1-default-uuid-sisedeploy1", "cloud1-default-uuid-sisedeploy2", ... ] "service": ["cloud1-default-uuid-sisesvc1", "cloud1-default-uuid-sisesvc2", ... ] }, */ - deserializedResourceNameMap := make(map[string][]string) - err = db.DeSerialize(serializedResourceNameMap, &deserializedResourceNameMap) + data := make(map[string][]string) + err = db.DBconn.Unmarshal(res, &data) if err != nil { werr := pkgerrors.Wrap(err, "Unmarshal VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -237,14 +223,14 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - err = csar.DestroyVNF(deserializedResourceNameMap, namespace, &kubeclient) + err = csar.DestroyVNF(data, namespace, &kubeclient) if err != nil { werr := pkgerrors.Wrap(err, "Delete VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) return } - err = db.DBconn.Delete(internalVNFID) + err = db.DBconn.Delete(storeName, internalVNFID, tagData) if err != nil { werr := pkgerrors.Wrap(err, "Delete VNF db record error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -337,25 +323,20 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.DBconn.Read(internalVNFID) + res, err := db.DBconn.Read(storeName, internalVNFID, tagData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if serializedResourceNameMap == "" { - w.WriteHeader(http.StatusNotFound) - return - } - /* { "deployment": ["cloud1-default-uuid-sisedeploy1", "cloud1-default-uuid-sisedeploy2", ... ] "service": ["cloud1-default-uuid-sisesvc1", "cloud1-default-uuid-sisesvc2", ... ] }, */ - deserializedResourceNameMap := make(map[string][]string) - err = db.DeSerialize(serializedResourceNameMap, &deserializedResourceNameMap) + data := make(map[string][]string) + err = db.DBconn.Unmarshal(res, &data) if err != nil { werr := pkgerrors.Wrap(err, "Unmarshal VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -366,7 +347,7 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { VNFID: externalVNFID, CloudRegionID: cloudRegionID, Namespace: namespace, - VNFComponents: deserializedResourceNameMap, + VNFComponents: data, } w.Header().Set("Content-Type", "application/json") diff --git a/src/k8splugin/api/handler_test.go b/src/k8splugin/api/handler_test.go index 3336bbc2..a3aeff7a 100644 --- a/src/k8splugin/api/handler_test.go +++ b/src/k8splugin/api/handler_test.go @@ -24,7 +24,6 @@ import ( "reflect" "testing" - "github.com/hashicorp/consul/api" pkgerrors "github.com/pkg/errors" "k8s.io/client-go/kubernetes" @@ -194,37 +193,18 @@ func TestListHandler(t *testing.T) { }, }, { - label: "Get result from DB non-records", - expectedCode: http.StatusNotFound, - mockStore: &db.MockDB{}, - }, - { label: "Get empty list", expectedCode: http.StatusOK, expectedResponse: []string{""}, - mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "", - Value: []byte("{}"), - }, - }, - }, + mockStore: &db.MockDB{}, }, { label: "Succesful get a list of VNF", expectedCode: http.StatusOK, - expectedResponse: []string{"uid1", "uid2"}, + expectedResponse: []string{"uid1"}, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "uuid1", - Value: []byte("{}"), - }, - &api.KVPair{ - Key: "uuid2", - Value: []byte("{}"), - }, + Items: map[string][]byte{ + "uuid1": []byte("{}"), }, }, }, @@ -275,20 +255,17 @@ func TestDeleteHandler(t *testing.T) { }, { label: "Fail to find VNF record be deleted", - expectedCode: http.StatusNotFound, + expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{}, + Items: map[string][]byte{}, }, }, { label: "Fail to unmarshal the DB record", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{invalid format}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte("{invalid format}"), }, }, }, @@ -297,14 +274,10 @@ func TestDeleteHandler(t *testing.T) { expectedCode: http.StatusInternalServerError, mockGetVNFClientErr: pkgerrors.New("Get VNF client error"), mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, }, @@ -312,14 +285,10 @@ func TestDeleteHandler(t *testing.T) { label: "Fail to destroy VNF", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, mockDeleteVNF: &mockCSAR{ @@ -330,14 +299,10 @@ func TestDeleteHandler(t *testing.T) { label: "Succesful delete a VNF", expectedCode: http.StatusAccepted, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, mockDeleteVNF: &mockCSAR{}, @@ -440,18 +405,15 @@ func TestGetHandler(t *testing.T) { }, { label: "Not found DB record", - expectedCode: http.StatusNotFound, + expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{}, }, { label: "Fail to unmarshal the DB record", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloud1-default-1", - Value: []byte("{invalid-format}"), - }, + Items: map[string][]byte{ + "cloud1-default-1": []byte("{invalid-format}"), }, }, }, @@ -468,18 +430,11 @@ func TestGetHandler(t *testing.T) { }, }, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloud1-default-1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, - &api.KVPair{ - Key: "cloud1-default-2", - Value: []byte("{}"), - }, + Items: map[string][]byte{ + "cloud1-default-1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), + "cloud1-default-2": []byte("{}"), }, }, }, diff --git a/src/k8splugin/db/consul.go b/src/k8splugin/db/consul.go index d7507242..a61a4c10 100644 --- a/src/k8splugin/db/consul.go +++ b/src/k8splugin/db/consul.go @@ -54,50 +54,64 @@ func NewConsulStore(store ConsulKVStore) (Store, error) { // HealthCheck verifies if the database is up and running func (c *ConsulStore) HealthCheck() error { - _, err := c.Read("test") + _, err := c.Read("test", "test", "test") if err != nil { return pkgerrors.New("[ERROR] Cannot talk to Datastore. Check if it is running/reachable.") } return nil } +// Unmarshal implements any unmarshaling that is needed when using consul +func (c *ConsulStore) Unmarshal(inp []byte, out interface{}) error { + return nil +} + // Create is used to create a DB entry -func (c *ConsulStore) Create(key, value string) error { +func (c *ConsulStore) Create(root, key, tag string, data interface{}) error { + + value, err := Serialize(data) + if err != nil { + return pkgerrors.Wrap(err, "Serializing input data") + } + p := &api.KVPair{ Key: key, Value: []byte(value), } - _, err := c.client.Put(p, nil) + _, err = c.client.Put(p, nil) return err } // Read method returns the internalID for a particular externalID -func (c *ConsulStore) Read(key string) (string, error) { +func (c *ConsulStore) Read(root, key, tag string) ([]byte, error) { + key = root + "/" + key + "/" + tag pair, _, err := c.client.Get(key, nil) if err != nil { - return "", err + return nil, err } if pair == nil { - return "", nil + return nil, nil } - return string(pair.Value), nil + return pair.Value, nil } // Delete method removes an internalID from the Database -func (c *ConsulStore) Delete(key string) error { +func (c *ConsulStore) Delete(root, key, tag string) error { _, err := c.client.Delete(key, nil) return err } // ReadAll is used to get all ExternalIDs in a namespace -func (c *ConsulStore) ReadAll(prefix string) ([]string, error) { - pairs, _, err := c.client.List(prefix, nil) +func (c *ConsulStore) ReadAll(root, tag string) (map[string][]byte, error) { + pairs, _, err := c.client.List(root, nil) if err != nil { return nil, err } - var result []string + + //TODO: Filter results by tag and return it + result := make(map[string][]byte) for _, keypair := range pairs { - result = append(result, keypair.Key) + result[keypair.Key] = keypair.Value } return result, nil diff --git a/src/k8splugin/db/consul_test.go b/src/k8splugin/db/consul_test.go index ede1a5e9..754112ad 100644 --- a/src/k8splugin/db/consul_test.go +++ b/src/k8splugin/db/consul_test.go @@ -107,12 +107,14 @@ func TestConsulCreate(t *testing.T) { }{ { label: "Sucessful register a record to Consul Database", - input: map[string]string{"key": "test-key", "value": "test-value"}, - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data", "value": "test-value"}, + mock: &mockConsulKVStore{}, }, { label: "Fail to create a new record in Consul Database", - input: map[string]string{"key": "test-key", "value": "test-value"}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data", "value": "test-value"}, mock: &mockConsulKVStore{ Err: pkgerrors.New("DB error"), }, @@ -123,7 +125,8 @@ func TestConsulCreate(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - err := client.Create(testCase.input["key"], testCase.input["value"]) + err := client.Create(testCase.input["root"], testCase.input["key"], + testCase.input["tag"], testCase.input["value"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("Create method return an un-expected (%s)", err) @@ -139,18 +142,19 @@ func TestConsulCreate(t *testing.T) { func TestConsulRead(t *testing.T) { testCases := []struct { label string - input string + input map[string]string mock *mockConsulKVStore expectedError string expectedResult string }{ { label: "Sucessful retrieve a record from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test", + "tag": "data"}, mock: &mockConsulKVStore{ Items: api.KVPairs{ &api.KVPair{ - Key: "test", + Key: "rbinst/test/data", Value: []byte("test-value"), }, }, @@ -159,12 +163,14 @@ func TestConsulRead(t *testing.T) { }, { label: "Fail retrieve a non-existing record from Consul Database", - input: "test", - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, + mock: &mockConsulKVStore{}, }, { label: "Fail retrieve a record from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, mock: &mockConsulKVStore{ Err: pkgerrors.New("DB error"), }, @@ -175,7 +181,8 @@ func TestConsulRead(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - result, err := client.Read(testCase.input) + result, err := client.Read(testCase.input["root"], testCase.input["key"], + testCase.input["tag"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("Read method return an un-expected (%s)", err) @@ -187,7 +194,7 @@ func TestConsulRead(t *testing.T) { if testCase.expectedError != "" && testCase.expectedResult == "" { t.Fatalf("Read method was expecting \"%s\" error message", testCase.expectedError) } - if !reflect.DeepEqual(testCase.expectedResult, result) { + if !reflect.DeepEqual(testCase.expectedResult, string(result)) { t.Fatalf("Read method returned: \n%v\n and it was expected: \n%v", result, testCase.expectedResult) } @@ -199,14 +206,15 @@ func TestConsulRead(t *testing.T) { func TestConsulDelete(t *testing.T) { testCases := []struct { label string - input string + input map[string]string mock *mockConsulKVStore expectedError string }{ { label: "Sucessful delete a record to Consul Database", - input: "test", - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, + mock: &mockConsulKVStore{}, }, { label: "Fail to delete a record in Consul Database", @@ -220,7 +228,8 @@ func TestConsulDelete(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - err := client.Delete(testCase.input) + err := client.Delete(testCase.input["root"], testCase.input["key"], + testCase.input["tag"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("Delete method return an un-expected (%s)", err) @@ -236,14 +245,15 @@ func TestConsulDelete(t *testing.T) { func TestConsulReadAll(t *testing.T) { testCases := []struct { label string - input string + input map[string]string mock *mockConsulKVStore expectedError string - expectedResult []string + expectedResult map[string][]byte }{ { label: "Sucessful retrieve a list from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, mock: &mockConsulKVStore{ Items: api.KVPairs{ &api.KVPair{ @@ -256,16 +266,20 @@ func TestConsulReadAll(t *testing.T) { }, }, }, - expectedResult: []string{"test", "test2"}, + expectedResult: map[string][]byte{"test": []byte("test-value"), + "test2": []byte("test-value2")}, }, { label: "Sucessful retrieve an empty list from Consul Database", - input: "test", - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, + mock: &mockConsulKVStore{}, + expectedResult: map[string][]byte{}, }, { label: "Fail retrieve a record from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, mock: &mockConsulKVStore{ Err: pkgerrors.New("DB error"), }, @@ -276,7 +290,8 @@ func TestConsulReadAll(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - result, err := client.ReadAll(testCase.input) + result, err := client.ReadAll(testCase.input["root"], + testCase.input["tag"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("ReadAll method return an un-expected (%s)", err) diff --git a/src/k8splugin/db/mongo.go b/src/k8splugin/db/mongo.go new file mode 100644 index 00000000..311f044c --- /dev/null +++ b/src/k8splugin/db/mongo.go @@ -0,0 +1,323 @@ +/* + * Copyright 2018 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package db + +import ( + "github.com/mongodb/mongo-go-driver/bson" + "github.com/mongodb/mongo-go-driver/bson/primitive" + "github.com/mongodb/mongo-go-driver/mongo" + "github.com/mongodb/mongo-go-driver/mongo/options" + pkgerrors "github.com/pkg/errors" + "golang.org/x/net/context" + "log" + "os" +) + +// MongoCollection defines the a subset of MongoDB operations +// Note: This interface is defined mainly for mock testing +type MongoCollection interface { + InsertOne(ctx context.Context, document interface{}, + opts ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) + FindOne(ctx context.Context, filter interface{}, + opts ...*options.FindOneOptions) *mongo.SingleResult + FindOneAndUpdate(ctx context.Context, filter interface{}, + update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult + DeleteOne(ctx context.Context, filter interface{}, + opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) + Find(ctx context.Context, filter interface{}, + opts ...*options.FindOptions) (mongo.Cursor, error) +} + +// MongoStore is an implementation of the db.Store interface +type MongoStore struct { + db *mongo.Database +} + +// This exists only for allowing us to mock the collection object +// for testing purposes +var getCollection = func(coll string, m *MongoStore) MongoCollection { + return m.db.Collection(coll) +} + +// This exists only for allowing us to mock the DecodeBytes function +// Mainly because we cannot construct a SingleResult struct from our +// tests. All fields in that struct are private. +var decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return sr.DecodeBytes() +} + +// NewMongoStore initializes a Mongo Database with the name provided +// If a database with that name exists, it will be returned +func NewMongoStore(name string, store *mongo.Database) (Store, error) { + if store == nil { + ip := "mongodb://" + os.Getenv("DATABASE_IP") + ":27017" + mongoClient, err := mongo.NewClient(ip) + if err != nil { + return nil, err + } + + err = mongoClient.Connect(context.Background()) + if err != nil { + return nil, err + } + store = mongoClient.Database(name) + } + + return &MongoStore{ + db: store, + }, nil +} + +// HealthCheck verifies if the database is up and running +func (m *MongoStore) HealthCheck() error { + + _, err := decodeBytes(m.db.RunCommand(context.Background(), bson.D{{"serverStatus", 1}})) + if err != nil { + return pkgerrors.Wrap(err, "Error getting server status") + } + + return nil +} + +// validateParams checks to see if any parameters are empty +func (m *MongoStore) validateParams(args ...string) bool { + for _, v := range args { + if v == "" { + return false + } + } + + return true +} + +// Create is used to create a DB entry +func (m *MongoStore) Create(coll, key, tag string, data interface{}) error { + if data == nil || !m.validateParams(coll, key, tag) { + return pkgerrors.New("No Data to store") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Insert the data and then add the objectID to the masterTable + res, err := c.InsertOne(ctx, bson.D{ + {tag, data}, + }) + if err != nil { + return pkgerrors.Errorf("Error inserting into database: %s", err.Error()) + } + + //Add objectID of created data to masterKey document + //Create masterkey document if it does not exist + filter := bson.D{{"key", key}} + + _, err = decodeBytes( + c.FindOneAndUpdate( + ctx, + filter, + bson.D{ + {"$set", bson.D{ + {tag, res.InsertedID}, + }}, + }, + options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After))) + + if err != nil { + return pkgerrors.Errorf("Error updating master table: %s", err.Error()) + } + + return nil +} + +// Unmarshal implements an unmarshaler for bson data that +// is produced from the mongo database +func (m *MongoStore) Unmarshal(inp []byte, out interface{}) error { + err := bson.Unmarshal(inp, out) + if err != nil { + return pkgerrors.Wrap(err, "Unmarshaling bson") + } + return nil +} + +// Read method returns the data stored for this key and for this particular tag +func (m *MongoStore) Read(coll, key, tag string) ([]byte, error) { + if !m.validateParams(coll, key, tag) { + return nil, pkgerrors.New("Mandatory fields are missing") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get the masterkey document based on given key + filter := bson.D{{"key", key}} + keydata, err := decodeBytes(c.FindOne(context.Background(), filter)) + if err != nil { + return nil, pkgerrors.Errorf("Error finding master table: %s", err.Error()) + } + + //Read the tag objectID from document + tagoid, ok := keydata.Lookup(tag).ObjectIDOK() + if !ok { + return nil, pkgerrors.Errorf("Error finding objectID for tag %s", tag) + } + + //Use tag objectID to read the data from store + filter = bson.D{{"_id", tagoid}} + tagdata, err := decodeBytes(c.FindOne(ctx, filter)) + if err != nil { + return nil, pkgerrors.Errorf("Error reading found object: %s", err.Error()) + } + + //Return the data as a byte array + return tagdata.Lookup(tag).Value, nil +} + +// Helper function that deletes an object by its ID +func (m *MongoStore) deleteObjectByID(coll string, objID primitive.ObjectID) error { + + c := getCollection(coll, m) + ctx := context.Background() + + _, err := c.DeleteOne(ctx, bson.D{{"_id", objID}}) + if err != nil { + return pkgerrors.Errorf("Error Deleting from database: %s", err.Error()) + } + + log.Printf("Deleted Obj with ID %s", objID.String()) + return nil +} + +// Delete method removes a document from the Database that matches key +// TODO: delete all referenced docs if tag is empty string +func (m *MongoStore) Delete(coll, key, tag string) error { + if !m.validateParams(coll, key, tag) { + return pkgerrors.New("Mandatory fields are missing") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get the masterkey document based on given key + filter := bson.D{{"key", key}} + //Remove the tag ID entry from masterkey table + update := bson.D{ + { + "$unset", bson.D{ + {tag, ""}, + }, + }, + } + keydata, err := decodeBytes(c.FindOneAndUpdate(ctx, filter, update, + options.FindOneAndUpdate().SetReturnDocument(options.Before))) + if err != nil { + return pkgerrors.Errorf("Error decoding master table after update: %s", + err.Error()) + } + + //Read the tag objectID from document + elems, err := keydata.Elements() + if err != nil { + return pkgerrors.Errorf("Error reading elements from database: %s", err.Error()) + } + + tagoid, ok := keydata.Lookup(tag).ObjectIDOK() + if !ok { + return pkgerrors.Errorf("Error finding objectID for tag %s", tag) + } + + //Use tag objectID to read the data from store + err = m.deleteObjectByID(coll, tagoid) + if err != nil { + return pkgerrors.Errorf("Error deleting from database: %s", err.Error()) + } + + //Delete master table if no more tags left + //_id, key and tag should be elements in before doc + //if master table needs to be removed too + if len(elems) == 3 { + keyid, ok := keydata.Lookup("_id").ObjectIDOK() + if !ok { + return pkgerrors.Errorf("Error finding objectID for key %s", key) + } + err = m.deleteObjectByID(coll, keyid) + if err != nil { + return pkgerrors.Errorf("Error deleting master table from database: %s", err.Error()) + } + } + + return nil +} + +// ReadAll is used to get all documents in db of a particular tag +func (m *MongoStore) ReadAll(coll, tag string) (map[string][]byte, error) { + if !m.validateParams(coll, tag) { + return nil, pkgerrors.New("Missing collection or tag name") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get all master tables in this collection + filter := bson.D{ + {"key", bson.D{ + {"$exists", true}, + }}, + } + cursor, err := c.Find(ctx, filter) + if err != nil { + return nil, pkgerrors.Errorf("Error reading from database: %s", err.Error()) + } + defer cursor.Close(ctx) + + //Iterate over all the master tables + result := make(map[string][]byte) + for cursor.Next(ctx) { + d, err := cursor.DecodeBytes() + if err != nil { + log.Printf("Unable to decode data in Readall: %s", err.Error()) + continue + } + + //Read key of each master table + key, ok := d.Lookup("key").StringValueOK() + if !ok { + log.Printf("Unable to read key string from mastertable %s", err.Error()) + continue + } + + //Get objectID of tag document + tid, ok := d.Lookup(tag).ObjectIDOK() + if !ok { + log.Printf("Did not find tag: %s", tag) + continue + } + + //Find tag document and unmarshal it into []byte + tagData, err := decodeBytes(c.FindOne(ctx, bson.D{{"_id", tid}})) + if err != nil { + log.Printf("Unable to decode tag data %s", err.Error()) + continue + } + result[key] = tagData.Lookup(tag).Value + } + + if len(result) == 0 { + return result, pkgerrors.Errorf("Did not find any objects with tag: %s", tag) + } + + return result, nil +} diff --git a/src/k8splugin/db/mongo_test.go b/src/k8splugin/db/mongo_test.go new file mode 100644 index 00000000..1663e774 --- /dev/null +++ b/src/k8splugin/db/mongo_test.go @@ -0,0 +1,530 @@ +// +build unit + +/* + * Copyright 2018 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package db + +import ( + "bytes" + "context" + "github.com/mongodb/mongo-go-driver/bson" + "github.com/mongodb/mongo-go-driver/mongo" + "github.com/mongodb/mongo-go-driver/mongo/options" + pkgerrors "github.com/pkg/errors" + "reflect" + "strings" + "testing" +) + +// Implements the mongo.Cursor interface +type mockCursor struct { + mongo.Cursor + err error + bson bson.Raw + count int +} + +func (mc *mockCursor) Next(ctx context.Context) bool { + if mc.count > 0 { + mc.count = mc.count - 1 + return true + } + return false +} + +func (mc *mockCursor) DecodeBytes() (bson.Raw, error) { + return mc.bson, mc.err +} + +func (mc *mockCursor) Close(ctx context.Context) error { + return nil +} + +//Implements the functions used currently in mongo.go +type mockCollection struct { + Err error + mCursor mongo.Cursor +} + +func (c *mockCollection) InsertOne(ctx context.Context, document interface{}, + opts ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) { + + if c.Err != nil { + return nil, c.Err + } + + return &mongo.InsertOneResult{InsertedID: "_id1234"}, nil +} + +func (c *mockCollection) FindOne(ctx context.Context, filter interface{}, + opts ...*options.FindOneOptions) *mongo.SingleResult { + + return &mongo.SingleResult{} +} + +func (c *mockCollection) FindOneAndUpdate(ctx context.Context, filter interface{}, + update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult { + + return &mongo.SingleResult{} +} + +func (c *mockCollection) DeleteOne(ctx context.Context, filter interface{}, + opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) { + + return nil, c.Err +} + +func (c *mockCollection) Find(ctx context.Context, filter interface{}, + opts ...*options.FindOptions) (mongo.Cursor, error) { + + return c.mCursor, c.Err +} + +func TestCreate(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + }{ + { + label: "Successfull creation of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + "data": "Data In String Format", + }, + bson: bson.Raw{'\x08', '\x00', '\x00', '\x00', '\x0A', 'x', '\x00', '\x00'}, + mockColl: &mockCollection{}, + }, + { + label: "UnSuccessfull creation of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + "data": "Data In String Format", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "key": "", + "tag": "", + "data": "", + }, + expectedError: "No Data to store", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.bson, testCase.mockColl.Err + } + + err := m.Create(testCase.input["coll"].(string), testCase.input["key"].(string), + testCase.input["tag"].(string), testCase.input["data"]) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Create method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Create method returned an error (%s)", err) + } + } + }) + } +} + +func TestRead(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + expected []byte + }{ + { + label: "Successfull Read of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "metadata", + }, + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + // This is not the document because we are mocking decodeBytes + expected: []byte{92, 17, 81, 86, 201, 117, 80, 71, 227, 24, 187, 253}, + }, + { + label: "UnSuccessfull Read of entry: object not found", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "badtag", + }, + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + expectedError: "Error finding objectID", + }, + { + label: "UnSuccessfull Read of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "key": "", + "tag": "", + }, + expectedError: "Mandatory fields are missing", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.bson, testCase.mockColl.Err + } + got, err := m.Read(testCase.input["coll"].(string), testCase.input["key"].(string), + testCase.input["tag"].(string)) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Read method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Read method returned an error (%s)", err) + } + } else { + if bytes.Compare(got, testCase.expected) != 0 { + t.Fatalf("Read returned unexpected data: %s, expected: %s", + string(got), testCase.expected) + } + } + }) + } +} + +func TestDelete(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + }{ + { + label: "Successfull Delete of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "metadata", + }, + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + }, + { + label: "UnSuccessfull Delete of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "UnSuccessfull Delete, key not found", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + }, + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + expectedError: "Error finding objectID", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "key": "", + "tag": "", + }, + expectedError: "Mandatory fields are missing", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.bson, testCase.mockColl.Err + } + err := m.Delete(testCase.input["coll"].(string), testCase.input["key"].(string), + testCase.input["tag"].(string)) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Delete method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Delete method returned an error (%s)", err) + } + } + }) + } +} + +func TestReadAll(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + expected map[string][]byte + }{ + { + label: "Successfully Read all entries", + input: map[string]interface{}{ + "coll": "collname", + "tag": "metadata", + }, + mockColl: &mockCollection{ + mCursor: &mockCursor{ + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + count: 1, + }, + }, + expected: map[string][]byte{ + "b82c4bb1-09ff-6093-4d58-8327b94e1e20": []byte{ + 92, 17, 81, 86, 201, 117, 80, 71, 227, 24, 187, 253}, + }, + }, + { + label: "UnSuccessfully Read of all entries", + input: map[string]interface{}{ + "coll": "collname", + "tag": "tagName", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "UnSuccessfull Readall, tag not found", + input: map[string]interface{}{ + "coll": "collname", + "tag": "tagName", + }, + mockColl: &mockCollection{ + mCursor: &mockCursor{ + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + count: 1, + }, + }, + expectedError: "Did not find any objects with tag", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "tag": "", + }, + expectedError: "Missing collection or tag name", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.mockColl.mCursor.DecodeBytes() + } + + got, err := m.ReadAll(testCase.input["coll"].(string), testCase.input["tag"].(string)) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Readall method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Readall method returned an error (%s)", err) + } + } else { + if reflect.DeepEqual(got, testCase.expected) == false { + t.Fatalf("Readall returned unexpected data: %v, expected: %v", + got, testCase.expected) + } + } + }) + } +} diff --git a/src/k8splugin/db/store.go b/src/k8splugin/db/store.go index c1a8b31f..a235597a 100644 --- a/src/k8splugin/db/store.go +++ b/src/k8splugin/db/store.go @@ -25,21 +25,38 @@ var DBconn Store // Store is an interface for accessing a database type Store interface { + // Returns nil if db health is good HealthCheck() error - Create(string, string) error - Read(string) (string, error) - // Update(string) (string, error) - Delete(string) error + // Unmarshal implements any unmarshaling needed for the database + Unmarshal(inp []byte, out interface{}) error - ReadAll(string) ([]string, error) + // Creates a new master table with key and links data with tag and + // creates a pointer to the newly added data in the master table + Create(table, key, tag string, data interface{}) error + + // Reads data for a particular key with specific tag. + Read(table, key, tag string) ([]byte, error) + + //TODO: Update(context.Context, string, interface{}) error + + // Deletes a specific tag data for key. + // TODO: If tag is empty, it will delete all tags under key. + Delete(table, key, tag string) error + + // Reads all master tables and data from the specified tag in table + ReadAll(table, tag string) (map[string][]byte, error) } // CreateDBClient creates the DB client func CreateDBClient(dbType string) error { var err error switch dbType { + case "mongo": + // create a mongodb database with k8splugin as the name + DBconn, err = NewMongoStore("k8splugin", nil) case "consul": + // create a consul kv store DBconn, err = NewConsulStore(nil) default: return pkgerrors.New(dbType + "DB not supported") diff --git a/src/k8splugin/db/store_test.go b/src/k8splugin/db/store_test.go index 9bbe4a92..eed7065f 100644 --- a/src/k8splugin/db/store_test.go +++ b/src/k8splugin/db/store_test.go @@ -23,9 +23,9 @@ import ( func TestCreateDBClient(t *testing.T) { t.Run("Successfully create DB client", func(t *testing.T) { - expected := &ConsulStore{} + expected := &MongoStore{} - err := CreateDBClient("consul") + err := CreateDBClient("mongo") if err != nil { t.Fatalf("CreateDBClient returned an error (%s)", err) } diff --git a/src/k8splugin/db/testing.go b/src/k8splugin/db/testing.go index 672fcbfb..4b7e6078 100644 --- a/src/k8splugin/db/testing.go +++ b/src/k8splugin/db/testing.go @@ -16,7 +16,8 @@ limitations under the License. package db import ( - "github.com/hashicorp/consul/api" + "encoding/json" + pkgerrors "github.com/pkg/errors" ) //Creating an embedded interface via anonymous variable @@ -24,42 +25,45 @@ import ( //interface even if we are not implementing all the methods in it type MockDB struct { Store - Items api.KVPairs + Items map[string][]byte Err error } -func (m *MockDB) Create(key string, value string) error { +func (m *MockDB) Create(table, key, tag string, data interface{}) error { return m.Err } -func (m *MockDB) Read(key string) (string, error) { +// MockDB uses simple JSON and not BSON +func (m *MockDB) Unmarshal(inp []byte, out interface{}) error { + err := json.Unmarshal(inp, out) + if err != nil { + return pkgerrors.Wrap(err, "Unmarshaling json") + } + return nil +} + +func (m *MockDB) Read(table, key, tag string) ([]byte, error) { if m.Err != nil { - return "", m.Err + return nil, m.Err } - for _, kvpair := range m.Items { - if kvpair.Key == key { - return string(kvpair.Value), nil + for k, v := range m.Items { + if k == key { + return v, nil } } - return "", nil + return nil, m.Err } -func (m *MockDB) Delete(key string) error { +func (m *MockDB) Delete(table, key, tag string) error { return m.Err } -func (m *MockDB) ReadAll(prefix string) ([]string, error) { +func (m *MockDB) ReadAll(table, tag string) (map[string][]byte, error) { if m.Err != nil { - return []string{}, m.Err - } - - var res []string - - for _, keypair := range m.Items { - res = append(res, keypair.Key) + return nil, m.Err } - return res, nil + return m.Items, nil } diff --git a/src/k8splugin/go.mod b/src/k8splugin/go.mod index b4f4558b..cf2c7d85 100644 --- a/src/k8splugin/go.mod +++ b/src/k8splugin/go.mod @@ -1,35 +1,45 @@ module k8splugin require ( - github.com/ghodss/yaml v1.0.0 - github.com/gogo/protobuf v1.0.0 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/protobuf v1.1.0 - github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf - github.com/googleapis/gnostic v0.2.0 - github.com/gorilla/context v1.1.1 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-stack/stack v1.8.0 // indirect + github.com/gogo/protobuf v1.0.0 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/protobuf v1.2.0 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect + github.com/google/go-cmp v0.2.0 // indirect + github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect + github.com/googleapis/gnostic v0.2.0 // indirect + github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.6.2 - github.com/hashicorp/consul v1.2.2 - github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186 - github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 + github.com/hashicorp/consul v1.4.0 + github.com/hashicorp/go-cleanhttp v0.5.0 // indirect + github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 // indirect github.com/hashicorp/go-uuid v1.0.0 - github.com/hashicorp/serf v0.8.1 - github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c - github.com/imdario/mergo v0.3.5 - github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece - github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff - github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd - github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81 + github.com/hashicorp/serf v0.8.1 // indirect + github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece // indirect + github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81 // indirect + github.com/mongodb/mongo-go-driver v0.1.0 github.com/pkg/errors v0.8.0 - github.com/spf13/pflag v1.0.1 - golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4 - golang.org/x/net v0.0.0-20180611182652-db08ff08e862 - golang.org/x/sys v0.0.0-20180611080425-bff228c7b664 - golang.org/x/text v0.3.0 - golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 - gopkg.in/inf.v0 v0.9.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.1 // indirect + github.com/stretchr/testify v1.2.2 // indirect + github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 // indirect + github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect + github.com/xdg/stringprep v1.0.0 // indirect + golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4 // indirect + golang.org/x/net v0.0.0-20180724234803-3673e40ba225 + golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect + golang.org/x/sys v0.0.0-20180611080425-bff228c7b664 // indirect + golang.org/x/text v0.3.0 // indirect + golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.2.1 k8s.io/api v0.0.0-20180607235014-72d6e4405f81 k8s.io/apimachinery v0.0.0-20180515182440-31dade610c05 diff --git a/src/k8splugin/go.sum b/src/k8splugin/go.sum index 4a10051a..bcb622c7 100644 --- a/src/k8splugin/go.sum +++ b/src/k8splugin/go.sum @@ -1,24 +1,33 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.0.0 h1:2jyBKDKU/8v3v2xVR2PtiWQviFUyiaGk2rpfyFT8rTM= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/hashicorp/consul v1.2.2 h1:C5FurAZWLQ+XAjmL9g6rXbPlwxyyz8DvTL0WCAxTLAo= -github.com/hashicorp/consul v1.2.2/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= -github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186 h1:URgjUo+bs1KwatoNbwG0uCO4dHN4r1jsp4a5AGgHRjo= -github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/consul v1.4.0 h1:PQTW4xCuAExEiSbhrsFsikzbW5gVBoi74BjUvYFyKHw= +github.com/hashicorp/consul v1.4.0/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= +github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 h1:9HVkPxOpo+yO93Ah4yrO67d/qh0fbLLWbKqhYjyHq9A= github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= @@ -31,28 +40,41 @@ github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece h1:3HJXp/18JmMk5sjBP3LDUBtWjczCvynxaeAF6b6kWp8= github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff h1:jM4Eo4qMmmcqePS3u6X2lcEELtVuXWkWJIS/pRI3oSk= -github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699 h1:KXZJFdun9knAVAR8tg/aHJEr5DgtcbqyvzacK+CDCaI= -github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81 h1:ImOHKpmdLPXWX5KSYquUWXKaopEPuY7TPPUo18u9aOI= github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mongodb/mongo-go-driver v0.1.0 h1:LcpPFw0tNumIAakvNrkI9S9wdX0iOxvMLw/+hcAdHaU= +github.com/mongodb/mongo-go-driver v0.1.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 h1:BP2bjP495BBPaBcS5rmqviTfrOkN5rO5ceKAMRZCRFc= +github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4 h1:wviDUSmtheHRBfoY8B9U8ELl2USoXi2YFwdGdpIIkzI= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20180611182652-db08ff08e862 h1:JZi6BqOZ+iSgmLWe6llhGrNnEnK+YB/MRkStwnEfbqM= -golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180611080425-bff228c7b664 h1:GvcVmbE8Pa64iW3MTrVA9mxHx1HEjSSWV6zF1JSlFcg= golang.org/x/sys v0.0.0-20180611080425-bff228c7b664/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/src/k8splugin/rb/definition.go b/src/k8splugin/rb/definition.go index 2d1c2cd0..02ecc5ae 100644 --- a/src/k8splugin/rb/definition.go +++ b/src/k8splugin/rb/definition.go @@ -47,15 +47,19 @@ type DefinitionManager interface { // DefinitionClient implements the DefinitionManager // It will also be used to maintain some localized state type DefinitionClient struct { - keyPrefix string + storeName string + tagMeta, tagContent string } // NewDefinitionClient returns an instance of the DefinitionClient // which implements the DefinitionManager -// Uses rb/def prefix +// Uses rbdef collection in underlying db func NewDefinitionClient() *DefinitionClient { return &DefinitionClient{ - keyPrefix: "rb/def/"} + storeName: "rbdef", + tagMeta: "metadata", + tagContent: "content", + } } // Create an entry for the resource in the database @@ -64,14 +68,9 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) { if def.UUID == "" { def.UUID, _ = uuid.GenerateUUID() } - key := v.keyPrefix + def.UUID - - serData, err := db.Serialize(def) - if err != nil { - return Definition{}, pkgerrors.Wrap(err, "Serialize Resource Bundle Definition") - } + key := def.UUID - err = db.DBconn.Create(key, serData) + err := db.DBconn.Create(v.storeName, key, v.tagMeta, def) if err != nil { return Definition{}, pkgerrors.Wrap(err, "Creating DB Entry") } @@ -81,45 +80,39 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) { // List all resource entries in the database func (v *DefinitionClient) List() ([]Definition, error) { - strArray, err := db.DBconn.ReadAll(v.keyPrefix) - if err != nil { + res, err := db.DBconn.ReadAll(v.storeName, v.tagMeta) + if err != nil || len(res) == 0 { return []Definition{}, pkgerrors.Wrap(err, "Listing Resource Bundle Definitions") } - var retData []Definition - - for _, key := range strArray { - value, err := db.DBconn.Read(key) - if err != nil { - log.Printf("Error Reading Key: %s", key) - continue - } - if value != "" { + var results []Definition + for key, value := range res { + if len(value) > 0 { def := Definition{} - err = db.DeSerialize(value, &def) + err = db.DBconn.Unmarshal(value, &def) if err != nil { - log.Printf("Error Deserializing Value: %s", value) + log.Printf("Error Unmarshaling value for: %s", key) continue } - retData = append(retData, def) + results = append(results, def) } } - return retData, nil + return results, nil } // Get returns the Resource Bundle Definition for corresponding ID func (v *DefinitionClient) Get(id string) (Definition, error) { - value, err := db.DBconn.Read(v.keyPrefix + id) + value, err := db.DBconn.Read(v.storeName, id, v.tagMeta) if err != nil { return Definition{}, pkgerrors.Wrap(err, "Get Resource Bundle definition") } - if value != "" { + if value != nil { def := Definition{} - err = db.DeSerialize(value, &def) + err = db.DBconn.Unmarshal(value, &def) if err != nil { - return Definition{}, pkgerrors.Wrap(err, "Deserializing Value") + return Definition{}, pkgerrors.Wrap(err, "Unmarshaling Value") } return def, nil } @@ -129,7 +122,7 @@ func (v *DefinitionClient) Get(id string) (Definition, error) { // Delete the Resource Bundle definition from database func (v *DefinitionClient) Delete(id string) error { - err := db.DBconn.Delete(v.keyPrefix + id) + err := db.DBconn.Delete(v.storeName, id, v.tagMeta) if err != nil { return pkgerrors.Wrap(err, "Delete Resource Bundle Definitions") } @@ -140,22 +133,22 @@ func (v *DefinitionClient) Delete(id string) error { // Upload the contents of resource bundle into database func (v *DefinitionClient) Upload(id string, inp []byte) error { - //ignore the returned data here. + //ignore the returned data here _, err := v.Get(id) if err != nil { - return pkgerrors.Errorf("Invalid ID provided %s", err.Error()) + return pkgerrors.Errorf("Invalid ID provided: %s", err.Error()) } err = isTarGz(bytes.NewBuffer(inp)) if err != nil { - return pkgerrors.Errorf("Error in file format %s", err.Error()) + return pkgerrors.Errorf("Error in file format: %s", err.Error()) } + //Encode given byte stream to text for storage encodedStr := base64.StdEncoding.EncodeToString(inp) - key := v.keyPrefix + id + "/content" - err = db.DBconn.Create(key, encodedStr) + err = db.DBconn.Create(v.storeName, id, encodedStr, v.tagContent) if err != nil { - return pkgerrors.Errorf("Error uploading data to db %s", err.Error()) + return pkgerrors.Errorf("Error uploading data to db: %s", err.Error()) } return nil diff --git a/src/k8splugin/rb/definition_test.go b/src/k8splugin/rb/definition_test.go index 58eb718d..1e488678 100644 --- a/src/k8splugin/rb/definition_test.go +++ b/src/k8splugin/rb/definition_test.go @@ -24,7 +24,6 @@ import ( "strings" "testing" - "github.com/hashicorp/consul/api" pkgerrors "github.com/pkg/errors" ) @@ -110,21 +109,17 @@ func TestList(t *testing.T) { }, expectedError: "", mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655441111", - Value: []byte("{\"name\":\"testresourcebundle2\"," + + "123e4567-e89b-12d3-a456-426655441111": []byte( + "{\"name\":\"testresourcebundle2\"," + "\"description\":\"testresourcebundle2\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," + "\"service-type\":\"dns\"}"), - }, }, }, }, @@ -179,14 +174,12 @@ func TestGet(t *testing.T) { }, expectedError: "", mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -293,14 +286,12 @@ func TestUpload(t *testing.T) { 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -330,14 +321,12 @@ func TestUpload(t *testing.T) { 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655441111", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655441111": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -350,14 +339,12 @@ func TestUpload(t *testing.T) { 0x00, 0xff, 0xf2, 0x48, 0xcd, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, diff --git a/vagrant/tests/plugin.sh b/vagrant/tests/plugin.sh index 16d8d306..55be1686 100755 --- a/vagrant/tests/plugin.sh +++ b/vagrant/tests/plugin.sh @@ -88,7 +88,7 @@ echo "VNF details $vnf_details" echo "Deleting $vnf_id VNF Instance" curl -X DELETE "${base_url}${cloud_region_id}/${namespace}/${vnf_id}" -if [[ -n $(curl -s -X GET "${base_url}${cloud_region_id}/${namespace}/${vnf_id}") ]]; then +if [[ 200 -eq $(curl -o /dev/null -w %{http_code} -s -X GET "${base_url}${cloud_region_id}/${namespace}/${vnf_id}") ]]; then echo "VNF Instance not deleted" exit 1 fi |