diff options
author | Victor Morales <victor.morales@intel.com> | 2018-12-15 00:34:27 +0000 |
---|---|---|
committer | Gerrit Code Review <gerrit@onap.org> | 2018-12-15 00:34:27 +0000 |
commit | 2da9b6c0498528a6eac478fa7b334906457f7b82 (patch) | |
tree | be33f0fadf12c321c9a20eaf943eda5b43335948 | |
parent | b878686931d404568af000df1f0f86927a730680 (diff) | |
parent | d203ccf9174e450b6f5a6a9aabea95b73c9973ff (diff) |
Merge "Migrating from consul to mongodb for backend"
-rw-r--r-- | deployments/docker-compose.yml | 15 | ||||
-rwxr-xr-x | deployments/start.sh | 8 | ||||
-rw-r--r-- | src/k8splugin/api/handler.go | 55 | ||||
-rw-r--r-- | src/k8splugin/api/handler_test.go | 101 | ||||
-rw-r--r-- | src/k8splugin/db/consul.go | 38 | ||||
-rw-r--r-- | src/k8splugin/db/consul_test.go | 63 | ||||
-rw-r--r-- | src/k8splugin/db/mongo.go | 323 | ||||
-rw-r--r-- | src/k8splugin/db/mongo_test.go | 530 | ||||
-rw-r--r-- | src/k8splugin/db/store.go | 27 | ||||
-rw-r--r-- | src/k8splugin/db/store_test.go | 4 | ||||
-rw-r--r-- | src/k8splugin/db/testing.go | 42 | ||||
-rw-r--r-- | src/k8splugin/go.mod | 60 | ||||
-rw-r--r-- | src/k8splugin/go.sum | 46 | ||||
-rw-r--r-- | src/k8splugin/rb/definition.go | 65 | ||||
-rw-r--r-- | src/k8splugin/rb/definition_test.go | 47 | ||||
-rwxr-xr-x | vagrant/tests/plugin.sh | 2 |
16 files changed, 1136 insertions, 290 deletions
diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml index 73d5651c..a72bd096 100644 --- a/deployments/docker-compose.yml +++ b/deployments/docker-compose.yml @@ -28,33 +28,28 @@ services: environment: - CSAR_DIR=/opt/csar - KUBE_CONFIG_DIR=/opt/kubeconfig - - DATABASE_TYPE=consul + - DATABASE_TYPE=mongo - DATABASE_IP=172.19.0.2 - PLUGINS_DIR=/opt/multicloud/k8s - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY,172.19.0.2 depends_on: - - consul + - mongo links: - - consul + - mongo volumes: - /opt/csar:/opt/csar - /opt/kubeconfig:/opt/kubeconfig - consul: - image: consul + mongo: + image: mongo networks: multicloud_net: ipv4_address: 172.19.0.2 environment: - CONSUL_CLIENT_INTERFACE: 'eth0' - CONSUL_BIND_INTERFACE: 'eth0' HTTP_PROXY: $HTTP_PROXY HTTPS_PROXY: $HTTPS_PROXY NO_PROXY: $NO_PROXY - command: ["agent", "-server", "-bootstrap-expect=1"] - volumes: - - /opt/consul/config:/consul/config networks: multicloud_net: diff --git a/deployments/start.sh b/deployments/start.sh index da2eacee..d1b9f68a 100755 --- a/deployments/start.sh +++ b/deployments/start.sh @@ -19,13 +19,13 @@ export IMAGE_NAME="nexus3.onap.org:10003/onap/multicloud/k8s" export CSAR_DIR=/opt/csar export KUBE_CONFIG_DIR=/opt/kubeconfig -export DATABASE_TYPE=consul +export DATABASE_TYPE=mongo export PLUGINS_DIR=$k8s_path/src/k8splugin/plugins -echo "Starting consul services" +echo "Starting mongo services" docker-compose kill -docker-compose up -d consul -export DATABASE_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=consul")) +docker-compose up -d mongo +export DATABASE_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=mongo")) export no_proxy=$no_proxy,$DATABASE_IP export NO_PROXY=$NO_PROXY,$DATABASE_IP diff --git a/src/k8splugin/api/handler.go b/src/k8splugin/api/handler.go index 53fa2317..4c49ba78 100644 --- a/src/k8splugin/api/handler.go +++ b/src/k8splugin/api/handler.go @@ -30,6 +30,10 @@ import ( "k8splugin/krd" ) +//TODO: Separate the http handler code and backend code out +var storeName = "rbinst" +var tagData = "data" + // GetVNFClient retrieves the client used to communicate with a Kubernetes Cluster var GetVNFClient = func(kubeConfigPath string) (kubernetes.Clientset, error) { client, err := krd.GetKubeClient(kubeConfigPath) @@ -117,17 +121,9 @@ func CreateHandler(w http.ResponseWriter, r *http.Request) { // TODO: Uncomment when annotations are done // krd.AddNetworkAnnotationsToPod(kubeData, resource.Networks) - // "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.Serialize(resourceNameMap) - if err != nil { - werr := pkgerrors.Wrap(err, "Create VNF deployment JSON Marshalling error") - http.Error(w, werr.Error(), http.StatusInternalServerError) - return - } - // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - err = db.DBconn.Create(internalVNFID, serializedResourceNameMap) + err = db.DBconn.Create(storeName, internalVNFID, tagData, resourceNameMap) if err != nil { werr := pkgerrors.Wrap(err, "Create VNF deployment DB error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -154,27 +150,22 @@ func ListHandler(w http.ResponseWriter, r *http.Request) { namespace := vars["namespace"] prefix := cloudRegionID + "-" + namespace - internalVNFIDs, err := db.DBconn.ReadAll(prefix) + res, err := db.DBconn.ReadAll(storeName, tagData) if err != nil { http.Error(w, pkgerrors.Wrap(err, "Get VNF list error").Error(), http.StatusInternalServerError) return } - if len(internalVNFIDs) == 0 { - w.WriteHeader(http.StatusNotFound) - return - } - // TODO: There is an edge case where if namespace is passed but is missing some characters // trailing, it will print the result with those excluding characters. This is because of // the way I am trimming the Prefix. This fix is needed. var editedList []string - for _, id := range internalVNFIDs { - if len(id) > 0 { - editedList = append(editedList, strings.TrimPrefix(id, prefix)[1:]) + for key, value := range res { + if len(value) > 0 { + editedList = append(editedList, strings.TrimPrefix(key, prefix)[1:]) } } @@ -204,25 +195,20 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) { // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.DBconn.Read(internalVNFID) + res, err := db.DBconn.Read(storeName, internalVNFID, tagData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if serializedResourceNameMap == "" { - w.WriteHeader(http.StatusNotFound) - return - } - /* { "deployment": ["cloud1-default-uuid-sisedeploy1", "cloud1-default-uuid-sisedeploy2", ... ] "service": ["cloud1-default-uuid-sisesvc1", "cloud1-default-uuid-sisesvc2", ... ] }, */ - deserializedResourceNameMap := make(map[string][]string) - err = db.DeSerialize(serializedResourceNameMap, &deserializedResourceNameMap) + data := make(map[string][]string) + err = db.DBconn.Unmarshal(res, &data) if err != nil { werr := pkgerrors.Wrap(err, "Unmarshal VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -237,14 +223,14 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - err = csar.DestroyVNF(deserializedResourceNameMap, namespace, &kubeclient) + err = csar.DestroyVNF(data, namespace, &kubeclient) if err != nil { werr := pkgerrors.Wrap(err, "Delete VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) return } - err = db.DBconn.Delete(internalVNFID) + err = db.DBconn.Delete(storeName, internalVNFID, tagData) if err != nil { werr := pkgerrors.Wrap(err, "Delete VNF db record error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -337,25 +323,20 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { // key: cloud1-default-uuid // value: "{"deployment":<>,"service":<>}" - serializedResourceNameMap, err := db.DBconn.Read(internalVNFID) + res, err := db.DBconn.Read(storeName, internalVNFID, tagData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if serializedResourceNameMap == "" { - w.WriteHeader(http.StatusNotFound) - return - } - /* { "deployment": ["cloud1-default-uuid-sisedeploy1", "cloud1-default-uuid-sisedeploy2", ... ] "service": ["cloud1-default-uuid-sisesvc1", "cloud1-default-uuid-sisesvc2", ... ] }, */ - deserializedResourceNameMap := make(map[string][]string) - err = db.DeSerialize(serializedResourceNameMap, &deserializedResourceNameMap) + data := make(map[string][]string) + err = db.DBconn.Unmarshal(res, &data) if err != nil { werr := pkgerrors.Wrap(err, "Unmarshal VNF error") http.Error(w, werr.Error(), http.StatusInternalServerError) @@ -366,7 +347,7 @@ func GetHandler(w http.ResponseWriter, r *http.Request) { VNFID: externalVNFID, CloudRegionID: cloudRegionID, Namespace: namespace, - VNFComponents: deserializedResourceNameMap, + VNFComponents: data, } w.Header().Set("Content-Type", "application/json") diff --git a/src/k8splugin/api/handler_test.go b/src/k8splugin/api/handler_test.go index 3336bbc2..a3aeff7a 100644 --- a/src/k8splugin/api/handler_test.go +++ b/src/k8splugin/api/handler_test.go @@ -24,7 +24,6 @@ import ( "reflect" "testing" - "github.com/hashicorp/consul/api" pkgerrors "github.com/pkg/errors" "k8s.io/client-go/kubernetes" @@ -194,37 +193,18 @@ func TestListHandler(t *testing.T) { }, }, { - label: "Get result from DB non-records", - expectedCode: http.StatusNotFound, - mockStore: &db.MockDB{}, - }, - { label: "Get empty list", expectedCode: http.StatusOK, expectedResponse: []string{""}, - mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "", - Value: []byte("{}"), - }, - }, - }, + mockStore: &db.MockDB{}, }, { label: "Succesful get a list of VNF", expectedCode: http.StatusOK, - expectedResponse: []string{"uid1", "uid2"}, + expectedResponse: []string{"uid1"}, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "uuid1", - Value: []byte("{}"), - }, - &api.KVPair{ - Key: "uuid2", - Value: []byte("{}"), - }, + Items: map[string][]byte{ + "uuid1": []byte("{}"), }, }, }, @@ -275,20 +255,17 @@ func TestDeleteHandler(t *testing.T) { }, { label: "Fail to find VNF record be deleted", - expectedCode: http.StatusNotFound, + expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{}, + Items: map[string][]byte{}, }, }, { label: "Fail to unmarshal the DB record", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{invalid format}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte("{invalid format}"), }, }, }, @@ -297,14 +274,10 @@ func TestDeleteHandler(t *testing.T) { expectedCode: http.StatusInternalServerError, mockGetVNFClientErr: pkgerrors.New("Get VNF client error"), mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, }, @@ -312,14 +285,10 @@ func TestDeleteHandler(t *testing.T) { label: "Fail to destroy VNF", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, mockDeleteVNF: &mockCSAR{ @@ -330,14 +299,10 @@ func TestDeleteHandler(t *testing.T) { label: "Succesful delete a VNF", expectedCode: http.StatusAccepted, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloudregion1-testnamespace-uuid1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, + Items: map[string][]byte{ + "cloudregion1-testnamespace-uuid1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), }, }, mockDeleteVNF: &mockCSAR{}, @@ -440,18 +405,15 @@ func TestGetHandler(t *testing.T) { }, { label: "Not found DB record", - expectedCode: http.StatusNotFound, + expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{}, }, { label: "Fail to unmarshal the DB record", expectedCode: http.StatusInternalServerError, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloud1-default-1", - Value: []byte("{invalid-format}"), - }, + Items: map[string][]byte{ + "cloud1-default-1": []byte("{invalid-format}"), }, }, }, @@ -468,18 +430,11 @@ func TestGetHandler(t *testing.T) { }, }, mockStore: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "cloud1-default-1", - Value: []byte("{" + - "\"deployment\": [\"deploy1\", \"deploy2\"]," + - "\"service\": [\"svc1\", \"svc2\"]" + - "}"), - }, - &api.KVPair{ - Key: "cloud1-default-2", - Value: []byte("{}"), - }, + Items: map[string][]byte{ + "cloud1-default-1": []byte( + "{\"deployment\": [\"deploy1\", \"deploy2\"]," + + "\"service\": [\"svc1\", \"svc2\"]}"), + "cloud1-default-2": []byte("{}"), }, }, }, diff --git a/src/k8splugin/db/consul.go b/src/k8splugin/db/consul.go index d7507242..a61a4c10 100644 --- a/src/k8splugin/db/consul.go +++ b/src/k8splugin/db/consul.go @@ -54,50 +54,64 @@ func NewConsulStore(store ConsulKVStore) (Store, error) { // HealthCheck verifies if the database is up and running func (c *ConsulStore) HealthCheck() error { - _, err := c.Read("test") + _, err := c.Read("test", "test", "test") if err != nil { return pkgerrors.New("[ERROR] Cannot talk to Datastore. Check if it is running/reachable.") } return nil } +// Unmarshal implements any unmarshaling that is needed when using consul +func (c *ConsulStore) Unmarshal(inp []byte, out interface{}) error { + return nil +} + // Create is used to create a DB entry -func (c *ConsulStore) Create(key, value string) error { +func (c *ConsulStore) Create(root, key, tag string, data interface{}) error { + + value, err := Serialize(data) + if err != nil { + return pkgerrors.Wrap(err, "Serializing input data") + } + p := &api.KVPair{ Key: key, Value: []byte(value), } - _, err := c.client.Put(p, nil) + _, err = c.client.Put(p, nil) return err } // Read method returns the internalID for a particular externalID -func (c *ConsulStore) Read(key string) (string, error) { +func (c *ConsulStore) Read(root, key, tag string) ([]byte, error) { + key = root + "/" + key + "/" + tag pair, _, err := c.client.Get(key, nil) if err != nil { - return "", err + return nil, err } if pair == nil { - return "", nil + return nil, nil } - return string(pair.Value), nil + return pair.Value, nil } // Delete method removes an internalID from the Database -func (c *ConsulStore) Delete(key string) error { +func (c *ConsulStore) Delete(root, key, tag string) error { _, err := c.client.Delete(key, nil) return err } // ReadAll is used to get all ExternalIDs in a namespace -func (c *ConsulStore) ReadAll(prefix string) ([]string, error) { - pairs, _, err := c.client.List(prefix, nil) +func (c *ConsulStore) ReadAll(root, tag string) (map[string][]byte, error) { + pairs, _, err := c.client.List(root, nil) if err != nil { return nil, err } - var result []string + + //TODO: Filter results by tag and return it + result := make(map[string][]byte) for _, keypair := range pairs { - result = append(result, keypair.Key) + result[keypair.Key] = keypair.Value } return result, nil diff --git a/src/k8splugin/db/consul_test.go b/src/k8splugin/db/consul_test.go index ede1a5e9..754112ad 100644 --- a/src/k8splugin/db/consul_test.go +++ b/src/k8splugin/db/consul_test.go @@ -107,12 +107,14 @@ func TestConsulCreate(t *testing.T) { }{ { label: "Sucessful register a record to Consul Database", - input: map[string]string{"key": "test-key", "value": "test-value"}, - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data", "value": "test-value"}, + mock: &mockConsulKVStore{}, }, { label: "Fail to create a new record in Consul Database", - input: map[string]string{"key": "test-key", "value": "test-value"}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data", "value": "test-value"}, mock: &mockConsulKVStore{ Err: pkgerrors.New("DB error"), }, @@ -123,7 +125,8 @@ func TestConsulCreate(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - err := client.Create(testCase.input["key"], testCase.input["value"]) + err := client.Create(testCase.input["root"], testCase.input["key"], + testCase.input["tag"], testCase.input["value"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("Create method return an un-expected (%s)", err) @@ -139,18 +142,19 @@ func TestConsulCreate(t *testing.T) { func TestConsulRead(t *testing.T) { testCases := []struct { label string - input string + input map[string]string mock *mockConsulKVStore expectedError string expectedResult string }{ { label: "Sucessful retrieve a record from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test", + "tag": "data"}, mock: &mockConsulKVStore{ Items: api.KVPairs{ &api.KVPair{ - Key: "test", + Key: "rbinst/test/data", Value: []byte("test-value"), }, }, @@ -159,12 +163,14 @@ func TestConsulRead(t *testing.T) { }, { label: "Fail retrieve a non-existing record from Consul Database", - input: "test", - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, + mock: &mockConsulKVStore{}, }, { label: "Fail retrieve a record from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, mock: &mockConsulKVStore{ Err: pkgerrors.New("DB error"), }, @@ -175,7 +181,8 @@ func TestConsulRead(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - result, err := client.Read(testCase.input) + result, err := client.Read(testCase.input["root"], testCase.input["key"], + testCase.input["tag"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("Read method return an un-expected (%s)", err) @@ -187,7 +194,7 @@ func TestConsulRead(t *testing.T) { if testCase.expectedError != "" && testCase.expectedResult == "" { t.Fatalf("Read method was expecting \"%s\" error message", testCase.expectedError) } - if !reflect.DeepEqual(testCase.expectedResult, result) { + if !reflect.DeepEqual(testCase.expectedResult, string(result)) { t.Fatalf("Read method returned: \n%v\n and it was expected: \n%v", result, testCase.expectedResult) } @@ -199,14 +206,15 @@ func TestConsulRead(t *testing.T) { func TestConsulDelete(t *testing.T) { testCases := []struct { label string - input string + input map[string]string mock *mockConsulKVStore expectedError string }{ { label: "Sucessful delete a record to Consul Database", - input: "test", - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, + mock: &mockConsulKVStore{}, }, { label: "Fail to delete a record in Consul Database", @@ -220,7 +228,8 @@ func TestConsulDelete(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - err := client.Delete(testCase.input) + err := client.Delete(testCase.input["root"], testCase.input["key"], + testCase.input["tag"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("Delete method return an un-expected (%s)", err) @@ -236,14 +245,15 @@ func TestConsulDelete(t *testing.T) { func TestConsulReadAll(t *testing.T) { testCases := []struct { label string - input string + input map[string]string mock *mockConsulKVStore expectedError string - expectedResult []string + expectedResult map[string][]byte }{ { label: "Sucessful retrieve a list from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, mock: &mockConsulKVStore{ Items: api.KVPairs{ &api.KVPair{ @@ -256,16 +266,20 @@ func TestConsulReadAll(t *testing.T) { }, }, }, - expectedResult: []string{"test", "test2"}, + expectedResult: map[string][]byte{"test": []byte("test-value"), + "test2": []byte("test-value2")}, }, { label: "Sucessful retrieve an empty list from Consul Database", - input: "test", - mock: &mockConsulKVStore{}, + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, + mock: &mockConsulKVStore{}, + expectedResult: map[string][]byte{}, }, { label: "Fail retrieve a record from Consul Database", - input: "test", + input: map[string]string{"root": "rbinst", "key": "test-key", + "tag": "data"}, mock: &mockConsulKVStore{ Err: pkgerrors.New("DB error"), }, @@ -276,7 +290,8 @@ func TestConsulReadAll(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.label, func(t *testing.T) { client, _ := NewConsulStore(testCase.mock) - result, err := client.ReadAll(testCase.input) + result, err := client.ReadAll(testCase.input["root"], + testCase.input["tag"]) if err != nil { if testCase.expectedError == "" { t.Fatalf("ReadAll method return an un-expected (%s)", err) diff --git a/src/k8splugin/db/mongo.go b/src/k8splugin/db/mongo.go new file mode 100644 index 00000000..311f044c --- /dev/null +++ b/src/k8splugin/db/mongo.go @@ -0,0 +1,323 @@ +/* + * Copyright 2018 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package db + +import ( + "github.com/mongodb/mongo-go-driver/bson" + "github.com/mongodb/mongo-go-driver/bson/primitive" + "github.com/mongodb/mongo-go-driver/mongo" + "github.com/mongodb/mongo-go-driver/mongo/options" + pkgerrors "github.com/pkg/errors" + "golang.org/x/net/context" + "log" + "os" +) + +// MongoCollection defines the a subset of MongoDB operations +// Note: This interface is defined mainly for mock testing +type MongoCollection interface { + InsertOne(ctx context.Context, document interface{}, + opts ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) + FindOne(ctx context.Context, filter interface{}, + opts ...*options.FindOneOptions) *mongo.SingleResult + FindOneAndUpdate(ctx context.Context, filter interface{}, + update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult + DeleteOne(ctx context.Context, filter interface{}, + opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) + Find(ctx context.Context, filter interface{}, + opts ...*options.FindOptions) (mongo.Cursor, error) +} + +// MongoStore is an implementation of the db.Store interface +type MongoStore struct { + db *mongo.Database +} + +// This exists only for allowing us to mock the collection object +// for testing purposes +var getCollection = func(coll string, m *MongoStore) MongoCollection { + return m.db.Collection(coll) +} + +// This exists only for allowing us to mock the DecodeBytes function +// Mainly because we cannot construct a SingleResult struct from our +// tests. All fields in that struct are private. +var decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return sr.DecodeBytes() +} + +// NewMongoStore initializes a Mongo Database with the name provided +// If a database with that name exists, it will be returned +func NewMongoStore(name string, store *mongo.Database) (Store, error) { + if store == nil { + ip := "mongodb://" + os.Getenv("DATABASE_IP") + ":27017" + mongoClient, err := mongo.NewClient(ip) + if err != nil { + return nil, err + } + + err = mongoClient.Connect(context.Background()) + if err != nil { + return nil, err + } + store = mongoClient.Database(name) + } + + return &MongoStore{ + db: store, + }, nil +} + +// HealthCheck verifies if the database is up and running +func (m *MongoStore) HealthCheck() error { + + _, err := decodeBytes(m.db.RunCommand(context.Background(), bson.D{{"serverStatus", 1}})) + if err != nil { + return pkgerrors.Wrap(err, "Error getting server status") + } + + return nil +} + +// validateParams checks to see if any parameters are empty +func (m *MongoStore) validateParams(args ...string) bool { + for _, v := range args { + if v == "" { + return false + } + } + + return true +} + +// Create is used to create a DB entry +func (m *MongoStore) Create(coll, key, tag string, data interface{}) error { + if data == nil || !m.validateParams(coll, key, tag) { + return pkgerrors.New("No Data to store") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Insert the data and then add the objectID to the masterTable + res, err := c.InsertOne(ctx, bson.D{ + {tag, data}, + }) + if err != nil { + return pkgerrors.Errorf("Error inserting into database: %s", err.Error()) + } + + //Add objectID of created data to masterKey document + //Create masterkey document if it does not exist + filter := bson.D{{"key", key}} + + _, err = decodeBytes( + c.FindOneAndUpdate( + ctx, + filter, + bson.D{ + {"$set", bson.D{ + {tag, res.InsertedID}, + }}, + }, + options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After))) + + if err != nil { + return pkgerrors.Errorf("Error updating master table: %s", err.Error()) + } + + return nil +} + +// Unmarshal implements an unmarshaler for bson data that +// is produced from the mongo database +func (m *MongoStore) Unmarshal(inp []byte, out interface{}) error { + err := bson.Unmarshal(inp, out) + if err != nil { + return pkgerrors.Wrap(err, "Unmarshaling bson") + } + return nil +} + +// Read method returns the data stored for this key and for this particular tag +func (m *MongoStore) Read(coll, key, tag string) ([]byte, error) { + if !m.validateParams(coll, key, tag) { + return nil, pkgerrors.New("Mandatory fields are missing") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get the masterkey document based on given key + filter := bson.D{{"key", key}} + keydata, err := decodeBytes(c.FindOne(context.Background(), filter)) + if err != nil { + return nil, pkgerrors.Errorf("Error finding master table: %s", err.Error()) + } + + //Read the tag objectID from document + tagoid, ok := keydata.Lookup(tag).ObjectIDOK() + if !ok { + return nil, pkgerrors.Errorf("Error finding objectID for tag %s", tag) + } + + //Use tag objectID to read the data from store + filter = bson.D{{"_id", tagoid}} + tagdata, err := decodeBytes(c.FindOne(ctx, filter)) + if err != nil { + return nil, pkgerrors.Errorf("Error reading found object: %s", err.Error()) + } + + //Return the data as a byte array + return tagdata.Lookup(tag).Value, nil +} + +// Helper function that deletes an object by its ID +func (m *MongoStore) deleteObjectByID(coll string, objID primitive.ObjectID) error { + + c := getCollection(coll, m) + ctx := context.Background() + + _, err := c.DeleteOne(ctx, bson.D{{"_id", objID}}) + if err != nil { + return pkgerrors.Errorf("Error Deleting from database: %s", err.Error()) + } + + log.Printf("Deleted Obj with ID %s", objID.String()) + return nil +} + +// Delete method removes a document from the Database that matches key +// TODO: delete all referenced docs if tag is empty string +func (m *MongoStore) Delete(coll, key, tag string) error { + if !m.validateParams(coll, key, tag) { + return pkgerrors.New("Mandatory fields are missing") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get the masterkey document based on given key + filter := bson.D{{"key", key}} + //Remove the tag ID entry from masterkey table + update := bson.D{ + { + "$unset", bson.D{ + {tag, ""}, + }, + }, + } + keydata, err := decodeBytes(c.FindOneAndUpdate(ctx, filter, update, + options.FindOneAndUpdate().SetReturnDocument(options.Before))) + if err != nil { + return pkgerrors.Errorf("Error decoding master table after update: %s", + err.Error()) + } + + //Read the tag objectID from document + elems, err := keydata.Elements() + if err != nil { + return pkgerrors.Errorf("Error reading elements from database: %s", err.Error()) + } + + tagoid, ok := keydata.Lookup(tag).ObjectIDOK() + if !ok { + return pkgerrors.Errorf("Error finding objectID for tag %s", tag) + } + + //Use tag objectID to read the data from store + err = m.deleteObjectByID(coll, tagoid) + if err != nil { + return pkgerrors.Errorf("Error deleting from database: %s", err.Error()) + } + + //Delete master table if no more tags left + //_id, key and tag should be elements in before doc + //if master table needs to be removed too + if len(elems) == 3 { + keyid, ok := keydata.Lookup("_id").ObjectIDOK() + if !ok { + return pkgerrors.Errorf("Error finding objectID for key %s", key) + } + err = m.deleteObjectByID(coll, keyid) + if err != nil { + return pkgerrors.Errorf("Error deleting master table from database: %s", err.Error()) + } + } + + return nil +} + +// ReadAll is used to get all documents in db of a particular tag +func (m *MongoStore) ReadAll(coll, tag string) (map[string][]byte, error) { + if !m.validateParams(coll, tag) { + return nil, pkgerrors.New("Missing collection or tag name") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get all master tables in this collection + filter := bson.D{ + {"key", bson.D{ + {"$exists", true}, + }}, + } + cursor, err := c.Find(ctx, filter) + if err != nil { + return nil, pkgerrors.Errorf("Error reading from database: %s", err.Error()) + } + defer cursor.Close(ctx) + + //Iterate over all the master tables + result := make(map[string][]byte) + for cursor.Next(ctx) { + d, err := cursor.DecodeBytes() + if err != nil { + log.Printf("Unable to decode data in Readall: %s", err.Error()) + continue + } + + //Read key of each master table + key, ok := d.Lookup("key").StringValueOK() + if !ok { + log.Printf("Unable to read key string from mastertable %s", err.Error()) + continue + } + + //Get objectID of tag document + tid, ok := d.Lookup(tag).ObjectIDOK() + if !ok { + log.Printf("Did not find tag: %s", tag) + continue + } + + //Find tag document and unmarshal it into []byte + tagData, err := decodeBytes(c.FindOne(ctx, bson.D{{"_id", tid}})) + if err != nil { + log.Printf("Unable to decode tag data %s", err.Error()) + continue + } + result[key] = tagData.Lookup(tag).Value + } + + if len(result) == 0 { + return result, pkgerrors.Errorf("Did not find any objects with tag: %s", tag) + } + + return result, nil +} diff --git a/src/k8splugin/db/mongo_test.go b/src/k8splugin/db/mongo_test.go new file mode 100644 index 00000000..1663e774 --- /dev/null +++ b/src/k8splugin/db/mongo_test.go @@ -0,0 +1,530 @@ +// +build unit + +/* + * Copyright 2018 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package db + +import ( + "bytes" + "context" + "github.com/mongodb/mongo-go-driver/bson" + "github.com/mongodb/mongo-go-driver/mongo" + "github.com/mongodb/mongo-go-driver/mongo/options" + pkgerrors "github.com/pkg/errors" + "reflect" + "strings" + "testing" +) + +// Implements the mongo.Cursor interface +type mockCursor struct { + mongo.Cursor + err error + bson bson.Raw + count int +} + +func (mc *mockCursor) Next(ctx context.Context) bool { + if mc.count > 0 { + mc.count = mc.count - 1 + return true + } + return false +} + +func (mc *mockCursor) DecodeBytes() (bson.Raw, error) { + return mc.bson, mc.err +} + +func (mc *mockCursor) Close(ctx context.Context) error { + return nil +} + +//Implements the functions used currently in mongo.go +type mockCollection struct { + Err error + mCursor mongo.Cursor +} + +func (c *mockCollection) InsertOne(ctx context.Context, document interface{}, + opts ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) { + + if c.Err != nil { + return nil, c.Err + } + + return &mongo.InsertOneResult{InsertedID: "_id1234"}, nil +} + +func (c *mockCollection) FindOne(ctx context.Context, filter interface{}, + opts ...*options.FindOneOptions) *mongo.SingleResult { + + return &mongo.SingleResult{} +} + +func (c *mockCollection) FindOneAndUpdate(ctx context.Context, filter interface{}, + update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult { + + return &mongo.SingleResult{} +} + +func (c *mockCollection) DeleteOne(ctx context.Context, filter interface{}, + opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) { + + return nil, c.Err +} + +func (c *mockCollection) Find(ctx context.Context, filter interface{}, + opts ...*options.FindOptions) (mongo.Cursor, error) { + + return c.mCursor, c.Err +} + +func TestCreate(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + }{ + { + label: "Successfull creation of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + "data": "Data In String Format", + }, + bson: bson.Raw{'\x08', '\x00', '\x00', '\x00', '\x0A', 'x', '\x00', '\x00'}, + mockColl: &mockCollection{}, + }, + { + label: "UnSuccessfull creation of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + "data": "Data In String Format", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "key": "", + "tag": "", + "data": "", + }, + expectedError: "No Data to store", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.bson, testCase.mockColl.Err + } + + err := m.Create(testCase.input["coll"].(string), testCase.input["key"].(string), + testCase.input["tag"].(string), testCase.input["data"]) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Create method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Create method returned an error (%s)", err) + } + } + }) + } +} + +func TestRead(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + expected []byte + }{ + { + label: "Successfull Read of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "metadata", + }, + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + // This is not the document because we are mocking decodeBytes + expected: []byte{92, 17, 81, 86, 201, 117, 80, 71, 227, 24, 187, 253}, + }, + { + label: "UnSuccessfull Read of entry: object not found", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "badtag", + }, + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + expectedError: "Error finding objectID", + }, + { + label: "UnSuccessfull Read of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "key": "", + "tag": "", + }, + expectedError: "Mandatory fields are missing", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.bson, testCase.mockColl.Err + } + got, err := m.Read(testCase.input["coll"].(string), testCase.input["key"].(string), + testCase.input["tag"].(string)) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Read method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Read method returned an error (%s)", err) + } + } else { + if bytes.Compare(got, testCase.expected) != 0 { + t.Fatalf("Read returned unexpected data: %s, expected: %s", + string(got), testCase.expected) + } + } + }) + } +} + +func TestDelete(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + }{ + { + label: "Successfull Delete of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "metadata", + }, + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + }, + { + label: "UnSuccessfull Delete of entry", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "UnSuccessfull Delete, key not found", + input: map[string]interface{}{ + "coll": "collname", + "key": "keyvalue", + "tag": "tagName", + }, + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + mockColl: &mockCollection{}, + expectedError: "Error finding objectID", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "key": "", + "tag": "", + }, + expectedError: "Mandatory fields are missing", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.bson, testCase.mockColl.Err + } + err := m.Delete(testCase.input["coll"].(string), testCase.input["key"].(string), + testCase.input["tag"].(string)) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Delete method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Delete method returned an error (%s)", err) + } + } + }) + } +} + +func TestReadAll(t *testing.T) { + testCases := []struct { + label string + input map[string]interface{} + mockColl *mockCollection + bson bson.Raw + expectedError string + expected map[string][]byte + }{ + { + label: "Successfully Read all entries", + input: map[string]interface{}{ + "coll": "collname", + "tag": "metadata", + }, + mockColl: &mockCollection{ + mCursor: &mockCursor{ + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + count: 1, + }, + }, + expected: map[string][]byte{ + "b82c4bb1-09ff-6093-4d58-8327b94e1e20": []byte{ + 92, 17, 81, 86, 201, 117, 80, 71, 227, 24, 187, 253}, + }, + }, + { + label: "UnSuccessfully Read of all entries", + input: map[string]interface{}{ + "coll": "collname", + "tag": "tagName", + }, + mockColl: &mockCollection{ + Err: pkgerrors.New("DB Error"), + }, + expectedError: "DB Error", + }, + { + label: "UnSuccessfull Readall, tag not found", + input: map[string]interface{}{ + "coll": "collname", + "tag": "tagName", + }, + mockColl: &mockCollection{ + mCursor: &mockCursor{ + // Binary form of + // { + // "_id" : ObjectId("5c115156777ff85654248ae1"), + // "key" : "b82c4bb1-09ff-6093-4d58-8327b94e1e20", + // "metadata" : ObjectId("5c115156c9755047e318bbfd") + // } + bson: bson.Raw{ + '\x5a', '\x00', '\x00', '\x00', '\x07', '\x5f', '\x69', '\x64', + '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', '\xf8', + '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x02', '\x6b', '\x65', + '\x79', '\x00', '\x25', '\x00', '\x00', '\x00', '\x62', '\x38', + '\x32', '\x63', '\x34', '\x62', '\x62', '\x31', '\x2d', '\x30', + '\x39', '\x66', '\x66', '\x2d', '\x36', '\x30', '\x39', '\x33', + '\x2d', '\x34', '\x64', '\x35', '\x38', '\x2d', '\x38', '\x33', + '\x32', '\x37', '\x62', '\x39', '\x34', '\x65', '\x31', '\x65', + '\x32', '\x30', '\x00', '\x07', '\x6d', '\x65', '\x74', '\x61', + '\x64', '\x61', '\x74', '\x61', '\x00', '\x5c', '\x11', '\x51', + '\x56', '\xc9', '\x75', '\x50', '\x47', '\xe3', '\x18', '\xbb', + '\xfd', '\x00', + }, + count: 1, + }, + }, + expectedError: "Did not find any objects with tag", + }, + { + label: "Missing input fields", + input: map[string]interface{}{ + "coll": "", + "tag": "", + }, + expectedError: "Missing collection or tag name", + mockColl: &mockCollection{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + m, _ := NewMongoStore("name", &mongo.Database{}) + // Override the getCollection function with our mocked version + getCollection = func(coll string, m *MongoStore) MongoCollection { + return testCase.mockColl + } + + decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return testCase.mockColl.mCursor.DecodeBytes() + } + + got, err := m.ReadAll(testCase.input["coll"].(string), testCase.input["tag"].(string)) + if err != nil { + if testCase.expectedError == "" { + t.Fatalf("Readall method returned an un-expected (%s)", err) + } + if !strings.Contains(string(err.Error()), testCase.expectedError) { + t.Fatalf("Readall method returned an error (%s)", err) + } + } else { + if reflect.DeepEqual(got, testCase.expected) == false { + t.Fatalf("Readall returned unexpected data: %v, expected: %v", + got, testCase.expected) + } + } + }) + } +} diff --git a/src/k8splugin/db/store.go b/src/k8splugin/db/store.go index c1a8b31f..a235597a 100644 --- a/src/k8splugin/db/store.go +++ b/src/k8splugin/db/store.go @@ -25,21 +25,38 @@ var DBconn Store // Store is an interface for accessing a database type Store interface { + // Returns nil if db health is good HealthCheck() error - Create(string, string) error - Read(string) (string, error) - // Update(string) (string, error) - Delete(string) error + // Unmarshal implements any unmarshaling needed for the database + Unmarshal(inp []byte, out interface{}) error - ReadAll(string) ([]string, error) + // Creates a new master table with key and links data with tag and + // creates a pointer to the newly added data in the master table + Create(table, key, tag string, data interface{}) error + + // Reads data for a particular key with specific tag. + Read(table, key, tag string) ([]byte, error) + + //TODO: Update(context.Context, string, interface{}) error + + // Deletes a specific tag data for key. + // TODO: If tag is empty, it will delete all tags under key. + Delete(table, key, tag string) error + + // Reads all master tables and data from the specified tag in table + ReadAll(table, tag string) (map[string][]byte, error) } // CreateDBClient creates the DB client func CreateDBClient(dbType string) error { var err error switch dbType { + case "mongo": + // create a mongodb database with k8splugin as the name + DBconn, err = NewMongoStore("k8splugin", nil) case "consul": + // create a consul kv store DBconn, err = NewConsulStore(nil) default: return pkgerrors.New(dbType + "DB not supported") diff --git a/src/k8splugin/db/store_test.go b/src/k8splugin/db/store_test.go index 9bbe4a92..eed7065f 100644 --- a/src/k8splugin/db/store_test.go +++ b/src/k8splugin/db/store_test.go @@ -23,9 +23,9 @@ import ( func TestCreateDBClient(t *testing.T) { t.Run("Successfully create DB client", func(t *testing.T) { - expected := &ConsulStore{} + expected := &MongoStore{} - err := CreateDBClient("consul") + err := CreateDBClient("mongo") if err != nil { t.Fatalf("CreateDBClient returned an error (%s)", err) } diff --git a/src/k8splugin/db/testing.go b/src/k8splugin/db/testing.go index 672fcbfb..4b7e6078 100644 --- a/src/k8splugin/db/testing.go +++ b/src/k8splugin/db/testing.go @@ -16,7 +16,8 @@ limitations under the License. package db import ( - "github.com/hashicorp/consul/api" + "encoding/json" + pkgerrors "github.com/pkg/errors" ) //Creating an embedded interface via anonymous variable @@ -24,42 +25,45 @@ import ( //interface even if we are not implementing all the methods in it type MockDB struct { Store - Items api.KVPairs + Items map[string][]byte Err error } -func (m *MockDB) Create(key string, value string) error { +func (m *MockDB) Create(table, key, tag string, data interface{}) error { return m.Err } -func (m *MockDB) Read(key string) (string, error) { +// MockDB uses simple JSON and not BSON +func (m *MockDB) Unmarshal(inp []byte, out interface{}) error { + err := json.Unmarshal(inp, out) + if err != nil { + return pkgerrors.Wrap(err, "Unmarshaling json") + } + return nil +} + +func (m *MockDB) Read(table, key, tag string) ([]byte, error) { if m.Err != nil { - return "", m.Err + return nil, m.Err } - for _, kvpair := range m.Items { - if kvpair.Key == key { - return string(kvpair.Value), nil + for k, v := range m.Items { + if k == key { + return v, nil } } - return "", nil + return nil, m.Err } -func (m *MockDB) Delete(key string) error { +func (m *MockDB) Delete(table, key, tag string) error { return m.Err } -func (m *MockDB) ReadAll(prefix string) ([]string, error) { +func (m *MockDB) ReadAll(table, tag string) (map[string][]byte, error) { if m.Err != nil { - return []string{}, m.Err - } - - var res []string - - for _, keypair := range m.Items { - res = append(res, keypair.Key) + return nil, m.Err } - return res, nil + return m.Items, nil } diff --git a/src/k8splugin/go.mod b/src/k8splugin/go.mod index b4f4558b..cf2c7d85 100644 --- a/src/k8splugin/go.mod +++ b/src/k8splugin/go.mod @@ -1,35 +1,45 @@ module k8splugin require ( - github.com/ghodss/yaml v1.0.0 - github.com/gogo/protobuf v1.0.0 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/protobuf v1.1.0 - github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf - github.com/googleapis/gnostic v0.2.0 - github.com/gorilla/context v1.1.1 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-stack/stack v1.8.0 // indirect + github.com/gogo/protobuf v1.0.0 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/protobuf v1.2.0 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect + github.com/google/go-cmp v0.2.0 // indirect + github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect + github.com/googleapis/gnostic v0.2.0 // indirect + github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.6.2 - github.com/hashicorp/consul v1.2.2 - github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186 - github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 + github.com/hashicorp/consul v1.4.0 + github.com/hashicorp/go-cleanhttp v0.5.0 // indirect + github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 // indirect github.com/hashicorp/go-uuid v1.0.0 - github.com/hashicorp/serf v0.8.1 - github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c - github.com/imdario/mergo v0.3.5 - github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece - github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff - github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd - github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81 + github.com/hashicorp/serf v0.8.1 // indirect + github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece // indirect + github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81 // indirect + github.com/mongodb/mongo-go-driver v0.1.0 github.com/pkg/errors v0.8.0 - github.com/spf13/pflag v1.0.1 - golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4 - golang.org/x/net v0.0.0-20180611182652-db08ff08e862 - golang.org/x/sys v0.0.0-20180611080425-bff228c7b664 - golang.org/x/text v0.3.0 - golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 - gopkg.in/inf.v0 v0.9.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.1 // indirect + github.com/stretchr/testify v1.2.2 // indirect + github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 // indirect + github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect + github.com/xdg/stringprep v1.0.0 // indirect + golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4 // indirect + golang.org/x/net v0.0.0-20180724234803-3673e40ba225 + golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect + golang.org/x/sys v0.0.0-20180611080425-bff228c7b664 // indirect + golang.org/x/text v0.3.0 // indirect + golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.2.1 k8s.io/api v0.0.0-20180607235014-72d6e4405f81 k8s.io/apimachinery v0.0.0-20180515182440-31dade610c05 diff --git a/src/k8splugin/go.sum b/src/k8splugin/go.sum index 4a10051a..bcb622c7 100644 --- a/src/k8splugin/go.sum +++ b/src/k8splugin/go.sum @@ -1,24 +1,33 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.0.0 h1:2jyBKDKU/8v3v2xVR2PtiWQviFUyiaGk2rpfyFT8rTM= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/hashicorp/consul v1.2.2 h1:C5FurAZWLQ+XAjmL9g6rXbPlwxyyz8DvTL0WCAxTLAo= -github.com/hashicorp/consul v1.2.2/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= -github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186 h1:URgjUo+bs1KwatoNbwG0uCO4dHN4r1jsp4a5AGgHRjo= -github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/consul v1.4.0 h1:PQTW4xCuAExEiSbhrsFsikzbW5gVBoi74BjUvYFyKHw= +github.com/hashicorp/consul v1.4.0/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= +github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 h1:9HVkPxOpo+yO93Ah4yrO67d/qh0fbLLWbKqhYjyHq9A= github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= @@ -31,28 +40,41 @@ github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece h1:3HJXp/18JmMk5sjBP3LDUBtWjczCvynxaeAF6b6kWp8= github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff h1:jM4Eo4qMmmcqePS3u6X2lcEELtVuXWkWJIS/pRI3oSk= -github.com/mitchellh/go-homedir v0.0.0-20180801233206-58046073cbff/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699 h1:KXZJFdun9knAVAR8tg/aHJEr5DgtcbqyvzacK+CDCaI= -github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81 h1:ImOHKpmdLPXWX5KSYquUWXKaopEPuY7TPPUo18u9aOI= github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mongodb/mongo-go-driver v0.1.0 h1:LcpPFw0tNumIAakvNrkI9S9wdX0iOxvMLw/+hcAdHaU= +github.com/mongodb/mongo-go-driver v0.1.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 h1:BP2bjP495BBPaBcS5rmqviTfrOkN5rO5ceKAMRZCRFc= +github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4 h1:wviDUSmtheHRBfoY8B9U8ELl2USoXi2YFwdGdpIIkzI= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20180611182652-db08ff08e862 h1:JZi6BqOZ+iSgmLWe6llhGrNnEnK+YB/MRkStwnEfbqM= -golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180611080425-bff228c7b664 h1:GvcVmbE8Pa64iW3MTrVA9mxHx1HEjSSWV6zF1JSlFcg= golang.org/x/sys v0.0.0-20180611080425-bff228c7b664/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/src/k8splugin/rb/definition.go b/src/k8splugin/rb/definition.go index 2d1c2cd0..02ecc5ae 100644 --- a/src/k8splugin/rb/definition.go +++ b/src/k8splugin/rb/definition.go @@ -47,15 +47,19 @@ type DefinitionManager interface { // DefinitionClient implements the DefinitionManager // It will also be used to maintain some localized state type DefinitionClient struct { - keyPrefix string + storeName string + tagMeta, tagContent string } // NewDefinitionClient returns an instance of the DefinitionClient // which implements the DefinitionManager -// Uses rb/def prefix +// Uses rbdef collection in underlying db func NewDefinitionClient() *DefinitionClient { return &DefinitionClient{ - keyPrefix: "rb/def/"} + storeName: "rbdef", + tagMeta: "metadata", + tagContent: "content", + } } // Create an entry for the resource in the database @@ -64,14 +68,9 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) { if def.UUID == "" { def.UUID, _ = uuid.GenerateUUID() } - key := v.keyPrefix + def.UUID - - serData, err := db.Serialize(def) - if err != nil { - return Definition{}, pkgerrors.Wrap(err, "Serialize Resource Bundle Definition") - } + key := def.UUID - err = db.DBconn.Create(key, serData) + err := db.DBconn.Create(v.storeName, key, v.tagMeta, def) if err != nil { return Definition{}, pkgerrors.Wrap(err, "Creating DB Entry") } @@ -81,45 +80,39 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) { // List all resource entries in the database func (v *DefinitionClient) List() ([]Definition, error) { - strArray, err := db.DBconn.ReadAll(v.keyPrefix) - if err != nil { + res, err := db.DBconn.ReadAll(v.storeName, v.tagMeta) + if err != nil || len(res) == 0 { return []Definition{}, pkgerrors.Wrap(err, "Listing Resource Bundle Definitions") } - var retData []Definition - - for _, key := range strArray { - value, err := db.DBconn.Read(key) - if err != nil { - log.Printf("Error Reading Key: %s", key) - continue - } - if value != "" { + var results []Definition + for key, value := range res { + if len(value) > 0 { def := Definition{} - err = db.DeSerialize(value, &def) + err = db.DBconn.Unmarshal(value, &def) if err != nil { - log.Printf("Error Deserializing Value: %s", value) + log.Printf("Error Unmarshaling value for: %s", key) continue } - retData = append(retData, def) + results = append(results, def) } } - return retData, nil + return results, nil } // Get returns the Resource Bundle Definition for corresponding ID func (v *DefinitionClient) Get(id string) (Definition, error) { - value, err := db.DBconn.Read(v.keyPrefix + id) + value, err := db.DBconn.Read(v.storeName, id, v.tagMeta) if err != nil { return Definition{}, pkgerrors.Wrap(err, "Get Resource Bundle definition") } - if value != "" { + if value != nil { def := Definition{} - err = db.DeSerialize(value, &def) + err = db.DBconn.Unmarshal(value, &def) if err != nil { - return Definition{}, pkgerrors.Wrap(err, "Deserializing Value") + return Definition{}, pkgerrors.Wrap(err, "Unmarshaling Value") } return def, nil } @@ -129,7 +122,7 @@ func (v *DefinitionClient) Get(id string) (Definition, error) { // Delete the Resource Bundle definition from database func (v *DefinitionClient) Delete(id string) error { - err := db.DBconn.Delete(v.keyPrefix + id) + err := db.DBconn.Delete(v.storeName, id, v.tagMeta) if err != nil { return pkgerrors.Wrap(err, "Delete Resource Bundle Definitions") } @@ -140,22 +133,22 @@ func (v *DefinitionClient) Delete(id string) error { // Upload the contents of resource bundle into database func (v *DefinitionClient) Upload(id string, inp []byte) error { - //ignore the returned data here. + //ignore the returned data here _, err := v.Get(id) if err != nil { - return pkgerrors.Errorf("Invalid ID provided %s", err.Error()) + return pkgerrors.Errorf("Invalid ID provided: %s", err.Error()) } err = isTarGz(bytes.NewBuffer(inp)) if err != nil { - return pkgerrors.Errorf("Error in file format %s", err.Error()) + return pkgerrors.Errorf("Error in file format: %s", err.Error()) } + //Encode given byte stream to text for storage encodedStr := base64.StdEncoding.EncodeToString(inp) - key := v.keyPrefix + id + "/content" - err = db.DBconn.Create(key, encodedStr) + err = db.DBconn.Create(v.storeName, id, encodedStr, v.tagContent) if err != nil { - return pkgerrors.Errorf("Error uploading data to db %s", err.Error()) + return pkgerrors.Errorf("Error uploading data to db: %s", err.Error()) } return nil diff --git a/src/k8splugin/rb/definition_test.go b/src/k8splugin/rb/definition_test.go index 58eb718d..1e488678 100644 --- a/src/k8splugin/rb/definition_test.go +++ b/src/k8splugin/rb/definition_test.go @@ -24,7 +24,6 @@ import ( "strings" "testing" - "github.com/hashicorp/consul/api" pkgerrors "github.com/pkg/errors" ) @@ -110,21 +109,17 @@ func TestList(t *testing.T) { }, expectedError: "", mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655441111", - Value: []byte("{\"name\":\"testresourcebundle2\"," + + "123e4567-e89b-12d3-a456-426655441111": []byte( + "{\"name\":\"testresourcebundle2\"," + "\"description\":\"testresourcebundle2\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," + "\"service-type\":\"dns\"}"), - }, }, }, }, @@ -179,14 +174,12 @@ func TestGet(t *testing.T) { }, expectedError: "", mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -293,14 +286,12 @@ func TestUpload(t *testing.T) { 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -330,14 +321,12 @@ func TestUpload(t *testing.T) { 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655441111", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655441111": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -350,14 +339,12 @@ func TestUpload(t *testing.T) { 0x00, 0xff, 0xf2, 0x48, 0xcd, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, diff --git a/vagrant/tests/plugin.sh b/vagrant/tests/plugin.sh index 16d8d306..55be1686 100755 --- a/vagrant/tests/plugin.sh +++ b/vagrant/tests/plugin.sh @@ -88,7 +88,7 @@ echo "VNF details $vnf_details" echo "Deleting $vnf_id VNF Instance" curl -X DELETE "${base_url}${cloud_region_id}/${namespace}/${vnf_id}" -if [[ -n $(curl -s -X GET "${base_url}${cloud_region_id}/${namespace}/${vnf_id}") ]]; then +if [[ 200 -eq $(curl -o /dev/null -w %{http_code} -s -X GET "${base_url}${cloud_region_id}/${namespace}/${vnf_id}") ]]; then echo "VNF Instance not deleted" exit 1 fi |