diff options
author | Kiran Kamineni <kiran.k.kamineni@intel.com> | 2018-12-10 13:34:16 -0800 |
---|---|---|
committer | Kiran Kamineni <kiran.k.kamineni@intel.com> | 2018-12-14 14:26:26 -0800 |
commit | d203ccf9174e450b6f5a6a9aabea95b73c9973ff (patch) | |
tree | 702034eafbd28b47f33023607ee213cf92519a99 /src/k8splugin/rb | |
parent | aa56022e0fba3c358e46e8671d9a0cd36094ebaa (diff) |
Migrating from consul to mongodb for backend
Migrating to mongodb from consul.
The main reason being the value size limitation of 512kb in consul.
See https://jira.onap.org/browse/MULTICLOUD-426 for details.
This requires a little bit of hierarchy management
and data management.
We are no longer converting structs to json encoded
strings. The underlying db supports structs without any modifications.
Also, since Mongo has the concept of collections, each submodule can
use its own collection for storage as needed.
Definition uses a collection called rbdef right now.
P10: Enabling unit tests for mongo.go. This requires the usage
of aliased functions.
P11: Expanded unit tests for all functions in mongo.go
P12: Refactored parameter validation.
Removed TestHealthCheck as we are not mocking any of the
db commands right now
Checking return value of read with an expected value
P13: Adding back consul support.
Fixing functional test
Full consul implementation check and modifications is
being tracked by MULTICLOUD-427
P15: Fix ReadAll unit test and corresponding code
ReadAll now returns error when no objects are found
Issue-ID: MULTICLOUD-426
Change-Id: I42d239b324025fc4ef4e561790aceeff794001ef
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>
Diffstat (limited to 'src/k8splugin/rb')
-rw-r--r-- | src/k8splugin/rb/definition.go | 65 | ||||
-rw-r--r-- | src/k8splugin/rb/definition_test.go | 47 |
2 files changed, 46 insertions, 66 deletions
diff --git a/src/k8splugin/rb/definition.go b/src/k8splugin/rb/definition.go index 2d1c2cd0..02ecc5ae 100644 --- a/src/k8splugin/rb/definition.go +++ b/src/k8splugin/rb/definition.go @@ -47,15 +47,19 @@ type DefinitionManager interface { // DefinitionClient implements the DefinitionManager // It will also be used to maintain some localized state type DefinitionClient struct { - keyPrefix string + storeName string + tagMeta, tagContent string } // NewDefinitionClient returns an instance of the DefinitionClient // which implements the DefinitionManager -// Uses rb/def prefix +// Uses rbdef collection in underlying db func NewDefinitionClient() *DefinitionClient { return &DefinitionClient{ - keyPrefix: "rb/def/"} + storeName: "rbdef", + tagMeta: "metadata", + tagContent: "content", + } } // Create an entry for the resource in the database @@ -64,14 +68,9 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) { if def.UUID == "" { def.UUID, _ = uuid.GenerateUUID() } - key := v.keyPrefix + def.UUID - - serData, err := db.Serialize(def) - if err != nil { - return Definition{}, pkgerrors.Wrap(err, "Serialize Resource Bundle Definition") - } + key := def.UUID - err = db.DBconn.Create(key, serData) + err := db.DBconn.Create(v.storeName, key, v.tagMeta, def) if err != nil { return Definition{}, pkgerrors.Wrap(err, "Creating DB Entry") } @@ -81,45 +80,39 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) { // List all resource entries in the database func (v *DefinitionClient) List() ([]Definition, error) { - strArray, err := db.DBconn.ReadAll(v.keyPrefix) - if err != nil { + res, err := db.DBconn.ReadAll(v.storeName, v.tagMeta) + if err != nil || len(res) == 0 { return []Definition{}, pkgerrors.Wrap(err, "Listing Resource Bundle Definitions") } - var retData []Definition - - for _, key := range strArray { - value, err := db.DBconn.Read(key) - if err != nil { - log.Printf("Error Reading Key: %s", key) - continue - } - if value != "" { + var results []Definition + for key, value := range res { + if len(value) > 0 { def := Definition{} - err = db.DeSerialize(value, &def) + err = db.DBconn.Unmarshal(value, &def) if err != nil { - log.Printf("Error Deserializing Value: %s", value) + log.Printf("Error Unmarshaling value for: %s", key) continue } - retData = append(retData, def) + results = append(results, def) } } - return retData, nil + return results, nil } // Get returns the Resource Bundle Definition for corresponding ID func (v *DefinitionClient) Get(id string) (Definition, error) { - value, err := db.DBconn.Read(v.keyPrefix + id) + value, err := db.DBconn.Read(v.storeName, id, v.tagMeta) if err != nil { return Definition{}, pkgerrors.Wrap(err, "Get Resource Bundle definition") } - if value != "" { + if value != nil { def := Definition{} - err = db.DeSerialize(value, &def) + err = db.DBconn.Unmarshal(value, &def) if err != nil { - return Definition{}, pkgerrors.Wrap(err, "Deserializing Value") + return Definition{}, pkgerrors.Wrap(err, "Unmarshaling Value") } return def, nil } @@ -129,7 +122,7 @@ func (v *DefinitionClient) Get(id string) (Definition, error) { // Delete the Resource Bundle definition from database func (v *DefinitionClient) Delete(id string) error { - err := db.DBconn.Delete(v.keyPrefix + id) + err := db.DBconn.Delete(v.storeName, id, v.tagMeta) if err != nil { return pkgerrors.Wrap(err, "Delete Resource Bundle Definitions") } @@ -140,22 +133,22 @@ func (v *DefinitionClient) Delete(id string) error { // Upload the contents of resource bundle into database func (v *DefinitionClient) Upload(id string, inp []byte) error { - //ignore the returned data here. + //ignore the returned data here _, err := v.Get(id) if err != nil { - return pkgerrors.Errorf("Invalid ID provided %s", err.Error()) + return pkgerrors.Errorf("Invalid ID provided: %s", err.Error()) } err = isTarGz(bytes.NewBuffer(inp)) if err != nil { - return pkgerrors.Errorf("Error in file format %s", err.Error()) + return pkgerrors.Errorf("Error in file format: %s", err.Error()) } + //Encode given byte stream to text for storage encodedStr := base64.StdEncoding.EncodeToString(inp) - key := v.keyPrefix + id + "/content" - err = db.DBconn.Create(key, encodedStr) + err = db.DBconn.Create(v.storeName, id, encodedStr, v.tagContent) if err != nil { - return pkgerrors.Errorf("Error uploading data to db %s", err.Error()) + return pkgerrors.Errorf("Error uploading data to db: %s", err.Error()) } return nil diff --git a/src/k8splugin/rb/definition_test.go b/src/k8splugin/rb/definition_test.go index 58eb718d..1e488678 100644 --- a/src/k8splugin/rb/definition_test.go +++ b/src/k8splugin/rb/definition_test.go @@ -24,7 +24,6 @@ import ( "strings" "testing" - "github.com/hashicorp/consul/api" pkgerrors "github.com/pkg/errors" ) @@ -110,21 +109,17 @@ func TestList(t *testing.T) { }, expectedError: "", mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655441111", - Value: []byte("{\"name\":\"testresourcebundle2\"," + + "123e4567-e89b-12d3-a456-426655441111": []byte( + "{\"name\":\"testresourcebundle2\"," + "\"description\":\"testresourcebundle2\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," + "\"service-type\":\"dns\"}"), - }, }, }, }, @@ -179,14 +174,12 @@ func TestGet(t *testing.T) { }, expectedError: "", mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -293,14 +286,12 @@ func TestUpload(t *testing.T) { 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -330,14 +321,12 @@ func TestUpload(t *testing.T) { 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655441111", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655441111": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, @@ -350,14 +339,12 @@ func TestUpload(t *testing.T) { 0x00, 0xff, 0xf2, 0x48, 0xcd, }, mockdb: &db.MockDB{ - Items: api.KVPairs{ - &api.KVPair{ - Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", - Value: []byte("{\"name\":\"testresourcebundle\"," + + Items: map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": []byte( + "{\"name\":\"testresourcebundle\"," + "\"description\":\"testresourcebundle\"," + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + "\"service-type\":\"firewall\"}"), - }, }, }, }, |