aboutsummaryrefslogtreecommitdiffstats
path: root/src/k8splugin/rb
diff options
context:
space:
mode:
authorKiran Kamineni <kiran.k.kamineni@intel.com>2018-12-10 13:34:16 -0800
committerKiran Kamineni <kiran.k.kamineni@intel.com>2018-12-14 14:26:26 -0800
commitd203ccf9174e450b6f5a6a9aabea95b73c9973ff (patch)
tree702034eafbd28b47f33023607ee213cf92519a99 /src/k8splugin/rb
parentaa56022e0fba3c358e46e8671d9a0cd36094ebaa (diff)
Migrating from consul to mongodb for backend
Migrating to mongodb from consul. The main reason being the value size limitation of 512kb in consul. See https://jira.onap.org/browse/MULTICLOUD-426 for details. This requires a little bit of hierarchy management and data management. We are no longer converting structs to json encoded strings. The underlying db supports structs without any modifications. Also, since Mongo has the concept of collections, each submodule can use its own collection for storage as needed. Definition uses a collection called rbdef right now. P10: Enabling unit tests for mongo.go. This requires the usage of aliased functions. P11: Expanded unit tests for all functions in mongo.go P12: Refactored parameter validation. Removed TestHealthCheck as we are not mocking any of the db commands right now Checking return value of read with an expected value P13: Adding back consul support. Fixing functional test Full consul implementation check and modifications is being tracked by MULTICLOUD-427 P15: Fix ReadAll unit test and corresponding code ReadAll now returns error when no objects are found Issue-ID: MULTICLOUD-426 Change-Id: I42d239b324025fc4ef4e561790aceeff794001ef Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>
Diffstat (limited to 'src/k8splugin/rb')
-rw-r--r--src/k8splugin/rb/definition.go65
-rw-r--r--src/k8splugin/rb/definition_test.go47
2 files changed, 46 insertions, 66 deletions
diff --git a/src/k8splugin/rb/definition.go b/src/k8splugin/rb/definition.go
index 2d1c2cd0..02ecc5ae 100644
--- a/src/k8splugin/rb/definition.go
+++ b/src/k8splugin/rb/definition.go
@@ -47,15 +47,19 @@ type DefinitionManager interface {
// DefinitionClient implements the DefinitionManager
// It will also be used to maintain some localized state
type DefinitionClient struct {
- keyPrefix string
+ storeName string
+ tagMeta, tagContent string
}
// NewDefinitionClient returns an instance of the DefinitionClient
// which implements the DefinitionManager
-// Uses rb/def prefix
+// Uses rbdef collection in underlying db
func NewDefinitionClient() *DefinitionClient {
return &DefinitionClient{
- keyPrefix: "rb/def/"}
+ storeName: "rbdef",
+ tagMeta: "metadata",
+ tagContent: "content",
+ }
}
// Create an entry for the resource in the database
@@ -64,14 +68,9 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) {
if def.UUID == "" {
def.UUID, _ = uuid.GenerateUUID()
}
- key := v.keyPrefix + def.UUID
-
- serData, err := db.Serialize(def)
- if err != nil {
- return Definition{}, pkgerrors.Wrap(err, "Serialize Resource Bundle Definition")
- }
+ key := def.UUID
- err = db.DBconn.Create(key, serData)
+ err := db.DBconn.Create(v.storeName, key, v.tagMeta, def)
if err != nil {
return Definition{}, pkgerrors.Wrap(err, "Creating DB Entry")
}
@@ -81,45 +80,39 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) {
// List all resource entries in the database
func (v *DefinitionClient) List() ([]Definition, error) {
- strArray, err := db.DBconn.ReadAll(v.keyPrefix)
- if err != nil {
+ res, err := db.DBconn.ReadAll(v.storeName, v.tagMeta)
+ if err != nil || len(res) == 0 {
return []Definition{}, pkgerrors.Wrap(err, "Listing Resource Bundle Definitions")
}
- var retData []Definition
-
- for _, key := range strArray {
- value, err := db.DBconn.Read(key)
- if err != nil {
- log.Printf("Error Reading Key: %s", key)
- continue
- }
- if value != "" {
+ var results []Definition
+ for key, value := range res {
+ if len(value) > 0 {
def := Definition{}
- err = db.DeSerialize(value, &def)
+ err = db.DBconn.Unmarshal(value, &def)
if err != nil {
- log.Printf("Error Deserializing Value: %s", value)
+ log.Printf("Error Unmarshaling value for: %s", key)
continue
}
- retData = append(retData, def)
+ results = append(results, def)
}
}
- return retData, nil
+ return results, nil
}
// Get returns the Resource Bundle Definition for corresponding ID
func (v *DefinitionClient) Get(id string) (Definition, error) {
- value, err := db.DBconn.Read(v.keyPrefix + id)
+ value, err := db.DBconn.Read(v.storeName, id, v.tagMeta)
if err != nil {
return Definition{}, pkgerrors.Wrap(err, "Get Resource Bundle definition")
}
- if value != "" {
+ if value != nil {
def := Definition{}
- err = db.DeSerialize(value, &def)
+ err = db.DBconn.Unmarshal(value, &def)
if err != nil {
- return Definition{}, pkgerrors.Wrap(err, "Deserializing Value")
+ return Definition{}, pkgerrors.Wrap(err, "Unmarshaling Value")
}
return def, nil
}
@@ -129,7 +122,7 @@ func (v *DefinitionClient) Get(id string) (Definition, error) {
// Delete the Resource Bundle definition from database
func (v *DefinitionClient) Delete(id string) error {
- err := db.DBconn.Delete(v.keyPrefix + id)
+ err := db.DBconn.Delete(v.storeName, id, v.tagMeta)
if err != nil {
return pkgerrors.Wrap(err, "Delete Resource Bundle Definitions")
}
@@ -140,22 +133,22 @@ func (v *DefinitionClient) Delete(id string) error {
// Upload the contents of resource bundle into database
func (v *DefinitionClient) Upload(id string, inp []byte) error {
- //ignore the returned data here.
+ //ignore the returned data here
_, err := v.Get(id)
if err != nil {
- return pkgerrors.Errorf("Invalid ID provided %s", err.Error())
+ return pkgerrors.Errorf("Invalid ID provided: %s", err.Error())
}
err = isTarGz(bytes.NewBuffer(inp))
if err != nil {
- return pkgerrors.Errorf("Error in file format %s", err.Error())
+ return pkgerrors.Errorf("Error in file format: %s", err.Error())
}
+ //Encode given byte stream to text for storage
encodedStr := base64.StdEncoding.EncodeToString(inp)
- key := v.keyPrefix + id + "/content"
- err = db.DBconn.Create(key, encodedStr)
+ err = db.DBconn.Create(v.storeName, id, encodedStr, v.tagContent)
if err != nil {
- return pkgerrors.Errorf("Error uploading data to db %s", err.Error())
+ return pkgerrors.Errorf("Error uploading data to db: %s", err.Error())
}
return nil
diff --git a/src/k8splugin/rb/definition_test.go b/src/k8splugin/rb/definition_test.go
index 58eb718d..1e488678 100644
--- a/src/k8splugin/rb/definition_test.go
+++ b/src/k8splugin/rb/definition_test.go
@@ -24,7 +24,6 @@ import (
"strings"
"testing"
- "github.com/hashicorp/consul/api"
pkgerrors "github.com/pkg/errors"
)
@@ -110,21 +109,17 @@ func TestList(t *testing.T) {
},
expectedError: "",
mockdb: &db.MockDB{
- Items: api.KVPairs{
- &api.KVPair{
- Key: "rb/def/123e4567-e89b-12d3-a456-426655440000",
- Value: []byte("{\"name\":\"testresourcebundle\"," +
+ Items: map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655440000": []byte(
+ "{\"name\":\"testresourcebundle\"," +
"\"description\":\"testresourcebundle\"," +
"\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
"\"service-type\":\"firewall\"}"),
- },
- &api.KVPair{
- Key: "rb/def/123e4567-e89b-12d3-a456-426655441111",
- Value: []byte("{\"name\":\"testresourcebundle2\"," +
+ "123e4567-e89b-12d3-a456-426655441111": []byte(
+ "{\"name\":\"testresourcebundle2\"," +
"\"description\":\"testresourcebundle2\"," +
"\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," +
"\"service-type\":\"dns\"}"),
- },
},
},
},
@@ -179,14 +174,12 @@ func TestGet(t *testing.T) {
},
expectedError: "",
mockdb: &db.MockDB{
- Items: api.KVPairs{
- &api.KVPair{
- Key: "rb/def/123e4567-e89b-12d3-a456-426655440000",
- Value: []byte("{\"name\":\"testresourcebundle\"," +
+ Items: map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655440000": []byte(
+ "{\"name\":\"testresourcebundle\"," +
"\"description\":\"testresourcebundle\"," +
"\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
"\"service-type\":\"firewall\"}"),
- },
},
},
},
@@ -293,14 +286,12 @@ func TestUpload(t *testing.T) {
0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
},
mockdb: &db.MockDB{
- Items: api.KVPairs{
- &api.KVPair{
- Key: "rb/def/123e4567-e89b-12d3-a456-426655440000",
- Value: []byte("{\"name\":\"testresourcebundle\"," +
+ Items: map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655440000": []byte(
+ "{\"name\":\"testresourcebundle\"," +
"\"description\":\"testresourcebundle\"," +
"\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
"\"service-type\":\"firewall\"}"),
- },
},
},
},
@@ -330,14 +321,12 @@ func TestUpload(t *testing.T) {
0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
},
mockdb: &db.MockDB{
- Items: api.KVPairs{
- &api.KVPair{
- Key: "rb/def/123e4567-e89b-12d3-a456-426655441111",
- Value: []byte("{\"name\":\"testresourcebundle\"," +
+ Items: map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655441111": []byte(
+ "{\"name\":\"testresourcebundle\"," +
"\"description\":\"testresourcebundle\"," +
"\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
"\"service-type\":\"firewall\"}"),
- },
},
},
},
@@ -350,14 +339,12 @@ func TestUpload(t *testing.T) {
0x00, 0xff, 0xf2, 0x48, 0xcd,
},
mockdb: &db.MockDB{
- Items: api.KVPairs{
- &api.KVPair{
- Key: "rb/def/123e4567-e89b-12d3-a456-426655440000",
- Value: []byte("{\"name\":\"testresourcebundle\"," +
+ Items: map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655440000": []byte(
+ "{\"name\":\"testresourcebundle\"," +
"\"description\":\"testresourcebundle\"," +
"\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
"\"service-type\":\"firewall\"}"),
- },
},
},
},