Migrating from consul to mongodb for backend

Migrating to mongodb from consul.
The main reason being the value size limitation of 512kb in consul.
See https://jira.onap.org/browse/MULTICLOUD-426 for details.
This requires a little bit of hierarchy management
and data management.
We are no longer converting structs to json encoded
strings. The underlying db supports structs without any modifications.
Also, since Mongo has the concept of collections, each submodule can
use its own collection for storage as needed.
Definition uses a collection called rbdef right now.
P10: Enabling unit tests for mongo.go. This requires the usage
     of aliased functions.
P11: Expanded unit tests for all functions in mongo.go
P12: Refactored parameter validation.
     Removed TestHealthCheck as we are not mocking any of the
       db commands right now
     Checking return value of read with an expected value
P13: Adding back consul support.
     Fixing functional test
     Full consul implementation check and modifications is
     being tracked by MULTICLOUD-427
P15: Fix ReadAll unit test and corresponding code
     ReadAll now returns error when no objects are found

Issue-ID: MULTICLOUD-426
Change-Id: I42d239b324025fc4ef4e561790aceeff794001ef
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>

1 files changed, 29 insertions, 36 deletions

diff --git a/src/k8splugin/rb/definition.go b/src/k8splugin/rb/definition.go
index 2d1c2cd0..02ecc5ae 100644
--- a/src/k8splugin/rb/definition.go
+++ b/src/k8splugin/rb/definition.go
@@ -47,15 +47,19 @@ type DefinitionManager interface {
 // DefinitionClient implements the DefinitionManager
 // It will also be used to maintain some localized state
 type DefinitionClient struct {
-	keyPrefix string
+	storeName           string
+	tagMeta, tagContent string
 }
 
 // NewDefinitionClient returns an instance of the DefinitionClient
 // which implements the DefinitionManager
-// Uses rb/def prefix
+// Uses rbdef collection in underlying db
 func NewDefinitionClient() *DefinitionClient {
 	return &DefinitionClient{
-		keyPrefix: "rb/def/"}
+		storeName:  "rbdef",
+		tagMeta:    "metadata",
+		tagContent: "content",
+	}
 }
 
 // Create an entry for the resource in the database
@@ -64,14 +68,9 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) {
 	if def.UUID == "" {
 		def.UUID, _ = uuid.GenerateUUID()
 	}
-	key := v.keyPrefix + def.UUID
-
-	serData, err := db.Serialize(def)
-	if err != nil {
-		return Definition{}, pkgerrors.Wrap(err, "Serialize Resource Bundle Definition")
-	}
+	key := def.UUID
 
-	err = db.DBconn.Create(key, serData)
+	err := db.DBconn.Create(v.storeName, key, v.tagMeta, def)
 	if err != nil {
 		return Definition{}, pkgerrors.Wrap(err, "Creating DB Entry")
 	}
@@ -81,45 +80,39 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) {
 
 // List all resource entries in the database
 func (v *DefinitionClient) List() ([]Definition, error) {
-	strArray, err := db.DBconn.ReadAll(v.keyPrefix)
-	if err != nil {
+	res, err := db.DBconn.ReadAll(v.storeName, v.tagMeta)
+	if err != nil || len(res) == 0 {
 		return []Definition{}, pkgerrors.Wrap(err, "Listing Resource Bundle Definitions")
 	}
 
-	var retData []Definition
-
-	for _, key := range strArray {
-		value, err := db.DBconn.Read(key)
-		if err != nil {
-			log.Printf("Error Reading Key: %s", key)
-			continue
-		}
-		if value != "" {
+	var results []Definition
+	for key, value := range res {
+		if len(value) > 0 {
 			def := Definition{}
-			err = db.DeSerialize(value, &def)
+			err = db.DBconn.Unmarshal(value, &def)
 			if err != nil {
-				log.Printf("Error Deserializing Value: %s", value)
+				log.Printf("Error Unmarshaling value for: %s", key)
 				continue
 			}
-			retData = append(retData, def)
+			results = append(results, def)
 		}
 	}
 
-	return retData, nil
+	return results, nil
 }
 
 // Get returns the Resource Bundle Definition for corresponding ID
 func (v *DefinitionClient) Get(id string) (Definition, error) {
-	value, err := db.DBconn.Read(v.keyPrefix + id)
+	value, err := db.DBconn.Read(v.storeName, id, v.tagMeta)
 	if err != nil {
 		return Definition{}, pkgerrors.Wrap(err, "Get Resource Bundle definition")
 	}
 
-	if value != "" {
+	if value != nil {
 		def := Definition{}
-		err = db.DeSerialize(value, &def)
+		err = db.DBconn.Unmarshal(value, &def)
 		if err != nil {
-			return Definition{}, pkgerrors.Wrap(err, "Deserializing Value")
+			return Definition{}, pkgerrors.Wrap(err, "Unmarshaling Value")
 		}
 		return def, nil
 	}
@@ -129,7 +122,7 @@ func (v *DefinitionClient) Get(id string) (Definition, error) {
 
 // Delete the Resource Bundle definition from database
 func (v *DefinitionClient) Delete(id string) error {
-	err := db.DBconn.Delete(v.keyPrefix + id)
+	err := db.DBconn.Delete(v.storeName, id, v.tagMeta)
 	if err != nil {
 		return pkgerrors.Wrap(err, "Delete Resource Bundle Definitions")
 	}
@@ -140,22 +133,22 @@ func (v *DefinitionClient) Delete(id string) error {
 // Upload the contents of resource bundle into database
 func (v *DefinitionClient) Upload(id string, inp []byte) error {
 
-	//ignore the returned data here.
+	//ignore the returned data here
 	_, err := v.Get(id)
 	if err != nil {
-		return pkgerrors.Errorf("Invalid ID provided %s", err.Error())
+		return pkgerrors.Errorf("Invalid ID provided: %s", err.Error())
 	}
 
 	err = isTarGz(bytes.NewBuffer(inp))
 	if err != nil {
-		return pkgerrors.Errorf("Error in file format %s", err.Error())
+		return pkgerrors.Errorf("Error in file format: %s", err.Error())
 	}
 
+	//Encode given byte stream to text for storage
 	encodedStr := base64.StdEncoding.EncodeToString(inp)
-	key := v.keyPrefix + id + "/content"
-	err = db.DBconn.Create(key, encodedStr)
+	err = db.DBconn.Create(v.storeName, id, encodedStr, v.tagContent)
 	if err != nil {
-		return pkgerrors.Errorf("Error uploading data to db %s", err.Error())
+		return pkgerrors.Errorf("Error uploading data to db: %s", err.Error())
 	}
 
 	return nil