diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/orchestrator/pkg/infra/db/README.md | 154 | ||||
| -rw-r--r-- | src/orchestrator/pkg/infra/db/mock.go | 17 | ||||
| -rw-r--r-- | src/orchestrator/pkg/infra/db/mongo.go | 238 | ||||
| -rw-r--r-- | src/orchestrator/pkg/infra/db/mongo_test.go | 151 | ||||
| -rw-r--r-- | src/orchestrator/pkg/infra/db/store.go | 10 |
5 files changed, 310 insertions, 260 deletions
diff --git a/src/orchestrator/pkg/infra/db/README.md b/src/orchestrator/pkg/infra/db/README.md index cba1b7ea..ff482b98 100644 --- a/src/orchestrator/pkg/infra/db/README.md +++ b/src/orchestrator/pkg/infra/db/README.md @@ -11,81 +11,92 @@ type Store interface { // Unmarshal implements any unmarshaling needed for the database Unmarshal(inp []byte, out interface{}) error - // Creates a new master table with key and links data with tag and - // creates a pointer to the newly added data in the master table - Create(table string, key Key, tag string, data interface{}) error + // Inserts and Updates a tag with key and also adds query fields if provided + Insert(coll string, key Key, query interface{}, tag string, data interface{}) error - // Reads data for a particular key with specific tag. - Read(table string, key Key, tag string) ([]byte, error) + // Find the document(s) with key and get the tag values from the document(s) + Find(coll string, key Key, tag string) ([][]byte, error) - // Update data for particular key with specific tag - Update(table string, key Key, tag string, data interface{}) error - - // Deletes a specific tag data for key. - // TODO: If tag is empty, it will delete all tags under key. - Delete(table string, key Key, tag string) error - - // Reads all master tables and data from the specified tag in table - ReadAll(table string, tag string) (map[string][]byte, error) + // Removes the document(s) matching the key + Remove(coll string, key Key) error } ``` -Therefore, `mongo.go`, `consul.go` implement the above interface and can be used as the backend as needed based on initial configuration. +With this interface multiple database types can be supported by providing backends. ## Details on Mongo Implementation `mongo.go` implements the above interface using the `go.mongodb.org/mongo-driver` package. The code converts incoming binary data and creates a new document in the database. -### Create +### Insert Arguments: ```go collection string key interface +query interface tag string data []byte ``` -Create inserts the provided `data` into the `collection` which returns an auto-generated (by `mongodb`) ID which we then associate with the `key` that is provided as one of the arguments. +Insert function inserts the provided `data` into the `collection` as a document in MongoDB. `FindOneAndUpdate` mongo API is used to achieve this with the `upsert` option set to `true`. With this if the record doesn't exist it is created and if it exists it is updated with new data for the tag. -We use the `FindOneAndUpdate` mongo API to achieve this with the `upsert` option set to `true`. -We create the following documents in mongodb for each new definition added to the database: +Key and Query parameters are assumed to be json structures with each element as part of the key. Those key-value pairs are used as the key for the document. +Internally this API takes all the fields in the Key structure and adds them as fields in the document. Query parameter works just like key and it is used to add additional fields to the document. -There is a Master Key document that contains references to other documents which are related to this `key`. +With this key the document can be quried with Mongo `Find` function for both the key fields and Query fields. -#### Master Key Entry -```json -{ - "_id" : ObjectId("5e0a8554b78a15f71d2dce7e"), - "key" : { "rbname" : "edgex", "rbversion" : "v1"}, - "defmetadata" : ObjectId("5e0a8554be261ecb57f067eb"), - "defcontent" : ObjectId("5e0a8377bcfcdd0f01dc7b0d") +This API also adds another field called "Key" field to the document. The "Key" field is concatenation of the key part of the Key parameter. Internally this is used to figure out the type of the document. + +Assumption is that all the elememts of the key structure are strings. + +#### Example of Key Structure +```go +type CompositeAppKey struct { + CompositeAppName string `json:"compositeappname"` + Version string `json:"version"` + Project string `json:"project"` } ``` -#### Metadata Key Entry -```json -{ - "_id" : ObjectId("5e0a8554be261ecb57f067eb"), - "defmetadata" : { "rbname" : "edgex", "rbversion" : "v1", "chartname" : "", "description" : "", "labels" : null } +#### Example of Key Values +```go +key := CompositeAppKey{ + CompositeAppName: "ca1", + Version: "v1", + Project: "testProject", + } +``` + +#### Example of Query Structure +```go +type Query struct { + Userdata1 string `json:"userdata1"` } ``` -#### Definition Content +#### Example of Document store in MongoDB ```json { - "_id" : ObjectId("5e0a8377bcfcdd0f01dc7b0d"), - "defcontent" : "H4sICCVd3FwAA3Byb2ZpbGUxLnRhcgDt1NEKgjAUxvFd7ylG98aWOsGXiYELxLRwJvj2rbyoIPDGiuD/uzmwM9iB7Vvruvrgw7CdXHsUn6Ejm2W3aopcP9eZLYRJM1voPN+ZndAm16kVSn9onheXMLheKeGqfdM0rq07/3bfUv9PJUkiR9+H+tSVajRymM6+lEqN7njxoVSbU+z2deX388r9nWzkr8fGSt5d79pnLOZfm0f+dRrzb7P4DZD/LyDJAAAAAAAAAAAAAAAA/+0Ksq1N5QAoAAA=" + "_id":"ObjectId(" "5e54c206f53ca130893c8020" ")", + "compositeappname":"ca1", + "project":"testProject", + "version":"v1", + "compositeAppmetadata":{ + "metadata":{ + "name":"ca1", + "description":"Test ca", + "userdata1":"Data1", + "userdata2":"Data2" + }, + "spec":{ + "version":"v1" + } + }, + "key":"{compositeappname,project,version,}" } ``` -### Unmarshal - -Data in mongo is stored as `bson` which is a compressed form of `json`. We need mongo to convert the stored `bson` data to regular `json` -that we can use in our code when returned. - -We just use the `bson.Unmarshal` API to achieve this. - -### Read +### Find Arguments: ```go @@ -94,30 +105,45 @@ key interface tag string ``` -Read is straight forward and it uses the `FindOne` API to find our Mongo document based on the provided `key` and then gets the corresponding data for the given `tag`. It will return []byte which can then be passed to the `Unmarshal` function to get the desired GO object. +Find function return one or more tag data based on the Key value. If key has all the fields defined then an exact match is looked for based on the key passed in. +If some of the field value in structure are empty strings then this function returns all the documents which have the same type. (ANY operation) -### Delete +#### Example of Exact Match based on fields Key Values +```go +key := CompositeAppKey{ + CompositeAppName: "ca1", + Version: "v1", + Project: "testProject", + } +``` -Delete is similar to Read and deletes all the objectIDs being stored for a given `key` in the collection. +#### Example of Match based on some fields +This example will return all the compositeApp under project testProject. +```go +key := CompositeAppKey{ + Project: "testProject", + CompositeAppName: "", + Version: "", + + } +``` -## Testing Interfaces +NOTE: Key structure can be different from the original key and can include Query fields also. ANY operation is not supported for Query fields. -The following interface exists to allow for the development of unit tests which don't require mongo to be running. -It is mentioned so in the code as well. +### Remove +Arguments: ```go -// MongoCollection defines the a subset of MongoDB operations -// Note: This interface is defined mainly for mock testing -type MongoCollection interface { - InsertOne(ctx context.Context, document interface{}, - opts ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) - FindOne(ctx context.Context, filter interface{}, - opts ...*options.FindOneOptions) *mongo.SingleResult - FindOneAndUpdate(ctx context.Context, filter interface{}, - update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult - DeleteOne(ctx context.Context, filter interface{}, - opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) - Find(ctx context.Context, filter interface{}, - opts ...*options.FindOptions) (*mongo.Cursor, error) -} -```
\ No newline at end of file +collection string +key interface +``` +Similar to find. This will remove one or more documents based on the key structure. + +### Unmarshal + +Data in mongo is stored as `bson` which is a compressed form of `json`. We need mongo to convert the stored `bson` data to regular `json` +that we can use in our code when returned. + +`bson.Unmarshal` API is used to achieve this. + + diff --git a/src/orchestrator/pkg/infra/db/mock.go b/src/orchestrator/pkg/infra/db/mock.go index 1dbca4b4..afa4b024 100644 --- a/src/orchestrator/pkg/infra/db/mock.go +++ b/src/orchestrator/pkg/infra/db/mock.go @@ -75,20 +75,3 @@ func (m *MockDB) Delete(table string, key Key, tag string) error { return m.Err } -func (m *MockDB) ReadAll(table string, tag string) (map[string][]byte, error) { - if m.Err != nil { - return nil, m.Err - } - - ret := make(map[string][]byte) - - for k, v := range m.Items { - for k1, v1 := range v { - if k1 == tag { - ret[k] = v1 - } - } - } - - return ret, nil -} diff --git a/src/orchestrator/pkg/infra/db/mongo.go b/src/orchestrator/pkg/infra/db/mongo.go index 32d0b549..30eb899f 100644 --- a/src/orchestrator/pkg/infra/db/mongo.go +++ b/src/orchestrator/pkg/infra/db/mongo.go @@ -17,7 +17,9 @@ package db import ( + "encoding/json" "log" + "sort" "golang.org/x/net/context" @@ -41,8 +43,12 @@ type MongoCollection interface { update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult DeleteOne(ctx context.Context, filter interface{}, opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) + DeleteMany(ctx context.Context, filter interface{}, + opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) Find(ctx context.Context, filter interface{}, opts ...*options.FindOptions) (*mongo.Cursor, error) + UpdateOne(ctx context.Context, filter interface{}, update interface{}, + opts ...*options.UpdateOptions) (*mongo.UpdateResult, error) } // MongoStore is an implementation of the db.Store interface @@ -339,58 +345,218 @@ func (m *MongoStore) Delete(coll string, key Key, tag string) error { return nil } -// ReadAll is used to get all documents in db of a particular tag -func (m *MongoStore) ReadAll(coll, tag string) (map[string][]byte, error) { - if !m.validateParams(coll, tag) { - return nil, pkgerrors.New("Missing collection or tag name") +func (m *MongoStore) findFilter(key Key) (primitive.M, error) { + + var bsonMap bson.M + st, err := json.Marshal(key) + if err != nil { + return primitive.M{}, pkgerrors.Errorf("Error Marshalling key: %s", err.Error()) + } + err = json.Unmarshal([]byte(st), &bsonMap) + if err != nil { + return primitive.M{}, pkgerrors.Errorf("Error Unmarshalling key to Bson Map: %s", err.Error()) + } + filter := bson.M{ + "$and": []bson.M{bsonMap}, + } + return filter, nil +} + +func (m *MongoStore) findFilterWithKey(key Key) (primitive.M, error) { + + var bsonMap bson.M + var bsonMapFinal bson.M + st, err := json.Marshal(key) + if err != nil { + return primitive.M{}, pkgerrors.Errorf("Error Marshalling key: %s", err.Error()) + } + err = json.Unmarshal([]byte(st), &bsonMap) + if err != nil { + return primitive.M{}, pkgerrors.Errorf("Error Unmarshalling key to Bson Map: %s", err.Error()) + } + bsonMapFinal = make(bson.M) + for k, v := range bsonMap { + if v == "" { + if _, ok := bsonMapFinal["key"]; !ok { + // add type of key to filter + s, err := m.createKeyField(key) + if err != nil { + return primitive.M{}, err + } + bsonMapFinal["key"] = s + } + } else { + bsonMapFinal[k] = v + } + } + filter := bson.M{ + "$and": []bson.M{bsonMapFinal}, + } + return filter, nil +} + +func (m *MongoStore) updateFilter(key interface{}) (primitive.M, error) { + + var n map[string]string + st, err := json.Marshal(key) + if err != nil { + return primitive.M{}, pkgerrors.Errorf("Error Marshalling key: %s", err.Error()) + } + err = json.Unmarshal([]byte(st), &n) + if err != nil { + return primitive.M{}, pkgerrors.Errorf("Error Unmarshalling key to Bson Map: %s", err.Error()) + } + p := make(bson.M, len(n)) + for k, v := range n { + p[k] = v + } + filter := bson.M{ + "$set": p, + } + return filter, nil +} + +func (m *MongoStore) createKeyField(key interface{}) (string, error) { + + var n map[string]string + st, err := json.Marshal(key) + if err != nil { + return "", pkgerrors.Errorf("Error Marshalling key: %s", err.Error()) + } + err = json.Unmarshal([]byte(st), &n) + if err != nil { + return "", pkgerrors.Errorf("Error Unmarshalling key to Bson Map: %s", err.Error()) + } + var keys []string + for k := range n { + keys = append(keys, k) + } + sort.Strings(keys) + s := "{" + for _, k := range keys { + s = s + k + "," + } + s = s + "}" + return s, nil +} + +// Insert is used to insert/add element to a document +func (m *MongoStore) Insert(coll string, key Key, query interface{}, tag string, data interface{}) error { + if data == nil || !m.validateParams(coll, key, tag) { + return pkgerrors.New("No Data to store") } c := getCollection(coll, m) ctx := context.Background() - //Get all master tables in this collection - filter := bson.D{ - {"key", bson.D{ - {"$exists", true}, - }}, + filter, err := m.findFilter(key) + if err != nil { + return err } - cursor, err := c.Find(ctx, filter) + // Create and add key tag + s, err := m.createKeyField(key) if err != nil { - return nil, pkgerrors.Errorf("Error reading from database: %s", err.Error()) + return err } - defer cursorClose(ctx, cursor) + _, err = decodeBytes( + c.FindOneAndUpdate( + ctx, + filter, + bson.D{ + {"$set", bson.D{ + {tag, data}, + {"key", s}, + }}, + }, + options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After))) - //Iterate over all the master tables - result := make(map[string][]byte) - for cursorNext(ctx, cursor) { - d := cursor.Current + if err != nil { + return pkgerrors.Errorf("Error updating master table: %s", err.Error()) + } + if query == nil { + return nil + } - //Read key of each master table - key, ok := d.Lookup("key").DocumentOK() - if !ok { - //Throw error if key is not found - pkgerrors.New("Unable to read key from mastertable") - } + // Update to add Query fields + update, err := m.updateFilter(query) + if err != nil { + return err + } + _, err = c.UpdateOne( + ctx, + filter, + update) - //Get objectID of tag document - tid, ok := d.Lookup(tag).ObjectIDOK() - if !ok { - log.Printf("Did not find tag: %s", tag) - continue - } + if err != nil { + return pkgerrors.Errorf("Error updating Query fields: %s", err.Error()) + } + return nil +} - //Find tag document and unmarshal it into []byte - tagData, err := decodeBytes(c.FindOne(ctx, bson.D{{"_id", tid}})) - if err != nil { - log.Printf("Unable to decode tag data %s", err.Error()) - continue - } - result[key.String()] = tagData.Lookup(tag).Value +// Find method returns the data stored for this key and for this particular tag +func (m *MongoStore) Find(coll string, key Key, tag string) ([][]byte, error) { + + //result, err := m.findInternal(coll, key, tag, "") + //return result, err + if !m.validateParams(coll, key, tag) { + return nil, pkgerrors.New("Mandatory fields are missing") } + c := getCollection(coll, m) + ctx := context.Background() + + filter, err := m.findFilterWithKey(key) + if err != nil { + return nil, err + } + // Find only the field requested + projection := bson.D{ + {tag, 1}, + {"_id", 0}, + } + + cursor, err := c.Find(context.Background(), filter, options.Find().SetProjection(projection)) + if err != nil { + return nil, pkgerrors.Errorf("Error finding element: %s", err.Error()) + } + defer cursorClose(ctx, cursor) + var data []byte + var result [][]byte + for cursorNext(ctx, cursor) { + d := cursor.Current + switch d.Lookup(tag).Type { + case bson.TypeString: + data = []byte(d.Lookup(tag).StringValue()) + default: + r, err := d.LookupErr(tag) + if err != nil { + // Throw error if not found + pkgerrors.New("Unable to read data ") + } + data = r.Value + } + result = append(result, data) + } if len(result) == 0 { return result, pkgerrors.Errorf("Did not find any objects with tag: %s", tag) } - return result, nil } + +// Remove method to remove the documet by key +func (m *MongoStore) Remove(coll string, key Key) error { + if !m.validateParams(coll, key) { + return pkgerrors.New("Mandatory fields are missing") + } + c := getCollection(coll, m) + ctx := context.Background() + filter, err := m.findFilterWithKey(key) + if err != nil { + return err + } + _, err = c.DeleteMany(ctx, filter) + if err != nil { + return pkgerrors.Errorf("Error Deleting from database: %s", err.Error()) + } + return nil +} diff --git a/src/orchestrator/pkg/infra/db/mongo_test.go b/src/orchestrator/pkg/infra/db/mongo_test.go index 171c908f..d506dbda 100644 --- a/src/orchestrator/pkg/infra/db/mongo_test.go +++ b/src/orchestrator/pkg/infra/db/mongo_test.go @@ -19,7 +19,6 @@ package db import ( "bytes" "context" - "reflect" "strings" "testing" @@ -70,6 +69,16 @@ func (c *mockCollection) Find(ctx context.Context, filter interface{}, return c.mCursor, c.Err } +func (c *mockCollection) DeleteMany(ctx context.Context, filter interface{}, + opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) { + return nil, c.Err +} + +func (c *mockCollection) UpdateOne(ctx context.Context, filter interface{}, update interface{}, + opts ...*options.UpdateOptions) (*mongo.UpdateResult, error) { + return nil, c.Err +} + func TestCreate(t *testing.T) { testCases := []struct { label string @@ -455,143 +464,3 @@ func TestDelete(t *testing.T) { } } -func TestReadAll(t *testing.T) { - testCases := []struct { - label string - input map[string]interface{} - mockColl *mockCollection - bson bson.Raw - expectedError string - expected map[string][]byte - }{ - { - label: "Successfully Read all entries", - input: map[string]interface{}{ - "coll": "collname", - "tag": "metadata", - }, - mockColl: &mockCollection{ - mCursor: &mongo.Cursor{ - // Binary form of - // { - // "_id" : ObjectId("5c115156777ff85654248ae1"), - // "key" : bson.D{{"name","testdef"},{"version","v1"}}, - // "metadata" : ObjectId("5c115156c9755047e318bbfd") - // } - - Current: bson.Raw{ - '\x58', '\x00', '\x00', '\x00', '\x03', '\x6b', '\x65', '\x79', - '\x00', '\x27', '\x00', '\x00', '\x00', '\x02', '\x6e', '\x61', - '\x6d', '\x65', '\x00', '\x08', '\x00', '\x00', '\x00', '\x74', - '\x65', '\x73', '\x74', '\x64', '\x65', '\x66', '\x00', '\x02', - '\x76', '\x65', '\x72', '\x73', '\x69', '\x6f', '\x6e', '\x00', - '\x03', '\x00', '\x00', '\x00', '\x76', '\x31', '\x00', '\x00', - '\x07', '\x6d', '\x65', '\x74', '\x61', '\x64', '\x61', '\x74', - '\x61', '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', - '\xf8', '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x07', '\x5f', - '\x69', '\x64', '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', - '\x7f', '\xf8', '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x00', - }, - }, - mCursorCount: 1, - }, - expected: map[string][]byte{ - `{"name": "testdef","version": "v1"}`: []byte{ - 92, 17, 81, 86, 119, 127, 248, 86, 84, 36, 138, 225}, - }, - }, - { - label: "UnSuccessfully Read of all entries", - input: map[string]interface{}{ - "coll": "collname", - "tag": "tagName", - }, - mockColl: &mockCollection{ - Err: pkgerrors.New("DB Error"), - }, - expectedError: "DB Error", - }, - { - label: "UnSuccessfull Readall, tag not found", - input: map[string]interface{}{ - "coll": "collname", - "tag": "tagName", - }, - mockColl: &mockCollection{ - mCursor: &mongo.Cursor{ - // Binary form of - // { - // "_id" : ObjectId("5c115156777ff85654248ae1"), - // "key" : bson.D{{"name","testdef"},{"version","v1"}}, - // "metadata" : ObjectId("5c115156c9755047e318bbfd") - // } - Current: bson.Raw{ - '\x58', '\x00', '\x00', '\x00', '\x03', '\x6b', '\x65', '\x79', - '\x00', '\x27', '\x00', '\x00', '\x00', '\x02', '\x6e', '\x61', - '\x6d', '\x65', '\x00', '\x08', '\x00', '\x00', '\x00', '\x74', - '\x65', '\x73', '\x74', '\x64', '\x65', '\x66', '\x00', '\x02', - '\x76', '\x65', '\x72', '\x73', '\x69', '\x6f', '\x6e', '\x00', - '\x03', '\x00', '\x00', '\x00', '\x76', '\x31', '\x00', '\x00', - '\x07', '\x6d', '\x65', '\x74', '\x61', '\x64', '\x61', '\x74', - '\x61', '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', '\x7f', - '\xf8', '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x07', '\x5f', - '\x69', '\x64', '\x00', '\x5c', '\x11', '\x51', '\x56', '\x77', - '\x7f', '\xf8', '\x56', '\x54', '\x24', '\x8a', '\xe1', '\x00', - }, - }, - mCursorCount: 1, - }, - expectedError: "Did not find any objects with tag", - }, - { - label: "Missing input fields", - input: map[string]interface{}{ - "coll": "", - "tag": "", - }, - expectedError: "Missing collection or tag name", - mockColl: &mockCollection{}, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.label, func(t *testing.T) { - m, _ := NewMongoStore("name", &mongo.Database{}) - // Override the getCollection function with our mocked version - getCollection = func(coll string, m *MongoStore) MongoCollection { - return testCase.mockColl - } - - decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { - return testCase.mockColl.mCursor.Current, testCase.mockColl.Err - } - - cursorNext = func(ctx context.Context, cursor *mongo.Cursor) bool { - if testCase.mockColl.mCursorCount > 0 { - testCase.mockColl.mCursorCount -= 1 - return true - } - return false - } - - cursorClose = func(ctx context.Context, cursor *mongo.Cursor) error { - return nil - } - - got, err := m.ReadAll(testCase.input["coll"].(string), testCase.input["tag"].(string)) - if err != nil { - if testCase.expectedError == "" { - t.Fatalf("Readall method returned an un-expected (%s)", err) - } - if !strings.Contains(string(err.Error()), testCase.expectedError) { - t.Fatalf("Readall method returned an error (%s)", err) - } - } else { - if reflect.DeepEqual(got, testCase.expected) == false { - t.Fatalf("Readall returned unexpected data: %v, expected: %v", - got, testCase.expected) - } - } - }) - } -} diff --git a/src/orchestrator/pkg/infra/db/store.go b/src/orchestrator/pkg/infra/db/store.go index 1a9632e7..c8a8f744 100644 --- a/src/orchestrator/pkg/infra/db/store.go +++ b/src/orchestrator/pkg/infra/db/store.go @@ -54,8 +54,14 @@ type Store interface { // TODO: If tag is empty, it will delete all tags under key. Delete(table string, key Key, tag string) error - // Reads all master tables and data from the specified tag in table - ReadAll(table string, tag string) (map[string][]byte, error) + // Inserts and Updates a tag with key and also adds query fields if provided + Insert(coll string, key Key, query interface{}, tag string, data interface{}) error + + // Find the document(s) with key and get the tag values from the document(s) + Find(coll string, key Key, tag string) ([][]byte, error) + + // Removes the document(s) matching the key + Remove(coll string, key Key) error } // CreateDBClient creates the DB client |