From f54fee24c32465478e2c6fc4cd6fc6bfb44fe096 Mon Sep 17 00:00:00 2001 From: Kiran Kamineni Date: Tue, 20 Nov 2018 14:32:50 -0800 Subject: Add upload backend implementation Upload is a seperate API where it takes a binary stream and stores it. The api supports tar.gz file format only. P2: Check if ID is valid before trying upload Add test with an invalid ID Issue-ID: MULTICLOUD-393 Change-Id: Id636a95823a046e1795d3be72d0214e953a8c5fc Signed-off-by: Kiran Kamineni --- src/k8splugin/api/api.go | 2 +- src/k8splugin/api/defhandler.go | 25 +++++- src/k8splugin/api/defhandler_test.go | 65 +++++++++++++++ src/k8splugin/rb/archive.go | 65 +++++++++++++++ src/k8splugin/rb/archive_test.go | 66 ++++++++++++++++ src/k8splugin/rb/definition.go | 29 ++++++- src/k8splugin/rb/definition_test.go | 149 ++++++++++++++++++++++++++++++++++- 7 files changed, 396 insertions(+), 5 deletions(-) create mode 100644 src/k8splugin/rb/archive.go create mode 100644 src/k8splugin/rb/archive_test.go (limited to 'src') diff --git a/src/k8splugin/api/api.go b/src/k8splugin/api/api.go index 571a9576..530537dc 100644 --- a/src/k8splugin/api/api.go +++ b/src/k8splugin/api/api.go @@ -109,7 +109,7 @@ func NewRouter(kubeconfig string) *mux.Router { resRouter := router.PathPrefix("/v1/rb").Subrouter() rbdef := rbDefinitionHandler{client: rb.NewDefinitionClient()} resRouter.HandleFunc("/definition", rbdef.createHandler).Methods("POST") - resRouter.HandleFunc("/definition/{rbdID}/upload", rbdef.uploadHandler).Methods("POST") + resRouter.HandleFunc("/definition/{rbdID}/content", rbdef.uploadHandler).Methods("POST") resRouter.HandleFunc("/definition", rbdef.listHandler).Methods("GET") resRouter.HandleFunc("/definition/{rbdID}", rbdef.getHandler).Methods("GET") resRouter.HandleFunc("/definition/{rbdID}", rbdef.deleteHandler).Methods("DELETE") diff --git a/src/k8splugin/api/defhandler.go b/src/k8splugin/api/defhandler.go index c8c03496..222baaee 100644 --- a/src/k8splugin/api/defhandler.go +++ b/src/k8splugin/api/defhandler.go @@ -18,9 +18,9 @@ package api import ( "encoding/json" - "net/http" - + "io/ioutil" "k8splugin/rb" + "net/http" "github.com/gorilla/mux" ) @@ -72,6 +72,27 @@ func (h rbDefinitionHandler) createHandler(w http.ResponseWriter, r *http.Reques // uploadHandler handles upload of the bundle tar file into the database // Note: This will be implemented in a different patch func (h rbDefinitionHandler) uploadHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + uuid := vars["rbdID"] + + if r.Body == nil { + http.Error(w, "Empty Body", http.StatusBadRequest) + return + } + + inpBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Unable to read body", http.StatusBadRequest) + return + } + + err = h.client.Upload(uuid, inpBytes) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) } // listHandler handles GET (list) operations on the endpoint diff --git a/src/k8splugin/api/defhandler_test.go b/src/k8splugin/api/defhandler_test.go index b83f0b7a..9739ab12 100644 --- a/src/k8splugin/api/defhandler_test.go +++ b/src/k8splugin/api/defhandler_test.go @@ -68,6 +68,10 @@ func (m *mockRBDefinition) Delete(id string) error { return m.Err } +func (m *mockRBDefinition) Upload(id string, inp []byte) error { + return m.Err +} + func TestRBDefCreateHandler(t *testing.T) { testCases := []struct { label string @@ -334,3 +338,64 @@ func TestRBDefDeleteHandler(t *testing.T) { }) } } + +func TestRBDefUploadHandler(t *testing.T) { + + testCases := []struct { + label string + inpUUID string + body io.Reader + expectedCode int + rbDefClient *mockRBDefinition + }{ + { + label: "Upload Bundle Definition Content", + expectedCode: http.StatusOK, + inpUUID: "123e4567-e89b-12d3-a456-426655441111", + body: bytes.NewBuffer([]byte{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xf2, 0x48, 0xcd, + }), + rbDefClient: &mockRBDefinition{}, + }, + { + label: "Upload Invalid Bundle Definition Content", + expectedCode: http.StatusInternalServerError, + inpUUID: "123e4567-e89b-12d3-a456-426655440000", + body: bytes.NewBuffer([]byte{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xf2, 0x48, 0xcd, + }), + rbDefClient: &mockRBDefinition{ + Err: pkgerrors.New("Internal Error"), + }, + }, + { + label: "Upload Empty Body Content", + expectedCode: http.StatusBadRequest, + inpUUID: "123e4567-e89b-12d3-a456-426655440000", + rbDefClient: &mockRBDefinition{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + vh := rbDefinitionHandler{client: testCase.rbDefClient} + req, err := http.NewRequest("POST", + "/v1/resource/definition/"+testCase.inpUUID+"/content", testCase.body) + + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + hr := http.HandlerFunc(vh.uploadHandler) + + hr.ServeHTTP(rr, req) + //Check returned code + if rr.Code != testCase.expectedCode { + t.Fatalf("Expected %d; Got: %d", testCase.expectedCode, rr.Code) + } + }) + } +} diff --git a/src/k8splugin/rb/archive.go b/src/k8splugin/rb/archive.go new file mode 100644 index 00000000..8eb0fbed --- /dev/null +++ b/src/k8splugin/rb/archive.go @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rb + +import ( + "archive/tar" + "compress/gzip" + pkgerrors "github.com/pkg/errors" + "io" +) + +func isTarGz(r io.Reader) error { + //Check if it is a valid gz + gzf, err := gzip.NewReader(r) + if err != nil { + return pkgerrors.Errorf("Invalid gz format %s", err.Error()) + } + + //Check if it is a valid tar file + //Unfortunately this can only be done by inspecting all the tar contents + tarR := tar.NewReader(gzf) + first := true + + for true { + header, err := tarR.Next() + + if err == io.EOF { + //Check if we have just a gzip file without a tar archive inside + if first { + return pkgerrors.New("Empty or non-existant Tar file found") + } + //End of archive + break + } + + if err != nil { + return pkgerrors.Errorf("Error reading tar file %s", err.Error()) + } + + //Check if files are of type directory and regular file + if header.Typeflag != tar.TypeDir && + header.Typeflag != tar.TypeReg { + return pkgerrors.Errorf("Unknown header in tar %s, %s", + header.Name, string(header.Typeflag)) + } + + first = false + } + + return nil +} diff --git a/src/k8splugin/rb/archive_test.go b/src/k8splugin/rb/archive_test.go new file mode 100644 index 00000000..a327dfd4 --- /dev/null +++ b/src/k8splugin/rb/archive_test.go @@ -0,0 +1,66 @@ +/* + * Copyright 2018 Intel Corporation, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rb + +import ( + "bytes" + "testing" +) + +func TestIsTarGz(t *testing.T) { + + t.Run("Valid tar.gz", func(t *testing.T) { + content := []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b, + 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2, + 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e, + 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01, + 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48, + 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff, + 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c, + 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4, + 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d, + 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f, + 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f, + 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92, + 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6, + 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3, + 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8, + 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, + } + + err := isTarGz(bytes.NewBuffer(content)) + if err != nil { + t.Errorf("Error reading valid Zip file %s", err.Error()) + } + }) + + t.Run("Invalid tar.gz", func(t *testing.T) { + content := []byte{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xf2, 0x48, 0xcd, + } + + err := isTarGz(bytes.NewBuffer(content)) + if err == nil { + t.Errorf("Error should NOT be nil") + } + }) +} diff --git a/src/k8splugin/rb/definition.go b/src/k8splugin/rb/definition.go index 03fffdda..2d1c2cd0 100644 --- a/src/k8splugin/rb/definition.go +++ b/src/k8splugin/rb/definition.go @@ -17,6 +17,8 @@ package rb import ( + "bytes" + "encoding/base64" "k8splugin/db" "log" @@ -39,6 +41,7 @@ type DefinitionManager interface { List() ([]Definition, error) Get(resID string) (Definition, error) Delete(resID string) error + Upload(resID string, inp []byte) error } // DefinitionClient implements the DefinitionManager @@ -63,7 +66,7 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) { } key := v.keyPrefix + def.UUID - serData, err := db.Serialize(v) + serData, err := db.Serialize(def) if err != nil { return Definition{}, pkgerrors.Wrap(err, "Serialize Resource Bundle Definition") } @@ -133,3 +136,27 @@ func (v *DefinitionClient) Delete(id string) error { return nil } + +// Upload the contents of resource bundle into database +func (v *DefinitionClient) Upload(id string, inp []byte) error { + + //ignore the returned data here. + _, err := v.Get(id) + if err != nil { + return pkgerrors.Errorf("Invalid ID provided %s", err.Error()) + } + + err = isTarGz(bytes.NewBuffer(inp)) + if err != nil { + return pkgerrors.Errorf("Error in file format %s", err.Error()) + } + + encodedStr := base64.StdEncoding.EncodeToString(inp) + key := v.keyPrefix + id + "/content" + err = db.DBconn.Create(key, encodedStr) + if err != nil { + return pkgerrors.Errorf("Error uploading data to db %s", err.Error()) + } + + return nil +} diff --git a/src/k8splugin/rb/definition_test.go b/src/k8splugin/rb/definition_test.go index a3993c8c..58eb718d 100644 --- a/src/k8splugin/rb/definition_test.go +++ b/src/k8splugin/rb/definition_test.go @@ -228,7 +228,6 @@ func TestDelete(t *testing.T) { inp string expectedError string mockdb *db.MockDB - expected []Definition }{ { label: "Delete Resource Bundle Definition", @@ -260,3 +259,151 @@ func TestDelete(t *testing.T) { }) } } + +func TestUpload(t *testing.T) { + testCases := []struct { + label string + inp string + content []byte + expectedError string + mockdb *db.MockDB + }{ + { + label: "Upload Resource Bundle Definition", + inp: "123e4567-e89b-12d3-a456-426655440000", + content: []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b, + 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2, + 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e, + 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01, + 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48, + 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff, + 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c, + 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4, + 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d, + 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f, + 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f, + 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92, + 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6, + 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3, + 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8, + 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, + }, + mockdb: &db.MockDB{ + Items: api.KVPairs{ + &api.KVPair{ + Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", + Value: []byte("{\"name\":\"testresourcebundle\"," + + "\"description\":\"testresourcebundle\"," + + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + + "\"service-type\":\"firewall\"}"), + }, + }, + }, + }, + { + label: "Upload with an Invalid Resource Bundle Definition", + inp: "123e4567-e89b-12d3-a456-426655440000", + expectedError: "Invalid ID provided", + content: []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b, + 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2, + 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e, + 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01, + 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48, + 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff, + 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c, + 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4, + 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d, + 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f, + 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f, + 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92, + 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6, + 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3, + 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8, + 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, + }, + mockdb: &db.MockDB{ + Items: api.KVPairs{ + &api.KVPair{ + Key: "rb/def/123e4567-e89b-12d3-a456-426655441111", + Value: []byte("{\"name\":\"testresourcebundle\"," + + "\"description\":\"testresourcebundle\"," + + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + + "\"service-type\":\"firewall\"}"), + }, + }, + }, + }, + { + label: "Invalid File Format Error", + inp: "123e4567-e89b-12d3-a456-426655440000", + expectedError: "Error in file format", + content: []byte{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xf2, 0x48, 0xcd, + }, + mockdb: &db.MockDB{ + Items: api.KVPairs{ + &api.KVPair{ + Key: "rb/def/123e4567-e89b-12d3-a456-426655440000", + Value: []byte("{\"name\":\"testresourcebundle\"," + + "\"description\":\"testresourcebundle\"," + + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + + "\"service-type\":\"firewall\"}"), + }, + }, + }, + }, + { + label: "Upload Error", + expectedError: "DB Error", + content: []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b, + 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2, + 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e, + 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01, + 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48, + 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff, + 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c, + 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4, + 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d, + 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f, + 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f, + 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92, + 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6, + 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3, + 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8, + 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, + }, + mockdb: &db.MockDB{ + Err: pkgerrors.New("DB Error"), + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + db.DBconn = testCase.mockdb + impl := NewDefinitionClient() + err := impl.Upload(testCase.inp, testCase.content) + if err != nil { + if testCase.expectedError == "" { + t.Errorf("Upload returned an unexpected error %s", err) + } + if strings.Contains(err.Error(), testCase.expectedError) == false { + t.Errorf("Upload returned an unexpected error %s", err) + } + } + }) + } +} -- cgit 1.2.3-korg