From ecd18ef2315095601454332dd44a8dbc3329c394 Mon Sep 17 00:00:00 2001 From: Kiran Kamineni Date: Fri, 18 Jan 2019 12:09:56 -0800 Subject: Add support for downloading content Add support for downloading content for creating the merged helm charts. This api will be used mainly by the profile api to create a converged chart which will then be created by the instantiation code P2: Add unit tests for archive.go Add download method for profile.go Update comments for download method P3: Add unit test for empty files P4: Add unit tests for Download functions P5: Rebase against new folder structure Issue-ID: MULTICLOUD-291 Change-Id: I9779eaf95366f527f0360eaddea663722c13b196 Signed-off-by: Kiran Kamineni --- src/k8splugin/internal/rb/archive.go | 72 +++++++++++++++++++- src/k8splugin/internal/rb/archive_test.go | 72 +++++++++++++++++++- src/k8splugin/internal/rb/definition.go | 31 +++++++++ src/k8splugin/internal/rb/definition_test.go | 97 +++++++++++++++++++++++++++ src/k8splugin/internal/rb/profile.go | 31 +++++++++ src/k8splugin/internal/rb/profile_test.go | 99 ++++++++++++++++++++++++++++ 6 files changed, 400 insertions(+), 2 deletions(-) diff --git a/src/k8splugin/internal/rb/archive.go b/src/k8splugin/internal/rb/archive.go index 8eb0fbed..624adfba 100644 --- a/src/k8splugin/internal/rb/archive.go +++ b/src/k8splugin/internal/rb/archive.go @@ -21,13 +21,16 @@ import ( "compress/gzip" pkgerrors "github.com/pkg/errors" "io" + "io/ioutil" + "os" + "path/filepath" ) func isTarGz(r io.Reader) error { //Check if it is a valid gz gzf, err := gzip.NewReader(r) if err != nil { - return pkgerrors.Errorf("Invalid gz format %s", err.Error()) + return pkgerrors.Wrap(err, "Invalid gzip format") } //Check if it is a valid tar file @@ -63,3 +66,70 @@ func isTarGz(r io.Reader) error { return nil } + +//ExtractTarBall provides functionality to extract a tar.gz file +//into a temporary location for later use. +//It returns the path to the new location +func ExtractTarBall(r io.Reader) (string, error) { + //Check if it is a valid gz + gzf, err := gzip.NewReader(r) + if err != nil { + return "", pkgerrors.Wrap(err, "Invalid gzip format") + } + + //Check if it is a valid tar file + //Unfortunately this can only be done by inspecting all the tar contents + tarR := tar.NewReader(gzf) + first := true + + outDir, _ := ioutil.TempDir("", "k8s-ext-") + + for true { + header, err := tarR.Next() + + if err == io.EOF { + //Check if we have just a gzip file without a tar archive inside + if first { + return "", pkgerrors.New("Empty or non-existant Tar file found") + } + //End of archive + break + } + + if err != nil { + return "", pkgerrors.Wrap(err, "Error reading tar file") + } + + target := filepath.Join(outDir, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + if _, err := os.Stat(target); err != nil { + // Using 755 read, write, execute for owner + // groups and others get read and execute permissions + // on the folder. + if err := os.MkdirAll(target, 0755); err != nil { + return "", pkgerrors.Wrap(err, "Creating directory") + } + } + case tar.TypeReg: + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return "", pkgerrors.Wrap(err, "Creating file") + } + + // copy over contents + if _, err := io.Copy(f, tarR); err != nil { + return "", pkgerrors.Wrap(err, "Copying file content") + } + + // close for each file instead of a defer for all + // at the end of the function + f.Close() + } + + first = false + } + + return outDir, nil +} diff --git a/src/k8splugin/internal/rb/archive_test.go b/src/k8splugin/internal/rb/archive_test.go index a327dfd4..5fa66e79 100644 --- a/src/k8splugin/internal/rb/archive_test.go +++ b/src/k8splugin/internal/rb/archive_test.go @@ -18,6 +18,9 @@ package rb import ( "bytes" + "io/ioutil" + "path/filepath" + "strings" "testing" ) @@ -48,7 +51,7 @@ func TestIsTarGz(t *testing.T) { err := isTarGz(bytes.NewBuffer(content)) if err != nil { - t.Errorf("Error reading valid Zip file %s", err.Error()) + t.Errorf("Error reading valid tar.gz file %s", err.Error()) } }) @@ -63,4 +66,71 @@ func TestIsTarGz(t *testing.T) { t.Errorf("Error should NOT be nil") } }) + + t.Run("Empty tar.gz", func(t *testing.T) { + content := []byte{} + err := isTarGz(bytes.NewBuffer(content)) + if err == nil { + t.Errorf("Error should NOT be nil") + } + }) +} + +func TestExtractTarBall(t *testing.T) { + + t.Run("Valid tar.gz", func(t *testing.T) { + content := []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b, + 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2, + 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e, + 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01, + 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48, + 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff, + 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c, + 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4, + 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d, + 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f, + 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f, + 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92, + 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6, + 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3, + 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8, + 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, + } + + path, err := ExtractTarBall(bytes.NewBuffer(content)) + if err != nil { + t.Errorf("Error reading valid tar.gz file %s", err.Error()) + } + fcontent, err := ioutil.ReadFile(filepath.Join(path, "test.txt")) + if err != nil { + t.Errorf("Error reading content of valid tar.gz file %s", err.Error()) + } + if strings.Contains(string(fcontent), "helloworld") == false { + t.Errorf("Incorrect content read from path: %s", path) + } + }) + + t.Run("Invalid tar.gz", func(t *testing.T) { + content := []byte{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xf2, 0x48, 0xcd, + } + + _, err := ExtractTarBall(bytes.NewBuffer(content)) + if err == nil { + t.Errorf("Error should NOT be nil") + } + }) + + t.Run("Empty tar.gz", func(t *testing.T) { + content := []byte{} + err := isTarGz(bytes.NewBuffer(content)) + if err == nil { + t.Errorf("Error should NOT be nil") + } + }) } diff --git a/src/k8splugin/internal/rb/definition.go b/src/k8splugin/internal/rb/definition.go index 8a26332b..19844990 100644 --- a/src/k8splugin/internal/rb/definition.go +++ b/src/k8splugin/internal/rb/definition.go @@ -162,3 +162,34 @@ func (v *DefinitionClient) Upload(id string, inp []byte) error { return nil } + +// Download the contents of the resource bundle definition from DB +// Returns a byte array of the contents which is used by the +// ExtractTarBall code to create the folder structure on disk +func (v *DefinitionClient) Download(id string) ([]byte, error) { + + //ignore the returned data here + //Check if id is valid + _, err := v.Get(id) + if err != nil { + return nil, pkgerrors.Errorf("Invalid Definition ID provided: %s", err.Error()) + } + + value, err := db.DBconn.Read(v.storeName, id, v.tagContent) + if err != nil { + return nil, pkgerrors.Wrap(err, "Get Resource Bundle definition content") + } + + if value != nil { + //Decode the string from base64 + out, err := base64.StdEncoding.DecodeString(string(value)) + if err != nil { + return nil, pkgerrors.Wrap(err, "Decode base64 string") + } + + if out != nil && len(out) != 0 { + return out, nil + } + } + return nil, pkgerrors.New("Error downloading Definition content") +} diff --git a/src/k8splugin/internal/rb/definition_test.go b/src/k8splugin/internal/rb/definition_test.go index 46ab3c07..f720b6a9 100644 --- a/src/k8splugin/internal/rb/definition_test.go +++ b/src/k8splugin/internal/rb/definition_test.go @@ -19,6 +19,7 @@ package rb import ( + "bytes" "k8splugin/internal/db" "reflect" "sort" @@ -418,3 +419,99 @@ func TestUploadDefinition(t *testing.T) { }) } } + +func TestDownloadDefinition(t *testing.T) { + testCases := []struct { + label string + inp string + expected []byte + expectedError string + mockdb *db.MockDB + }{ + { + label: "Download Resource Bundle Definition", + inp: "123e4567-e89b-12d3-a456-426655440000", + expected: []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b, + 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2, + 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e, + 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01, + 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48, + 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff, + 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c, + 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4, + 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d, + 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f, + 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f, + 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92, + 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6, + 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3, + 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8, + 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, + }, + mockdb: &db.MockDB{ + Items: map[string]map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": { + "metadata": []byte( + "{\"name\":\"testresourcebundle\"," + + "\"description\":\"testresourcebundle\"," + + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + + "\"service-type\":\"firewall\"}"), + "content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" + + "QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" + + "b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" + + "R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"), + }, + }, + }, + }, + { + label: "Download with an Invalid Resource Bundle Definition", + inp: "123e4567-e89b-12d3-a456-426655440000", + expectedError: "Invalid Definition ID provided", + mockdb: &db.MockDB{ + Items: map[string]map[string][]byte{ + "123e4567-e89b-12d3-a456-426655441111": { + "metadata": []byte( + "{\"name\":\"testresourcebundle\"," + + "\"description\":\"testresourcebundle\"," + + "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," + + "\"service-type\":\"firewall\"}"), + }, + }, + }, + }, + { + label: "Download Error", + expectedError: "DB Error", + inp: "123e4567-e89b-12d3-a456-426655440000", + mockdb: &db.MockDB{ + Err: pkgerrors.New("DB Error"), + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + db.DBconn = testCase.mockdb + impl := NewDefinitionClient() + data, err := impl.Download(testCase.inp) + if err != nil { + if testCase.expectedError == "" { + t.Errorf("Download returned an unexpected error %s", err) + } + if strings.Contains(err.Error(), testCase.expectedError) == false { + t.Errorf("Download returned an unexpected error %s", err) + } + } else { + if bytes.Equal(testCase.expected, data) == false { + t.Errorf("Download returned unexpected data: got %v - expected %v", + data, testCase.expected) + } + } + }) + } +} diff --git a/src/k8splugin/internal/rb/profile.go b/src/k8splugin/internal/rb/profile.go index a0245af1..2456ad2d 100644 --- a/src/k8splugin/internal/rb/profile.go +++ b/src/k8splugin/internal/rb/profile.go @@ -183,3 +183,34 @@ func (v *ProfileClient) Upload(id string, inp []byte) error { return nil } + +// Download the contents of the resource bundle profile from DB +// Returns a byte array of the contents which is used by the +// ExtractTarBall code to create the folder structure on disk +func (v *ProfileClient) Download(id string) ([]byte, error) { + + //ignore the returned data here + //Check if id is valid + _, err := v.Get(id) + if err != nil { + return nil, pkgerrors.Errorf("Invalid Profile ID provided: %s", err.Error()) + } + + value, err := db.DBconn.Read(v.storeName, id, v.tagContent) + if err != nil { + return nil, pkgerrors.Wrap(err, "Get Resource Bundle Profile content") + } + + if value != nil { + //Decode the string from base64 + out, err := base64.StdEncoding.DecodeString(string(value)) + if err != nil { + return nil, pkgerrors.Wrap(err, "Decode base64 string") + } + + if out != nil && len(out) != 0 { + return out, nil + } + } + return nil, pkgerrors.New("Error downloading Profile content") +} diff --git a/src/k8splugin/internal/rb/profile_test.go b/src/k8splugin/internal/rb/profile_test.go index 15ff8951..d97969de 100644 --- a/src/k8splugin/internal/rb/profile_test.go +++ b/src/k8splugin/internal/rb/profile_test.go @@ -19,6 +19,7 @@ package rb import ( + "bytes" "k8splugin/internal/db" "reflect" "sort" @@ -424,3 +425,101 @@ func TestUploadProfile(t *testing.T) { }) } } + +func TestDownloadProfile(t *testing.T) { + testCases := []struct { + label string + inp string + expected []byte + expectedError string + mockdb *db.MockDB + }{ + { + label: "Download Resource Bundle Profile", + inp: "123e4567-e89b-12d3-a456-426655440000", + expected: []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b, + 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2, + 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e, + 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01, + 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48, + 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff, + 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c, + 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4, + 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d, + 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f, + 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f, + 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92, + 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6, + 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3, + 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8, + 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00, + }, + mockdb: &db.MockDB{ + Items: map[string]map[string][]byte{ + "123e4567-e89b-12d3-a456-426655440000": { + "metadata": []byte( + "{\"name\":\"testresourcebundle\"," + + "\"namespace\":\"default\"," + + "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," + + "\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," + + "\"kubernetesversion\":\"1.12.3\"}"), + "content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" + + "QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" + + "b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" + + "R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"), + }, + }, + }, + }, + { + label: "Download with an Invalid Resource Bundle Profile", + inp: "123e4567-e89b-12d3-a456-426655440000", + expectedError: "Invalid Profile ID provided", + mockdb: &db.MockDB{ + Items: map[string]map[string][]byte{ + "123e4567-e89b-12d3-a456-426655441111": { + "metadata": []byte( + "{\"name\":\"testresourcebundle\"," + + "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," + + "\"namespace\":\"default\"," + + "\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," + + "\"kubernetesversion\":\"1.12.3\"}"), + }, + }, + }, + }, + { + label: "Download Error", + expectedError: "DB Error", + inp: "123e4567-e89b-12d3-a456-426655440000", + mockdb: &db.MockDB{ + Err: pkgerrors.New("DB Error"), + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.label, func(t *testing.T) { + db.DBconn = testCase.mockdb + impl := NewProfileClient() + data, err := impl.Download(testCase.inp) + if err != nil { + if testCase.expectedError == "" { + t.Errorf("Download returned an unexpected error %s", err) + } + if strings.Contains(err.Error(), testCase.expectedError) == false { + t.Errorf("Download returned an unexpected error %s", err) + } + } else { + if bytes.Equal(testCase.expected, data) == false { + t.Errorf("Download returned unexpected data: got %v - expected %v", + data, testCase.expected) + } + } + }) + } +} -- cgit 1.2.3-korg