aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKiran Kamineni <kiran.k.kamineni@intel.com>2019-01-18 12:09:56 -0800
committerKiran Kamineni <kiran.k.kamineni@intel.com>2019-01-28 15:22:58 -0800
commitecd18ef2315095601454332dd44a8dbc3329c394 (patch)
treeca6d97b8f0d47965425464b5fa8a1ad5e5f68bb0
parentf2eca1cca86160a2087b6bcbb8581898f8e77b1e (diff)
Add support for downloading content
Add support for downloading content for creating the merged helm charts. This api will be used mainly by the profile api to create a converged chart which will then be created by the instantiation code P2: Add unit tests for archive.go Add download method for profile.go Update comments for download method P3: Add unit test for empty files P4: Add unit tests for Download functions P5: Rebase against new folder structure Issue-ID: MULTICLOUD-291 Change-Id: I9779eaf95366f527f0360eaddea663722c13b196 Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>
-rw-r--r--src/k8splugin/internal/rb/archive.go72
-rw-r--r--src/k8splugin/internal/rb/archive_test.go72
-rw-r--r--src/k8splugin/internal/rb/definition.go31
-rw-r--r--src/k8splugin/internal/rb/definition_test.go97
-rw-r--r--src/k8splugin/internal/rb/profile.go31
-rw-r--r--src/k8splugin/internal/rb/profile_test.go99
6 files changed, 400 insertions, 2 deletions
diff --git a/src/k8splugin/internal/rb/archive.go b/src/k8splugin/internal/rb/archive.go
index 8eb0fbed..624adfba 100644
--- a/src/k8splugin/internal/rb/archive.go
+++ b/src/k8splugin/internal/rb/archive.go
@@ -21,13 +21,16 @@ import (
"compress/gzip"
pkgerrors "github.com/pkg/errors"
"io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
)
func isTarGz(r io.Reader) error {
//Check if it is a valid gz
gzf, err := gzip.NewReader(r)
if err != nil {
- return pkgerrors.Errorf("Invalid gz format %s", err.Error())
+ return pkgerrors.Wrap(err, "Invalid gzip format")
}
//Check if it is a valid tar file
@@ -63,3 +66,70 @@ func isTarGz(r io.Reader) error {
return nil
}
+
+//ExtractTarBall provides functionality to extract a tar.gz file
+//into a temporary location for later use.
+//It returns the path to the new location
+func ExtractTarBall(r io.Reader) (string, error) {
+ //Check if it is a valid gz
+ gzf, err := gzip.NewReader(r)
+ if err != nil {
+ return "", pkgerrors.Wrap(err, "Invalid gzip format")
+ }
+
+ //Check if it is a valid tar file
+ //Unfortunately this can only be done by inspecting all the tar contents
+ tarR := tar.NewReader(gzf)
+ first := true
+
+ outDir, _ := ioutil.TempDir("", "k8s-ext-")
+
+ for true {
+ header, err := tarR.Next()
+
+ if err == io.EOF {
+ //Check if we have just a gzip file without a tar archive inside
+ if first {
+ return "", pkgerrors.New("Empty or non-existant Tar file found")
+ }
+ //End of archive
+ break
+ }
+
+ if err != nil {
+ return "", pkgerrors.Wrap(err, "Error reading tar file")
+ }
+
+ target := filepath.Join(outDir, header.Name)
+
+ switch header.Typeflag {
+ case tar.TypeDir:
+ if _, err := os.Stat(target); err != nil {
+ // Using 755 read, write, execute for owner
+ // groups and others get read and execute permissions
+ // on the folder.
+ if err := os.MkdirAll(target, 0755); err != nil {
+ return "", pkgerrors.Wrap(err, "Creating directory")
+ }
+ }
+ case tar.TypeReg:
+ f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+ if err != nil {
+ return "", pkgerrors.Wrap(err, "Creating file")
+ }
+
+ // copy over contents
+ if _, err := io.Copy(f, tarR); err != nil {
+ return "", pkgerrors.Wrap(err, "Copying file content")
+ }
+
+ // close for each file instead of a defer for all
+ // at the end of the function
+ f.Close()
+ }
+
+ first = false
+ }
+
+ return outDir, nil
+}
diff --git a/src/k8splugin/internal/rb/archive_test.go b/src/k8splugin/internal/rb/archive_test.go
index a327dfd4..5fa66e79 100644
--- a/src/k8splugin/internal/rb/archive_test.go
+++ b/src/k8splugin/internal/rb/archive_test.go
@@ -18,6 +18,9 @@ package rb
import (
"bytes"
+ "io/ioutil"
+ "path/filepath"
+ "strings"
"testing"
)
@@ -48,7 +51,7 @@ func TestIsTarGz(t *testing.T) {
err := isTarGz(bytes.NewBuffer(content))
if err != nil {
- t.Errorf("Error reading valid Zip file %s", err.Error())
+ t.Errorf("Error reading valid tar.gz file %s", err.Error())
}
})
@@ -63,4 +66,71 @@ func TestIsTarGz(t *testing.T) {
t.Errorf("Error should NOT be nil")
}
})
+
+ t.Run("Empty tar.gz", func(t *testing.T) {
+ content := []byte{}
+ err := isTarGz(bytes.NewBuffer(content))
+ if err == nil {
+ t.Errorf("Error should NOT be nil")
+ }
+ })
+}
+
+func TestExtractTarBall(t *testing.T) {
+
+ t.Run("Valid tar.gz", func(t *testing.T) {
+ content := []byte{
+ 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+ 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+ 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+ 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+ 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+ 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+ 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+ 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+ 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+ 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+ 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+ 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+ 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+ 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+ 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+ 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+ 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+ }
+
+ path, err := ExtractTarBall(bytes.NewBuffer(content))
+ if err != nil {
+ t.Errorf("Error reading valid tar.gz file %s", err.Error())
+ }
+ fcontent, err := ioutil.ReadFile(filepath.Join(path, "test.txt"))
+ if err != nil {
+ t.Errorf("Error reading content of valid tar.gz file %s", err.Error())
+ }
+ if strings.Contains(string(fcontent), "helloworld") == false {
+ t.Errorf("Incorrect content read from path: %s", path)
+ }
+ })
+
+ t.Run("Invalid tar.gz", func(t *testing.T) {
+ content := []byte{
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0xff, 0xf2, 0x48, 0xcd,
+ }
+
+ _, err := ExtractTarBall(bytes.NewBuffer(content))
+ if err == nil {
+ t.Errorf("Error should NOT be nil")
+ }
+ })
+
+ t.Run("Empty tar.gz", func(t *testing.T) {
+ content := []byte{}
+ err := isTarGz(bytes.NewBuffer(content))
+ if err == nil {
+ t.Errorf("Error should NOT be nil")
+ }
+ })
}
diff --git a/src/k8splugin/internal/rb/definition.go b/src/k8splugin/internal/rb/definition.go
index 8a26332b..19844990 100644
--- a/src/k8splugin/internal/rb/definition.go
+++ b/src/k8splugin/internal/rb/definition.go
@@ -162,3 +162,34 @@ func (v *DefinitionClient) Upload(id string, inp []byte) error {
return nil
}
+
+// Download the contents of the resource bundle definition from DB
+// Returns a byte array of the contents which is used by the
+// ExtractTarBall code to create the folder structure on disk
+func (v *DefinitionClient) Download(id string) ([]byte, error) {
+
+ //ignore the returned data here
+ //Check if id is valid
+ _, err := v.Get(id)
+ if err != nil {
+ return nil, pkgerrors.Errorf("Invalid Definition ID provided: %s", err.Error())
+ }
+
+ value, err := db.DBconn.Read(v.storeName, id, v.tagContent)
+ if err != nil {
+ return nil, pkgerrors.Wrap(err, "Get Resource Bundle definition content")
+ }
+
+ if value != nil {
+ //Decode the string from base64
+ out, err := base64.StdEncoding.DecodeString(string(value))
+ if err != nil {
+ return nil, pkgerrors.Wrap(err, "Decode base64 string")
+ }
+
+ if out != nil && len(out) != 0 {
+ return out, nil
+ }
+ }
+ return nil, pkgerrors.New("Error downloading Definition content")
+}
diff --git a/src/k8splugin/internal/rb/definition_test.go b/src/k8splugin/internal/rb/definition_test.go
index 46ab3c07..f720b6a9 100644
--- a/src/k8splugin/internal/rb/definition_test.go
+++ b/src/k8splugin/internal/rb/definition_test.go
@@ -19,6 +19,7 @@
package rb
import (
+ "bytes"
"k8splugin/internal/db"
"reflect"
"sort"
@@ -418,3 +419,99 @@ func TestUploadDefinition(t *testing.T) {
})
}
}
+
+func TestDownloadDefinition(t *testing.T) {
+ testCases := []struct {
+ label string
+ inp string
+ expected []byte
+ expectedError string
+ mockdb *db.MockDB
+ }{
+ {
+ label: "Download Resource Bundle Definition",
+ inp: "123e4567-e89b-12d3-a456-426655440000",
+ expected: []byte{
+ 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+ 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+ 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+ 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+ 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+ 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+ 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+ 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+ 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+ 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+ 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+ 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+ 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+ 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+ 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+ 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+ 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+ },
+ mockdb: &db.MockDB{
+ Items: map[string]map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655440000": {
+ "metadata": []byte(
+ "{\"name\":\"testresourcebundle\"," +
+ "\"description\":\"testresourcebundle\"," +
+ "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+ "\"service-type\":\"firewall\"}"),
+ "content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" +
+ "QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" +
+ "b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" +
+ "R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"),
+ },
+ },
+ },
+ },
+ {
+ label: "Download with an Invalid Resource Bundle Definition",
+ inp: "123e4567-e89b-12d3-a456-426655440000",
+ expectedError: "Invalid Definition ID provided",
+ mockdb: &db.MockDB{
+ Items: map[string]map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655441111": {
+ "metadata": []byte(
+ "{\"name\":\"testresourcebundle\"," +
+ "\"description\":\"testresourcebundle\"," +
+ "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," +
+ "\"service-type\":\"firewall\"}"),
+ },
+ },
+ },
+ },
+ {
+ label: "Download Error",
+ expectedError: "DB Error",
+ inp: "123e4567-e89b-12d3-a456-426655440000",
+ mockdb: &db.MockDB{
+ Err: pkgerrors.New("DB Error"),
+ },
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.label, func(t *testing.T) {
+ db.DBconn = testCase.mockdb
+ impl := NewDefinitionClient()
+ data, err := impl.Download(testCase.inp)
+ if err != nil {
+ if testCase.expectedError == "" {
+ t.Errorf("Download returned an unexpected error %s", err)
+ }
+ if strings.Contains(err.Error(), testCase.expectedError) == false {
+ t.Errorf("Download returned an unexpected error %s", err)
+ }
+ } else {
+ if bytes.Equal(testCase.expected, data) == false {
+ t.Errorf("Download returned unexpected data: got %v - expected %v",
+ data, testCase.expected)
+ }
+ }
+ })
+ }
+}
diff --git a/src/k8splugin/internal/rb/profile.go b/src/k8splugin/internal/rb/profile.go
index a0245af1..2456ad2d 100644
--- a/src/k8splugin/internal/rb/profile.go
+++ b/src/k8splugin/internal/rb/profile.go
@@ -183,3 +183,34 @@ func (v *ProfileClient) Upload(id string, inp []byte) error {
return nil
}
+
+// Download the contents of the resource bundle profile from DB
+// Returns a byte array of the contents which is used by the
+// ExtractTarBall code to create the folder structure on disk
+func (v *ProfileClient) Download(id string) ([]byte, error) {
+
+ //ignore the returned data here
+ //Check if id is valid
+ _, err := v.Get(id)
+ if err != nil {
+ return nil, pkgerrors.Errorf("Invalid Profile ID provided: %s", err.Error())
+ }
+
+ value, err := db.DBconn.Read(v.storeName, id, v.tagContent)
+ if err != nil {
+ return nil, pkgerrors.Wrap(err, "Get Resource Bundle Profile content")
+ }
+
+ if value != nil {
+ //Decode the string from base64
+ out, err := base64.StdEncoding.DecodeString(string(value))
+ if err != nil {
+ return nil, pkgerrors.Wrap(err, "Decode base64 string")
+ }
+
+ if out != nil && len(out) != 0 {
+ return out, nil
+ }
+ }
+ return nil, pkgerrors.New("Error downloading Profile content")
+}
diff --git a/src/k8splugin/internal/rb/profile_test.go b/src/k8splugin/internal/rb/profile_test.go
index 15ff8951..d97969de 100644
--- a/src/k8splugin/internal/rb/profile_test.go
+++ b/src/k8splugin/internal/rb/profile_test.go
@@ -19,6 +19,7 @@
package rb
import (
+ "bytes"
"k8splugin/internal/db"
"reflect"
"sort"
@@ -424,3 +425,101 @@ func TestUploadProfile(t *testing.T) {
})
}
}
+
+func TestDownloadProfile(t *testing.T) {
+ testCases := []struct {
+ label string
+ inp string
+ expected []byte
+ expectedError string
+ mockdb *db.MockDB
+ }{
+ {
+ label: "Download Resource Bundle Profile",
+ inp: "123e4567-e89b-12d3-a456-426655440000",
+ expected: []byte{
+ 0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+ 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+ 0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+ 0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+ 0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+ 0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+ 0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+ 0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+ 0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+ 0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+ 0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+ 0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+ 0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+ 0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+ 0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+ 0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+ 0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+ },
+ mockdb: &db.MockDB{
+ Items: map[string]map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655440000": {
+ "metadata": []byte(
+ "{\"name\":\"testresourcebundle\"," +
+ "\"namespace\":\"default\"," +
+ "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+ "\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," +
+ "\"kubernetesversion\":\"1.12.3\"}"),
+ "content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" +
+ "QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" +
+ "b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" +
+ "R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"),
+ },
+ },
+ },
+ },
+ {
+ label: "Download with an Invalid Resource Bundle Profile",
+ inp: "123e4567-e89b-12d3-a456-426655440000",
+ expectedError: "Invalid Profile ID provided",
+ mockdb: &db.MockDB{
+ Items: map[string]map[string][]byte{
+ "123e4567-e89b-12d3-a456-426655441111": {
+ "metadata": []byte(
+ "{\"name\":\"testresourcebundle\"," +
+ "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," +
+ "\"namespace\":\"default\"," +
+ "\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," +
+ "\"kubernetesversion\":\"1.12.3\"}"),
+ },
+ },
+ },
+ },
+ {
+ label: "Download Error",
+ expectedError: "DB Error",
+ inp: "123e4567-e89b-12d3-a456-426655440000",
+ mockdb: &db.MockDB{
+ Err: pkgerrors.New("DB Error"),
+ },
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.label, func(t *testing.T) {
+ db.DBconn = testCase.mockdb
+ impl := NewProfileClient()
+ data, err := impl.Download(testCase.inp)
+ if err != nil {
+ if testCase.expectedError == "" {
+ t.Errorf("Download returned an unexpected error %s", err)
+ }
+ if strings.Contains(err.Error(), testCase.expectedError) == false {
+ t.Errorf("Download returned an unexpected error %s", err)
+ }
+ } else {
+ if bytes.Equal(testCase.expected, data) == false {
+ t.Errorf("Download returned unexpected data: got %v - expected %v",
+ data, testCase.expected)
+ }
+ }
+ })
+ }
+}