Add support for downloading content

Add support for downloading content for creating
the merged helm charts. This api will be used mainly
by the profile api to create a converged chart which
will then be created by the instantiation code
P2: Add unit tests for archive.go
    Add download method for profile.go
    Update comments for download method
P3: Add unit test for empty files
P4: Add unit tests for Download functions
P5: Rebase against new folder structure

Issue-ID: MULTICLOUD-291
Change-Id: I9779eaf95366f527f0360eaddea663722c13b196
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>

6 files changed, 400 insertions, 2 deletions

diff --git a/src/k8splugin/internal/rb/archive.go b/src/k8splugin/internal/rb/archive.go
index 8eb0fbed..624adfba 100644
--- a/src/k8splugin/internal/rb/archive.go
+++ b/src/k8splugin/internal/rb/archive.go
@@ -21,13 +21,16 @@ import (
 	"compress/gzip"
 	pkgerrors "github.com/pkg/errors"
 	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 )
 
 func isTarGz(r io.Reader) error {
 	//Check if it is a valid gz
 	gzf, err := gzip.NewReader(r)
 	if err != nil {
-		return pkgerrors.Errorf("Invalid gz format %s", err.Error())
+		return pkgerrors.Wrap(err, "Invalid gzip format")
 	}
 
 	//Check if it is a valid tar file
@@ -63,3 +66,70 @@ func isTarGz(r io.Reader) error {
 
 	return nil
 }
+
+//ExtractTarBall provides functionality to extract a tar.gz file
+//into a temporary location for later use.
+//It returns the path to the new location
+func ExtractTarBall(r io.Reader) (string, error) {
+	//Check if it is a valid gz
+	gzf, err := gzip.NewReader(r)
+	if err != nil {
+		return "", pkgerrors.Wrap(err, "Invalid gzip format")
+	}
+
+	//Check if it is a valid tar file
+	//Unfortunately this can only be done by inspecting all the tar contents
+	tarR := tar.NewReader(gzf)
+	first := true
+
+	outDir, _ := ioutil.TempDir("", "k8s-ext-")
+
+	for true {
+		header, err := tarR.Next()
+
+		if err == io.EOF {
+			//Check if we have just a gzip file without a tar archive inside
+			if first {
+				return "", pkgerrors.New("Empty or non-existant Tar file found")
+			}
+			//End of archive
+			break
+		}
+
+		if err != nil {
+			return "", pkgerrors.Wrap(err, "Error reading tar file")
+		}
+
+		target := filepath.Join(outDir, header.Name)
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				// Using 755 read, write, execute for owner
+				// groups and others get read and execute permissions
+				// on the folder.
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return "", pkgerrors.Wrap(err, "Creating directory")
+				}
+			}
+		case tar.TypeReg:
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return "", pkgerrors.Wrap(err, "Creating file")
+			}
+
+			// copy over contents
+			if _, err := io.Copy(f, tarR); err != nil {
+				return "", pkgerrors.Wrap(err, "Copying file content")
+			}
+
+			// close for each file instead of a defer for all
+			// at the end of the function
+			f.Close()
+		}
+
+		first = false
+	}
+
+	return outDir, nil
+}
diff --git a/src/k8splugin/internal/rb/archive_test.go b/src/k8splugin/internal/rb/archive_test.go
index a327dfd4..5fa66e79 100644
--- a/src/k8splugin/internal/rb/archive_test.go
+++ b/src/k8splugin/internal/rb/archive_test.go
@@ -18,6 +18,9 @@ package rb
 
 import (
 	"bytes"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -48,7 +51,7 @@ func TestIsTarGz(t *testing.T) {
 
 		err := isTarGz(bytes.NewBuffer(content))
 		if err != nil {
-			t.Errorf("Error reading valid Zip file %s", err.Error())
+			t.Errorf("Error reading valid tar.gz file %s", err.Error())
 		}
 	})
 
@@ -63,4 +66,71 @@ func TestIsTarGz(t *testing.T) {
 			t.Errorf("Error should NOT be nil")
 		}
 	})
+
+	t.Run("Empty tar.gz", func(t *testing.T) {
+		content := []byte{}
+		err := isTarGz(bytes.NewBuffer(content))
+		if err == nil {
+			t.Errorf("Error should NOT be nil")
+		}
+	})
+}
+
+func TestExtractTarBall(t *testing.T) {
+
+	t.Run("Valid tar.gz", func(t *testing.T) {
+		content := []byte{
+			0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+			0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+			0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+			0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+			0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+			0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+			0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+			0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+			0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+			0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+			0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+			0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+			0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+			0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+			0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+			0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+			0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+		}
+
+		path, err := ExtractTarBall(bytes.NewBuffer(content))
+		if err != nil {
+			t.Errorf("Error reading valid tar.gz file %s", err.Error())
+		}
+		fcontent, err := ioutil.ReadFile(filepath.Join(path, "test.txt"))
+		if err != nil {
+			t.Errorf("Error reading content of valid tar.gz file %s", err.Error())
+		}
+		if strings.Contains(string(fcontent), "helloworld") == false {
+			t.Errorf("Incorrect content read from path: %s", path)
+		}
+	})
+
+	t.Run("Invalid tar.gz", func(t *testing.T) {
+		content := []byte{
+			0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0xff, 0xf2, 0x48, 0xcd,
+		}
+
+		_, err := ExtractTarBall(bytes.NewBuffer(content))
+		if err == nil {
+			t.Errorf("Error should NOT be nil")
+		}
+	})
+
+	t.Run("Empty tar.gz", func(t *testing.T) {
+		content := []byte{}
+		err := isTarGz(bytes.NewBuffer(content))
+		if err == nil {
+			t.Errorf("Error should NOT be nil")
+		}
+	})
 }
diff --git a/src/k8splugin/internal/rb/definition.go b/src/k8splugin/internal/rb/definition.go
index 8a26332b..19844990 100644
--- a/src/k8splugin/internal/rb/definition.go
+++ b/src/k8splugin/internal/rb/definition.go
@@ -162,3 +162,34 @@ func (v *DefinitionClient) Upload(id string, inp []byte) error {
 
 	return nil
 }
+
+// Download the contents of the resource bundle definition from DB
+// Returns a byte array of the contents which is used by the
+// ExtractTarBall code to create the folder structure on disk
+func (v *DefinitionClient) Download(id string) ([]byte, error) {
+
+	//ignore the returned data here
+	//Check if id is valid
+	_, err := v.Get(id)
+	if err != nil {
+		return nil, pkgerrors.Errorf("Invalid Definition ID provided: %s", err.Error())
+	}
+
+	value, err := db.DBconn.Read(v.storeName, id, v.tagContent)
+	if err != nil {
+		return nil, pkgerrors.Wrap(err, "Get Resource Bundle definition content")
+	}
+
+	if value != nil {
+		//Decode the string from base64
+		out, err := base64.StdEncoding.DecodeString(string(value))
+		if err != nil {
+			return nil, pkgerrors.Wrap(err, "Decode base64 string")
+		}
+
+		if out != nil && len(out) != 0 {
+			return out, nil
+		}
+	}
+	return nil, pkgerrors.New("Error downloading Definition content")
+}
diff --git a/src/k8splugin/internal/rb/definition_test.go b/src/k8splugin/internal/rb/definition_test.go
index 46ab3c07..f720b6a9 100644
--- a/src/k8splugin/internal/rb/definition_test.go
+++ b/src/k8splugin/internal/rb/definition_test.go
@@ -19,6 +19,7 @@
 package rb
 
 import (
+	"bytes"
 	"k8splugin/internal/db"
 	"reflect"
 	"sort"
@@ -418,3 +419,99 @@ func TestUploadDefinition(t *testing.T) {
 		})
 	}
 }
+
+func TestDownloadDefinition(t *testing.T) {
+	testCases := []struct {
+		label         string
+		inp           string
+		expected      []byte
+		expectedError string
+		mockdb        *db.MockDB
+	}{
+		{
+			label: "Download Resource Bundle Definition",
+			inp:   "123e4567-e89b-12d3-a456-426655440000",
+			expected: []byte{
+				0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+				0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+				0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+				0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+				0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+				0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+				0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+				0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+				0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+				0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+				0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+				0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+				0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+				0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+				0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+				0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+				0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+			},
+			mockdb: &db.MockDB{
+				Items: map[string]map[string][]byte{
+					"123e4567-e89b-12d3-a456-426655440000": {
+						"metadata": []byte(
+							"{\"name\":\"testresourcebundle\"," +
+								"\"description\":\"testresourcebundle\"," +
+								"\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+								"\"service-type\":\"firewall\"}"),
+						"content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" +
+							"QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" +
+							"b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" +
+							"R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"),
+					},
+				},
+			},
+		},
+		{
+			label:         "Download with an Invalid Resource Bundle Definition",
+			inp:           "123e4567-e89b-12d3-a456-426655440000",
+			expectedError: "Invalid Definition ID provided",
+			mockdb: &db.MockDB{
+				Items: map[string]map[string][]byte{
+					"123e4567-e89b-12d3-a456-426655441111": {
+						"metadata": []byte(
+							"{\"name\":\"testresourcebundle\"," +
+								"\"description\":\"testresourcebundle\"," +
+								"\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," +
+								"\"service-type\":\"firewall\"}"),
+					},
+				},
+			},
+		},
+		{
+			label:         "Download Error",
+			expectedError: "DB Error",
+			inp:           "123e4567-e89b-12d3-a456-426655440000",
+			mockdb: &db.MockDB{
+				Err: pkgerrors.New("DB Error"),
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.label, func(t *testing.T) {
+			db.DBconn = testCase.mockdb
+			impl := NewDefinitionClient()
+			data, err := impl.Download(testCase.inp)
+			if err != nil {
+				if testCase.expectedError == "" {
+					t.Errorf("Download returned an unexpected error %s", err)
+				}
+				if strings.Contains(err.Error(), testCase.expectedError) == false {
+					t.Errorf("Download returned an unexpected error %s", err)
+				}
+			} else {
+				if bytes.Equal(testCase.expected, data) == false {
+					t.Errorf("Download returned unexpected data: got %v - expected %v",
+						data, testCase.expected)
+				}
+			}
+		})
+	}
+}
diff --git a/src/k8splugin/internal/rb/profile.go b/src/k8splugin/internal/rb/profile.go
index a0245af1..2456ad2d 100644
--- a/src/k8splugin/internal/rb/profile.go
+++ b/src/k8splugin/internal/rb/profile.go
@@ -183,3 +183,34 @@ func (v *ProfileClient) Upload(id string, inp []byte) error {
 
 	return nil
 }
+
+// Download the contents of the resource bundle profile from DB
+// Returns a byte array of the contents which is used by the
+// ExtractTarBall code to create the folder structure on disk
+func (v *ProfileClient) Download(id string) ([]byte, error) {
+
+	//ignore the returned data here
+	//Check if id is valid
+	_, err := v.Get(id)
+	if err != nil {
+		return nil, pkgerrors.Errorf("Invalid Profile ID provided: %s", err.Error())
+	}
+
+	value, err := db.DBconn.Read(v.storeName, id, v.tagContent)
+	if err != nil {
+		return nil, pkgerrors.Wrap(err, "Get Resource Bundle Profile content")
+	}
+
+	if value != nil {
+		//Decode the string from base64
+		out, err := base64.StdEncoding.DecodeString(string(value))
+		if err != nil {
+			return nil, pkgerrors.Wrap(err, "Decode base64 string")
+		}
+
+		if out != nil && len(out) != 0 {
+			return out, nil
+		}
+	}
+	return nil, pkgerrors.New("Error downloading Profile content")
+}
diff --git a/src/k8splugin/internal/rb/profile_test.go b/src/k8splugin/internal/rb/profile_test.go
index 15ff8951..d97969de 100644
--- a/src/k8splugin/internal/rb/profile_test.go
+++ b/src/k8splugin/internal/rb/profile_test.go
@@ -19,6 +19,7 @@
 package rb
 
 import (
+	"bytes"
 	"k8splugin/internal/db"
 	"reflect"
 	"sort"
@@ -424,3 +425,101 @@ func TestUploadProfile(t *testing.T) {
 		})
 	}
 }
+
+func TestDownloadProfile(t *testing.T) {
+	testCases := []struct {
+		label         string
+		inp           string
+		expected      []byte
+		expectedError string
+		mockdb        *db.MockDB
+	}{
+		{
+			label: "Download Resource Bundle Profile",
+			inp:   "123e4567-e89b-12d3-a456-426655440000",
+			expected: []byte{
+				0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+				0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+				0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+				0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+				0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+				0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+				0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+				0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+				0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+				0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+				0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+				0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+				0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+				0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+				0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+				0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+				0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+			},
+			mockdb: &db.MockDB{
+				Items: map[string]map[string][]byte{
+					"123e4567-e89b-12d3-a456-426655440000": {
+						"metadata": []byte(
+							"{\"name\":\"testresourcebundle\"," +
+								"\"namespace\":\"default\"," +
+								"\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+								"\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," +
+								"\"kubernetesversion\":\"1.12.3\"}"),
+						"content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" +
+							"QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" +
+							"b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" +
+							"R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"),
+					},
+				},
+			},
+		},
+		{
+			label:         "Download with an Invalid Resource Bundle Profile",
+			inp:           "123e4567-e89b-12d3-a456-426655440000",
+			expectedError: "Invalid Profile ID provided",
+			mockdb: &db.MockDB{
+				Items: map[string]map[string][]byte{
+					"123e4567-e89b-12d3-a456-426655441111": {
+						"metadata": []byte(
+							"{\"name\":\"testresourcebundle\"," +
+								"\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," +
+								"\"namespace\":\"default\"," +
+								"\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," +
+								"\"kubernetesversion\":\"1.12.3\"}"),
+					},
+				},
+			},
+		},
+		{
+			label:         "Download Error",
+			expectedError: "DB Error",
+			inp:           "123e4567-e89b-12d3-a456-426655440000",
+			mockdb: &db.MockDB{
+				Err: pkgerrors.New("DB Error"),
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.label, func(t *testing.T) {
+			db.DBconn = testCase.mockdb
+			impl := NewProfileClient()
+			data, err := impl.Download(testCase.inp)
+			if err != nil {
+				if testCase.expectedError == "" {
+					t.Errorf("Download returned an unexpected error %s", err)
+				}
+				if strings.Contains(err.Error(), testCase.expectedError) == false {
+					t.Errorf("Download returned an unexpected error %s", err)
+				}
+			} else {
+				if bytes.Equal(testCase.expected, data) == false {
+					t.Errorf("Download returned unexpected data: got %v - expected %v",
+						data, testCase.expected)
+				}
+			}
+		})
+	}
+}