Support pre/post install/delete hooks

Update instance create and delete handler to support pre/post install/delete hooks. 

Add hook.go: to execute and delete hook (base on delete policy). 

Implement watchUntilReady in generic plugin to wait for readiness of hook rss.

Add hook_sorter.go: to sort hook based on weight.

User can define timeout for each type of hooks in overwrite-values. Variable name is k8s-rb-instance-pre-install-timeout (default 60s),
k8s-rb-instance-post-install-timeout (default 600s), k8s-rb-instance-pre-delete-timeout (default 60s) and k8s-rb-instance-post-delete-timeout (600s). This is timeout
for each hook of a hook event (not a total time).

Add recovery capability to continue the execution of instantiation (create or delete) when the plugin stop unexpectedly.
For now, this is disabled because we have data-race issue during test. Will enable when we find the solution.

Add basic test for hooks (in hook_test.go)

Add test for hook in instance_test

For instance get request, we can request for full data by adding query param to the request: full=true.

Issue-ID: MULTICLOUD-1347
Signed-off-by: hthieu <huu_trung.thieu@nokia-bell-labs.com>
Change-Id: If2b4a90831b9bfce1af8b926e4062a7d706bee08

1 files changed, 60 insertions, 4 deletions

diff --git a/src/k8splugin/internal/app/client.go b/src/k8splugin/internal/app/client.go
index 87e5c379..9813333e 100644
--- a/src/k8splugin/internal/app/client.go
+++ b/src/k8splugin/internal/app/client.go
@@ -2,6 +2,7 @@
 Copyright 2018 Intel Corporation.
 Copyright © 2021 Samsung Electronics
 Copyright © 2021 Orange
+Copyright © 2021 Nokia Bell Labs.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,6 +20,15 @@ package app
 import (
 	"context"
 	"io/ioutil"
+	appsv1 "k8s.io/api/apps/v1"
+	//appsv1beta1 "k8s.io/api/apps/v1beta1"
+	//appsv1beta2 "k8s.io/api/apps/v1beta2"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	//extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+	//apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	//apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"os"
 	"strings"
 	"time"
@@ -28,10 +38,10 @@ import (
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
 	log "github.com/onap/multicloud-k8s/src/k8splugin/internal/logutils"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin"
+	logger "log"
 
 	pkgerrors "github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -62,6 +72,23 @@ type ResourceStatus struct {
 	Status unstructured.Unstructured `json:"status"`
 }
 
+func (k *KubernetesClient) getObjTypeForHook(kind string) (runtime.Object, error) {
+	switch kind {
+	case "Job":
+		return &batchv1.Job{}, nil
+	case "Pod":
+		return &corev1.Pod{}, nil
+	case "Deployment":
+		return &appsv1.Deployment{}, nil
+	case "DaemonSet":
+		return &appsv1.DaemonSet{}, nil
+	case "StatefulSet":
+		return &appsv1.StatefulSet{}, nil
+	default:
+		return nil, pkgerrors.New("kind " + kind + " unknown")
+	}
+}
+
 func (k *KubernetesClient) getRestApi(apiVersion string) (rest.Interface, error) {
 	//based on kubectl api-versions
 	switch apiVersion {
@@ -146,6 +173,36 @@ func (k *KubernetesClient) getRestApi(apiVersion string) (rest.Interface, error)
 	}
 }
 
+func (k *KubernetesClient) WatchHookUntilReady(timeout time.Duration, ns string, res helm.KubernetesResource) error {
+	//for now, only generic plugin has dedicated WatchUntilReady implemented. Later, we can implement this function
+	//for each plugin separately.
+	pluginImpl, err := plugin.GetPluginByKind("generic")
+	if err != nil {
+		return pkgerrors.Wrap(err, "Error loading plugin")
+	}
+
+	mapper := k.GetMapper()
+	apiVersion, kind := res.GVK.ToAPIVersionAndKind()
+	if apiVersion == "" {
+		//apiVersion is empty -> we can suppose that the rss is ready
+		logger.Printf("apiVersion is empty, consider that the rss is ready")
+		return nil
+	}
+	objType, err := k.getObjTypeForHook(kind)
+	if err != nil {
+		//have error from getObjTypeForHook -> this kind is not considered in hook -> consider ready
+		return nil
+	}
+
+	logger.Printf("apiVersion: %s, Kind: %s", apiVersion, kind)
+	restClient, err := k.getRestApi(apiVersion)
+	if err != nil {
+		return pkgerrors.Wrap(err, "Get rest client")
+	}
+
+	return pluginImpl.WatchUntilReady(timeout, ns, res, mapper, restClient, objType, k.clientSet)
+}
+
 // getPodsByLabel yields status of all pods under given instance ID
 func (k *KubernetesClient) getPodsByLabel(namespace string) ([]ResourceStatus, error) {
 	client := k.GetStandardClient().CoreV1().Pods(namespace)
@@ -363,8 +420,7 @@ func (k *KubernetesClient) ensureNamespace(namespace string) error {
 	return nil
 }
 
-func (k *KubernetesClient) CreateKind(resTempl helm.KubernetesResourceTemplate,
-	namespace string) (helm.KubernetesResource, error) {
+func (k *KubernetesClient) CreateKind(resTempl helm.KubernetesResourceTemplate, namespace string) (helm.KubernetesResource, error) {
 
 	if _, err := os.Stat(resTempl.FilePath); os.IsNotExist(err) {
 		return helm.KubernetesResource{}, pkgerrors.New("File " + resTempl.FilePath + "does not exists")
@@ -404,7 +460,7 @@ func (k *KubernetesClient) updateKind(resTempl helm.KubernetesResourceTemplate,
 	namespace string) (helm.KubernetesResource, error) {
 
 	if _, err := os.Stat(resTempl.FilePath); os.IsNotExist(err) {
-		return helm.KubernetesResource{}, pkgerrors.New("File " + resTempl.FilePath + "does not exists")
+		return helm.KubernetesResource{}, pkgerrors.New("File " + resTempl.FilePath + " does not exists")
 	}
 
 	log.Info("Processing Kubernetes Resource", log.Fields{