Update status check endpoint

 
Update status check endpoint to use helm (3.5) official implementation of resource status check.
Move utils to new module and update import.

Issue-ID: MULTICLOUD-1372
Signed-off-by: hthieu <huu_trung.thieu@nokia-bell-labs.com>
Change-Id: I57a827d09466f5f554c89c2fa5533696285f9c37

4 files changed, 172 insertions, 6 deletions

diff --git a/src/k8splugin/internal/app/client.go b/src/k8splugin/internal/app/client.go
index 4c5f7e1c..623a8dc7 100644
--- a/src/k8splugin/internal/app/client.go
+++ b/src/k8splugin/internal/app/client.go
@@ -62,6 +62,90 @@ type ResourceStatus struct {
 	Status unstructured.Unstructured `json:"status"`
 }
 
+func (k *KubernetesClient) getRestApi(apiVersion string) (rest.Interface, error) {
+	//based on kubectl api-versions
+	switch apiVersion {
+	case "admissionregistration.k8s.io/v1":
+		return k.clientSet.AdmissionregistrationV1().RESTClient(), nil
+	case "admissionregistration.k8s.io/v1beta1":
+		return k.clientSet.AdmissionregistrationV1beta1().RESTClient(), nil
+	case "apps/v1":
+		return k.clientSet.AppsV1().RESTClient(), nil
+	case "apps/v1beta1":
+		return k.clientSet.AppsV1beta1().RESTClient(), nil
+	case "apps/v1beta2":
+		return k.clientSet.AppsV1beta2().RESTClient(), nil
+	case "authentication.k8s.io/v1":
+		return k.clientSet.AuthenticationV1().RESTClient(), nil
+	case "authentication.k8s.io/v1beta1":
+		return k.clientSet.AuthenticationV1beta1().RESTClient(), nil
+	case "authorization.k8s.io/v1":
+		return k.clientSet.AuthorizationV1().RESTClient(), nil
+	case "authorization.k8s.io/v1beta1":
+		return k.clientSet.AuthorizationV1beta1().RESTClient(), nil
+	case "autoscaling/v1":
+		return k.clientSet.AutoscalingV1().RESTClient(), nil
+	case "autoscaling/v2beta1":
+		return k.clientSet.AutoscalingV2beta1().RESTClient(), nil
+	case "autoscaling/v2beta2":
+		return k.clientSet.AutoscalingV2beta2().RESTClient(), nil
+	case "batch/v1":
+		return k.clientSet.BatchV1().RESTClient(), nil
+	case "batch/v1beta1":
+		return k.clientSet.BatchV1beta1().RESTClient(), nil
+	case "certificates.k8s.io/v1":
+		return k.clientSet.CertificatesV1().RESTClient(), nil
+	case "certificates.k8s.io/v1beta1":
+		return k.clientSet.CertificatesV1beta1().RESTClient(), nil
+	case "coordination.k8s.io/v1":
+		return k.clientSet.CoordinationV1().RESTClient(), nil
+	case "coordination.k8s.io/v1beta1":
+		return k.clientSet.CoordinationV1beta1().RESTClient(), nil
+	case "v1":
+		return k.clientSet.CoreV1().RESTClient(), nil
+	case "discovery.k8s.io/v1beta1":
+		return k.clientSet.DiscoveryV1beta1().RESTClient(), nil
+	case "events.k8s.io/v1":
+		return k.clientSet.EventsV1().RESTClient(), nil
+	case "events.k8s.io/v1beta1":
+		return k.clientSet.EventsV1beta1().RESTClient(), nil
+	case "extensions/v1beta1":
+		return k.clientSet.ExtensionsV1beta1().RESTClient(), nil
+	case "flowcontrol.apiserver.k8s.io/v1alpha1":
+		return k.clientSet.FlowcontrolV1alpha1().RESTClient(), nil
+	case "networking.k8s.io/v1":
+		return k.clientSet.NetworkingV1().RESTClient(), nil
+	case "networking.k8s.io/v1beta1":
+		return k.clientSet.NetworkingV1beta1().RESTClient(), nil
+	case "node.k8s.io/v1alpha1":
+		return k.clientSet.NodeV1alpha1().RESTClient(), nil
+	case "node.k8s.io/v1beta1":
+		return k.clientSet.NodeV1beta1().RESTClient(), nil
+	case "policy/v1beta1":
+		return k.clientSet.PolicyV1beta1().RESTClient(), nil
+	case "rbac.authorization.k8s.io/v1":
+		return k.clientSet.RbacV1().RESTClient(), nil
+	case "rbac.authorization.k8s.io/v1alpha1":
+		return k.clientSet.RbacV1alpha1().RESTClient(), nil
+	case "rbac.authorization.k8s.io/v1beta1":
+		return k.clientSet.RbacV1beta1().RESTClient(), nil
+	case "scheduling.k8s.io/v1":
+		return k.clientSet.SchedulingV1().RESTClient(), nil
+	case "scheduling.k8s.io/v1alpha1":
+		return k.clientSet.SchedulingV1alpha1().RESTClient(), nil
+	case "scheduling.k8s.io/v1beta1":
+		return k.clientSet.SchedulingV1beta1().RESTClient(), nil
+	case "storage.k8s.io/v1":
+		return k.clientSet.StorageV1().RESTClient(), nil
+	case "storage.k8s.io/v1alpha1":
+		return k.clientSet.StorageV1alpha1().RESTClient(), nil
+	case "storage.k8s.io/v1beta1":
+		return k.clientSet.StorageV1beta1().RESTClient(), nil
+	default:
+		return nil, pkgerrors.New("Api version " + apiVersion + " unknown")
+	}
+}
+
 // getPodsByLabel yields status of all pods under given instance ID
 func (k *KubernetesClient) getPodsByLabel(namespace string) ([]ResourceStatus, error) {
 	client := k.GetStandardClient().CoreV1().Pods(namespace)
diff --git a/src/k8splugin/internal/app/client_test.go b/src/k8splugin/internal/app/client_test.go
index 6db541a4..0ba244d2 100644
--- a/src/k8splugin/internal/app/client_test.go
+++ b/src/k8splugin/internal/app/client_test.go
@@ -15,13 +15,13 @@ package app
 
 import (
 	"encoding/base64"
+	"github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
 	"io/ioutil"
 	"os"
 	"plugin"
 	"reflect"
 	"testing"
 
-	utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/connection"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
diff --git a/src/k8splugin/internal/app/instance.go b/src/k8splugin/internal/app/instance.go
index 01d924f9..caf756a0 100644
--- a/src/k8splugin/internal/app/instance.go
+++ b/src/k8splugin/internal/app/instance.go
@@ -19,14 +19,25 @@
 package app
 
 import (
+	"context"
 	"encoding/json"
+
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/resource"
 	"log"
 	"strings"
+	"time"
 
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/namegenerator"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/rb"
+	"github.com/onap/multicloud-k8s/src/k8splugin/internal/statuscheck"
 
 	pkgerrors "github.com/pkg/errors"
 )
@@ -282,25 +293,34 @@ func (v *InstanceClient) Status(id string) (InstanceStatus, error) {
 		cumulatedErrorMsg = append(cumulatedErrorMsg, err.Error())
 	}
 
+	isReady := true
 	generalStatus := make([]ResourceStatus, 0, len(resResp.Resources))
 Main:
-	for _, resource := range resResp.Resources {
+	for _, oneResource := range resResp.Resources {
 		for _, pod := range podsStatus {
-			if resource.GVK == pod.GVK && resource.Name == pod.Name {
+			if oneResource.GVK == pod.GVK && oneResource.Name == pod.Name {
 				continue Main //Don't double check pods if someone decided to define pod explicitly in helm chart
 			}
 		}
-		status, err := k8sClient.GetResourceStatus(resource, resResp.Namespace)
+		status, err := k8sClient.GetResourceStatus(oneResource, resResp.Namespace)
 		if err != nil {
 			cumulatedErrorMsg = append(cumulatedErrorMsg, err.Error())
 		} else {
 			generalStatus = append(generalStatus, status)
 		}
+
+		ready, err := v.checkRssStatus(oneResource, k8sClient, resResp.Namespace, status)
+
+		if !ready || err != nil {
+			isReady = false
+			cumulatedErrorMsg = append(cumulatedErrorMsg, err.Error())
+			break
+		}
 	}
 	resp := InstanceStatus{
 		Request:         resResp.Request,
 		ResourceCount:   int32(len(generalStatus) + len(podsStatus)),
-		Ready:           false, //FIXME To determine readiness, some parsing of status fields is necessary
+		Ready:           isReady, //FIXME To determine readiness, some parsing of status fields is necessary
 		ResourcesStatus: append(generalStatus, podsStatus...),
 	}
 
@@ -314,6 +334,68 @@ Main:
 	return resp, nil
 }
 
+func (v *InstanceClient) checkRssStatus(rss helm.KubernetesResource, k8sClient KubernetesClient, namespace string, status ResourceStatus) (bool, error){
+	readyChecker := statuscheck.NewReadyChecker(k8sClient.clientSet, statuscheck.PausedAsReady(true), statuscheck.CheckJobs(true))
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(60)*time.Second)
+	defer cancel()
+
+	apiVersion, kind := rss.GVK.ToAPIVersionAndKind()
+	log.Printf("apiVersion: %s, Kind: %s", apiVersion, kind)
+	restClient, err := k8sClient.getRestApi(apiVersion)
+	if err != nil {
+		return false, err
+	}
+	mapper := k8sClient.GetMapper()
+	mapping, err := mapper.RESTMapping(schema.GroupKind{
+		Group: rss.GVK.Group,
+		Kind:  rss.GVK.Kind,
+	}, rss.GVK.Version)
+	resourceInfo := resource.Info{
+		Client:          restClient,
+		Mapping:         mapping,
+		Namespace:       namespace,
+		Name:            rss.Name,
+		Source:          "",
+		Object:          nil,
+		ResourceVersion: "",
+	}
+
+	var parsedRes runtime.Object
+	//TODO: Should we care about different api version for a same kind?
+	switch kind {
+	case "Pod":
+		parsedRes = new(corev1.Pod)
+	case "Job":
+		parsedRes = new(batchv1.Job)
+	case "Deployment":
+		parsedRes = new(appsv1.Deployment)
+	case "PersistentVolumeClaim":
+		parsedRes = new(corev1.PersistentVolume)
+	case "Service":
+		parsedRes = new(corev1.Service)
+	case "DaemonSet":
+		parsedRes = new(appsv1.DaemonSet)
+	case "CustomResourceDefinition":
+		parsedRes = new(apiextv1.CustomResourceDefinition)
+	case "StatefulSet":
+		parsedRes = new(appsv1.StatefulSet)
+	case "ReplicationController":
+		parsedRes = new(corev1.ReplicationController)
+	case "ReplicaSet":
+		parsedRes = new(appsv1.ReplicaSet)
+	default:
+		//For not listed resource, consider ready
+		return true, nil
+	}
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(status.Status.Object, parsedRes)
+	if err != nil {
+		return false, err
+	}
+	resourceInfo.Object = parsedRes
+	ready, err := readyChecker.IsReady(ctx, &resourceInfo)
+	return ready, err
+}
+
 // List returns the instance for corresponding ID
 // Empty string returns all
 func (v *InstanceClient) List(rbname, rbversion, profilename string) ([]InstanceMiniResponse, error) {
diff --git a/src/k8splugin/internal/app/instance_test.go b/src/k8splugin/internal/app/instance_test.go
index 2711a52f..099e3b7f 100644
--- a/src/k8splugin/internal/app/instance_test.go
+++ b/src/k8splugin/internal/app/instance_test.go
@@ -15,13 +15,13 @@ package app
 
 import (
 	"encoding/base64"
+	"github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
 	"io/ioutil"
 	"log"
 	"reflect"
 	"sort"
 	"testing"
 
-	utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/connection"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
 	"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"