Introduce Monitor support for CSR resource

These changes allow the Monitor to also track CSR
(CertificateSigningResource) resources which will make
it possible to know when a certificate has been issued by
the K8s cluster signer. In turn, DCM will be able to read,
store and use that certificate to generate kubeconfigs.

Out-of-tree actions required:
- publish monitor's docker image built from this source
  onto emcov2/monitor:latest

Issue-ID: MULTICLOUD-1143
Change-Id: I7facd27bbfe08891151bb3b6a9a19948435e24e4
Signed-off-by: Igor D.C <igor.duarte.cardoso@intel.com>

7 files changed, 288 insertions, 11 deletions

diff --git a/src/monitor/deploy/cluster_role.yaml b/src/monitor/deploy/cluster_role.yaml
index 0732e8d3..9330a687 100644
--- a/src/monitor/deploy/cluster_role.yaml
+++ b/src/monitor/deploy/cluster_role.yaml
@@ -70,3 +70,9 @@ rules:
   - '*'
   verbs:
   - '*'
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
diff --git a/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go b/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
index 6476f0db..fba10bb3 100644
--- a/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
+++ b/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
@@ -3,6 +3,7 @@ package v1alpha1
 import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/batch/v1"
+	certsapi "k8s.io/api/certificates/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	v1beta1 "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,17 +32,18 @@ type ResourceBundleStateSpec struct {
 // ResourceBundleStatus defines the observed state of ResourceBundleState
 // +k8s:openapi-gen=true
 type ResourceBundleStatus struct {
-	Ready               bool                 `json:"ready" protobuf:"varint,1,opt,name=ready"`
-	ResourceCount       int32                `json:"resourceCount" protobuf:"varint,2,opt,name=resourceCount"`
-	PodStatuses         []corev1.Pod         `json:"podStatuses" protobuf:"varint,3,opt,name=podStatuses"`
-	ServiceStatuses     []corev1.Service     `json:"serviceStatuses" protobuf:"varint,4,opt,name=serviceStatuses"`
-	ConfigMapStatuses   []corev1.ConfigMap   `json:"configMapStatuses" protobuf:"varint,5,opt,name=configMapStatuses"`
-	DeploymentStatuses  []appsv1.Deployment  `json:"deploymentStatuses" protobuf:"varint,6,opt,name=deploymentStatuses"`
-	SecretStatuses      []corev1.Secret      `json:"secretStatuses" protobuf:"varint,7,opt,name=secretStatuses"`
-	DaemonSetStatuses   []appsv1.DaemonSet   `json:"daemonSetStatuses" protobuf:"varint,8,opt,name=daemonSetStatuses"`
-	IngressStatuses     []v1beta1.Ingress    `json:"ingressStatuses" protobuf:"varint,11,opt,name=ingressStatuses"`
-	JobStatuses         []v1.Job             `json:"jobStatuses" protobuf:"varint,12,opt,name=jobStatuses"`
-	StatefulSetStatuses []appsv1.StatefulSet `json:"statefulSetStatuses" protobuf:"varint,13,opt,name=statefulSetStatuses"`
+	Ready               bool                                 `json:"ready" protobuf:"varint,1,opt,name=ready"`
+	ResourceCount       int32                                `json:"resourceCount" protobuf:"varint,2,opt,name=resourceCount"`
+	PodStatuses         []corev1.Pod                         `json:"podStatuses" protobuf:"varint,3,opt,name=podStatuses"`
+	ServiceStatuses     []corev1.Service                     `json:"serviceStatuses" protobuf:"varint,4,opt,name=serviceStatuses"`
+	ConfigMapStatuses   []corev1.ConfigMap                   `json:"configMapStatuses" protobuf:"varint,5,opt,name=configMapStatuses"`
+	DeploymentStatuses  []appsv1.Deployment                  `json:"deploymentStatuses" protobuf:"varint,6,opt,name=deploymentStatuses"`
+	SecretStatuses      []corev1.Secret                      `json:"secretStatuses" protobuf:"varint,7,opt,name=secretStatuses"`
+	DaemonSetStatuses   []appsv1.DaemonSet                   `json:"daemonSetStatuses" protobuf:"varint,8,opt,name=daemonSetStatuses"`
+	IngressStatuses     []v1beta1.Ingress                    `json:"ingressStatuses" protobuf:"varint,11,opt,name=ingressStatuses"`
+	JobStatuses         []v1.Job                             `json:"jobStatuses" protobuf:"varint,12,opt,name=jobStatuses"`
+	StatefulSetStatuses []appsv1.StatefulSet                 `json:"statefulSetStatuses" protobuf:"varint,13,opt,name=statefulSetStatuses"`
+	CsrStatuses         []certsapi.CertificateSigningRequest `json:"csrStatuses" protobuf:"varint,3,opt,name=csrStatuses"`
 }
 
 // PodStatus defines the observed state of ResourceBundleState
diff --git a/src/monitor/pkg/controller/add_resourcebundlestate.go b/src/monitor/pkg/controller/add_resourcebundlestate.go
index ee42f9cf..d06e97d6 100644
--- a/src/monitor/pkg/controller/add_resourcebundlestate.go
+++ b/src/monitor/pkg/controller/add_resourcebundlestate.go
@@ -15,4 +15,5 @@ func init() {
 	AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddIngressController)
 	AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddJobController)
 	AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddStatefulSetController)
+	AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddCsrController)
 }
diff --git a/src/monitor/pkg/controller/resourcebundlestate/controller.go b/src/monitor/pkg/controller/resourcebundlestate/controller.go
index faee5892..5351ea99 100644
--- a/src/monitor/pkg/controller/resourcebundlestate/controller.go
+++ b/src/monitor/pkg/controller/resourcebundlestate/controller.go
@@ -8,6 +8,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/batch/v1"
+	certsapi "k8s.io/api/certificates/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	v1beta1 "k8s.io/api/extensions/v1beta1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -119,6 +120,12 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error)
 		return reconcile.Result{}, err
 	}
 
+	err = r.updateCsrs(rbstate, rbstate.Spec.Selector.MatchLabels)
+	if err != nil {
+		log.Printf("Error adding csrStatuses: %v\n", err)
+		return reconcile.Result{}, err
+	}
+
 	// TODO: Update this based on the statuses of the lower resources
 	rbstate.Status.Ready = false
 	err = r.client.Status().Update(context.TODO(), rbstate)
@@ -352,3 +359,28 @@ func (r *reconciler) updateStatefulSets(rbstate *v1alpha1.ResourceBundleState,
 
 	return nil
 }
+
+func (r *reconciler) updateCsrs(rbstate *v1alpha1.ResourceBundleState,
+	selectors map[string]string) error {
+
+	// Update the CR with the csrs tracked
+	csrList := &certsapi.CertificateSigningRequestList{}
+	err := listResources(r.client, rbstate.Namespace, selectors, csrList)
+	if err != nil {
+		log.Printf("Failed to list csrs: %v", err)
+		return err
+	}
+
+	rbstate.Status.CsrStatuses = []certsapi.CertificateSigningRequest{}
+
+	for _, csr := range csrList.Items {
+		resStatus := certsapi.CertificateSigningRequest{
+			TypeMeta:   csr.TypeMeta,
+			ObjectMeta: csr.ObjectMeta,
+			Status:     csr.Status,
+		}
+		rbstate.Status.CsrStatuses = append(rbstate.Status.CsrStatuses, resStatus)
+	}
+
+	return nil
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go b/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go
new file mode 100644
index 00000000..918fadfb
--- /dev/null
+++ b/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go
@@ -0,0 +1,183 @@
+package resourcebundlestate
+
+import (
+	"context"
+	"log"
+
+	"github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
+
+	certsapi "k8s.io/api/certificates/v1beta1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+// AddCsrController the new controller to the controller manager
+func AddCsrController(mgr manager.Manager) error {
+	return addCsrController(mgr, newCsrReconciler(mgr))
+}
+
+func addCsrController(mgr manager.Manager, r *csrReconciler) error {
+	// Create a new controller
+	c, err := controller.New("Csr-controller", mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return err
+	}
+
+	// Watch for changes to secondary resource Csrs
+	// Predicate filters csrs which don't have the k8splugin label
+	err = c.Watch(&source.Kind{Type: &certsapi.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}, &csrPredicate{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func newCsrReconciler(m manager.Manager) *csrReconciler {
+	return &csrReconciler{client: m.GetClient()}
+}
+
+type csrReconciler struct {
+	client client.Client
+}
+
+// Reconcile implements the loop that will update the ResourceBundleState CR
+// whenever we get any updates from all the csrs we watch.
+func (r *csrReconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) {
+	log.Printf("Updating ResourceBundleState for Csr: %+v\n", req)
+
+	csr := &certsapi.CertificateSigningRequest{}
+	err := r.client.Get(context.TODO(), req.NamespacedName, csr)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			log.Printf("Csr not found: %+v. Remove from CR if it is stored there.\n", req.NamespacedName)
+			// Remove the Csr's status from StatusList
+			// This can happen if we get the DeletionTimeStamp event
+			// after the POD has been deleted.
+			r.deleteCsrFromAllCRs(req.NamespacedName)
+			return reconcile.Result{}, nil
+		}
+		log.Printf("Failed to get csr: %+v\n", req.NamespacedName)
+		return reconcile.Result{}, err
+	}
+
+	// Find the CRs which track this csr via the labelselector
+	crSelector := returnLabel(csr.GetLabels())
+	if crSelector == nil {
+		log.Println("We should not be here. The predicate should have filtered this Csr")
+	}
+
+	// Get the CRs which have this label and update them all
+	// Ideally, we will have only one CR, but there is nothing
+	// preventing the creation of multiple.
+	// TODO: Consider using an admission validating webook to prevent multiple
+	rbStatusList := &v1alpha1.ResourceBundleStateList{}
+	err = listClusterResources(r.client, crSelector, rbStatusList)
+	if err != nil || len(rbStatusList.Items) == 0 {
+		log.Printf("Did not find any CRs tracking this resource\n")
+		return reconcile.Result{}, nil
+	}
+
+	err = r.updateCRs(rbStatusList, csr)
+	if err != nil {
+		// Requeue the update
+		return reconcile.Result{}, err
+	}
+
+	return reconcile.Result{}, nil
+}
+
+// deleteCsrFromAllCRs deletes csr status from all the CRs when the POD itself has been deleted
+// and we have not handled the updateCRs yet.
+// Since, we don't have the csr's labels, we need to look at all the CRs in this namespace
+func (r *csrReconciler) deleteCsrFromAllCRs(namespacedName types.NamespacedName) error {
+
+	rbStatusList := &v1alpha1.ResourceBundleStateList{}
+	err := listClusterResources(r.client, nil, rbStatusList)
+	if err != nil || len(rbStatusList.Items) == 0 {
+		log.Printf("Did not find any CRs tracking this resource\n")
+		return nil
+	}
+	for _, cr := range rbStatusList.Items {
+		r.deleteFromSingleCR(&cr, namespacedName.Name)
+	}
+
+	return nil
+}
+
+func (r *csrReconciler) updateCRs(crl *v1alpha1.ResourceBundleStateList, csr *certsapi.CertificateSigningRequest) error {
+
+	for _, cr := range crl.Items {
+		// Csr is not scheduled for deletion
+		if csr.DeletionTimestamp == nil {
+			err := r.updateSingleCR(&cr, csr)
+			if err != nil {
+				return err
+			}
+		} else {
+			// Csr is scheduled for deletion
+			r.deleteFromSingleCR(&cr, csr.Name)
+		}
+	}
+
+	return nil
+}
+
+func (r *csrReconciler) deleteFromSingleCR(cr *v1alpha1.ResourceBundleState, name string) error {
+	cr.Status.ResourceCount--
+	length := len(cr.Status.CsrStatuses)
+	for i, rstatus := range cr.Status.CsrStatuses {
+		if rstatus.Name == name {
+			//Delete that status from the array
+			cr.Status.CsrStatuses[i] = cr.Status.CsrStatuses[length-1]
+			cr.Status.CsrStatuses[length-1] = certsapi.CertificateSigningRequest{}
+			cr.Status.CsrStatuses = cr.Status.CsrStatuses[:length-1]
+			return nil
+		}
+	}
+
+	log.Println("Did not find a status for POD in CR")
+	return nil
+}
+
+func (r *csrReconciler) updateSingleCR(cr *v1alpha1.ResourceBundleState, csr *certsapi.CertificateSigningRequest) error {
+
+	// Update status after searching for it in the list of resourceStatuses
+	for i, rstatus := range cr.Status.CsrStatuses {
+		// Look for the status if we already have it in the CR
+		if rstatus.Name == csr.Name {
+			csr.Status.DeepCopyInto(&cr.Status.CsrStatuses[i].Status)
+			err := r.client.Status().Update(context.TODO(), cr)
+			if err != nil {
+				log.Printf("failed to update rbstate: %v\n", err)
+				return err
+			}
+			return nil
+		}
+	}
+
+	// Exited for loop with no status found
+	// Increment the number of tracked resources
+	cr.Status.ResourceCount++
+
+	// Add it to CR
+	cr.Status.CsrStatuses = append(cr.Status.CsrStatuses, certsapi.CertificateSigningRequest{
+		TypeMeta:   csr.TypeMeta,
+		ObjectMeta: csr.ObjectMeta,
+		Status:     csr.Status,
+	})
+
+	err := r.client.Status().Update(context.TODO(), cr)
+	if err != nil {
+		log.Printf("failed to update rbstate: %v\n", err)
+		return err
+	}
+
+	return nil
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go b/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go
new file mode 100644
index 00000000..cb83eec1
--- /dev/null
+++ b/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go
@@ -0,0 +1,44 @@
+package resourcebundlestate
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type csrPredicate struct {
+}
+
+func (p *csrPredicate) Create(evt event.CreateEvent) bool {
+
+	if evt.Meta == nil {
+		return false
+	}
+
+	labels := evt.Meta.GetLabels()
+	return checkLabel(labels)
+}
+
+func (p *csrPredicate) Delete(evt event.DeleteEvent) bool {
+
+	if evt.Meta == nil {
+		return false
+	}
+
+	labels := evt.Meta.GetLabels()
+	return checkLabel(labels)
+}
+
+func (p *csrPredicate) Update(evt event.UpdateEvent) bool {
+
+	if evt.MetaNew == nil {
+		return false
+	}
+
+	labels := evt.MetaNew.GetLabels()
+	return checkLabel(labels)
+}
+
+func (p *csrPredicate) Generic(evt event.GenericEvent) bool {
+
+	labels := evt.Meta.GetLabels()
+	return checkLabel(labels)
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/helpers.go b/src/monitor/pkg/controller/resourcebundlestate/helpers.go
index 9a6538b4..4d6be370 100644
--- a/src/monitor/pkg/controller/resourcebundlestate/helpers.go
+++ b/src/monitor/pkg/controller/resourcebundlestate/helpers.go
@@ -52,3 +52,12 @@ func listResources(cli client.Client, namespace string,
 
 	return nil
 }
+
+// listClusterResources lists non-namespace resources based
+// on the selectors provided.
+// The data is returned in the pointer to the runtime.Object
+// provided as argument.
+func listClusterResources(cli client.Client,
+	labelSelector map[string]string, returnData runtime.Object) error {
+	return listResources(cli, "", labelSelector, returnData)
+}