aboutsummaryrefslogtreecommitdiffstats
path: root/src/monitor
diff options
context:
space:
mode:
authorIgor D.C <igor.duarte.cardoso@intel.com>2020-08-31 22:46:10 +0000
committerIgor D.C <igor.duarte.cardoso@intel.com>2020-09-25 18:43:36 +0000
commit425795c7d4e6ce81932918aca2a1462384d4507f (patch)
tree68f9793b29b3d282e62426ab52669319b4dfd4eb /src/monitor
parent8b5c4236639a46f39cbfe852590f34e64f58a85a (diff)
Introduce Monitor support for CSR resource
These changes allow the Monitor to also track CSR (CertificateSigningResource) resources which will make it possible to know when a certificate has been issued by the K8s cluster signer. In turn, DCM will be able to read, store and use that certificate to generate kubeconfigs. Out-of-tree actions required: - publish monitor's docker image built from this source onto emcov2/monitor:latest Issue-ID: MULTICLOUD-1143 Change-Id: I7facd27bbfe08891151bb3b6a9a19948435e24e4 Signed-off-by: Igor D.C <igor.duarte.cardoso@intel.com>
Diffstat (limited to 'src/monitor')
-rw-r--r--src/monitor/deploy/cluster_role.yaml6
-rw-r--r--src/monitor/pkg/apis/k8splugin/v1alpha1/types.go24
-rw-r--r--src/monitor/pkg/controller/add_resourcebundlestate.go1
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/controller.go32
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/csr_controller.go183
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go44
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/helpers.go9
7 files changed, 288 insertions, 11 deletions
diff --git a/src/monitor/deploy/cluster_role.yaml b/src/monitor/deploy/cluster_role.yaml
index 0732e8d3..9330a687 100644
--- a/src/monitor/deploy/cluster_role.yaml
+++ b/src/monitor/deploy/cluster_role.yaml
@@ -70,3 +70,9 @@ rules:
- '*'
verbs:
- '*'
+- apiGroups:
+ - certificates.k8s.io
+ resources:
+ - '*'
+ verbs:
+ - '*'
diff --git a/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go b/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
index 6476f0db..fba10bb3 100644
--- a/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
+++ b/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
@@ -3,6 +3,7 @@ package v1alpha1
import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/batch/v1"
+ certsapi "k8s.io/api/certificates/v1beta1"
corev1 "k8s.io/api/core/v1"
v1beta1 "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,17 +32,18 @@ type ResourceBundleStateSpec struct {
// ResourceBundleStatus defines the observed state of ResourceBundleState
// +k8s:openapi-gen=true
type ResourceBundleStatus struct {
- Ready bool `json:"ready" protobuf:"varint,1,opt,name=ready"`
- ResourceCount int32 `json:"resourceCount" protobuf:"varint,2,opt,name=resourceCount"`
- PodStatuses []corev1.Pod `json:"podStatuses" protobuf:"varint,3,opt,name=podStatuses"`
- ServiceStatuses []corev1.Service `json:"serviceStatuses" protobuf:"varint,4,opt,name=serviceStatuses"`
- ConfigMapStatuses []corev1.ConfigMap `json:"configMapStatuses" protobuf:"varint,5,opt,name=configMapStatuses"`
- DeploymentStatuses []appsv1.Deployment `json:"deploymentStatuses" protobuf:"varint,6,opt,name=deploymentStatuses"`
- SecretStatuses []corev1.Secret `json:"secretStatuses" protobuf:"varint,7,opt,name=secretStatuses"`
- DaemonSetStatuses []appsv1.DaemonSet `json:"daemonSetStatuses" protobuf:"varint,8,opt,name=daemonSetStatuses"`
- IngressStatuses []v1beta1.Ingress `json:"ingressStatuses" protobuf:"varint,11,opt,name=ingressStatuses"`
- JobStatuses []v1.Job `json:"jobStatuses" protobuf:"varint,12,opt,name=jobStatuses"`
- StatefulSetStatuses []appsv1.StatefulSet `json:"statefulSetStatuses" protobuf:"varint,13,opt,name=statefulSetStatuses"`
+ Ready bool `json:"ready" protobuf:"varint,1,opt,name=ready"`
+ ResourceCount int32 `json:"resourceCount" protobuf:"varint,2,opt,name=resourceCount"`
+ PodStatuses []corev1.Pod `json:"podStatuses" protobuf:"varint,3,opt,name=podStatuses"`
+ ServiceStatuses []corev1.Service `json:"serviceStatuses" protobuf:"varint,4,opt,name=serviceStatuses"`
+ ConfigMapStatuses []corev1.ConfigMap `json:"configMapStatuses" protobuf:"varint,5,opt,name=configMapStatuses"`
+ DeploymentStatuses []appsv1.Deployment `json:"deploymentStatuses" protobuf:"varint,6,opt,name=deploymentStatuses"`
+ SecretStatuses []corev1.Secret `json:"secretStatuses" protobuf:"varint,7,opt,name=secretStatuses"`
+ DaemonSetStatuses []appsv1.DaemonSet `json:"daemonSetStatuses" protobuf:"varint,8,opt,name=daemonSetStatuses"`
+ IngressStatuses []v1beta1.Ingress `json:"ingressStatuses" protobuf:"varint,11,opt,name=ingressStatuses"`
+ JobStatuses []v1.Job `json:"jobStatuses" protobuf:"varint,12,opt,name=jobStatuses"`
+ StatefulSetStatuses []appsv1.StatefulSet `json:"statefulSetStatuses" protobuf:"varint,13,opt,name=statefulSetStatuses"`
+ CsrStatuses []certsapi.CertificateSigningRequest `json:"csrStatuses" protobuf:"varint,3,opt,name=csrStatuses"`
}
// PodStatus defines the observed state of ResourceBundleState
diff --git a/src/monitor/pkg/controller/add_resourcebundlestate.go b/src/monitor/pkg/controller/add_resourcebundlestate.go
index ee42f9cf..d06e97d6 100644
--- a/src/monitor/pkg/controller/add_resourcebundlestate.go
+++ b/src/monitor/pkg/controller/add_resourcebundlestate.go
@@ -15,4 +15,5 @@ func init() {
AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddIngressController)
AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddJobController)
AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddStatefulSetController)
+ AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddCsrController)
}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/controller.go b/src/monitor/pkg/controller/resourcebundlestate/controller.go
index faee5892..5351ea99 100644
--- a/src/monitor/pkg/controller/resourcebundlestate/controller.go
+++ b/src/monitor/pkg/controller/resourcebundlestate/controller.go
@@ -8,6 +8,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/batch/v1"
+ certsapi "k8s.io/api/certificates/v1beta1"
corev1 "k8s.io/api/core/v1"
v1beta1 "k8s.io/api/extensions/v1beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -119,6 +120,12 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error)
return reconcile.Result{}, err
}
+ err = r.updateCsrs(rbstate, rbstate.Spec.Selector.MatchLabels)
+ if err != nil {
+ log.Printf("Error adding csrStatuses: %v\n", err)
+ return reconcile.Result{}, err
+ }
+
// TODO: Update this based on the statuses of the lower resources
rbstate.Status.Ready = false
err = r.client.Status().Update(context.TODO(), rbstate)
@@ -352,3 +359,28 @@ func (r *reconciler) updateStatefulSets(rbstate *v1alpha1.ResourceBundleState,
return nil
}
+
+func (r *reconciler) updateCsrs(rbstate *v1alpha1.ResourceBundleState,
+ selectors map[string]string) error {
+
+ // Update the CR with the csrs tracked
+ csrList := &certsapi.CertificateSigningRequestList{}
+ err := listResources(r.client, rbstate.Namespace, selectors, csrList)
+ if err != nil {
+ log.Printf("Failed to list csrs: %v", err)
+ return err
+ }
+
+ rbstate.Status.CsrStatuses = []certsapi.CertificateSigningRequest{}
+
+ for _, csr := range csrList.Items {
+ resStatus := certsapi.CertificateSigningRequest{
+ TypeMeta: csr.TypeMeta,
+ ObjectMeta: csr.ObjectMeta,
+ Status: csr.Status,
+ }
+ rbstate.Status.CsrStatuses = append(rbstate.Status.CsrStatuses, resStatus)
+ }
+
+ return nil
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go b/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go
new file mode 100644
index 00000000..918fadfb
--- /dev/null
+++ b/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go
@@ -0,0 +1,183 @@
+package resourcebundlestate
+
+import (
+ "context"
+ "log"
+
+ "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
+
+ certsapi "k8s.io/api/certificates/v1beta1"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/controller"
+ "sigs.k8s.io/controller-runtime/pkg/handler"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+ "sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+// AddCsrController the new controller to the controller manager
+func AddCsrController(mgr manager.Manager) error {
+ return addCsrController(mgr, newCsrReconciler(mgr))
+}
+
+func addCsrController(mgr manager.Manager, r *csrReconciler) error {
+ // Create a new controller
+ c, err := controller.New("Csr-controller", mgr, controller.Options{Reconciler: r})
+ if err != nil {
+ return err
+ }
+
+ // Watch for changes to secondary resource Csrs
+ // Predicate filters csrs which don't have the k8splugin label
+ err = c.Watch(&source.Kind{Type: &certsapi.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}, &csrPredicate{})
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func newCsrReconciler(m manager.Manager) *csrReconciler {
+ return &csrReconciler{client: m.GetClient()}
+}
+
+type csrReconciler struct {
+ client client.Client
+}
+
+// Reconcile implements the loop that will update the ResourceBundleState CR
+// whenever we get any updates from all the csrs we watch.
+func (r *csrReconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) {
+ log.Printf("Updating ResourceBundleState for Csr: %+v\n", req)
+
+ csr := &certsapi.CertificateSigningRequest{}
+ err := r.client.Get(context.TODO(), req.NamespacedName, csr)
+ if err != nil {
+ if k8serrors.IsNotFound(err) {
+ log.Printf("Csr not found: %+v. Remove from CR if it is stored there.\n", req.NamespacedName)
+ // Remove the Csr's status from StatusList
+ // This can happen if we get the DeletionTimeStamp event
+ // after the POD has been deleted.
+ r.deleteCsrFromAllCRs(req.NamespacedName)
+ return reconcile.Result{}, nil
+ }
+ log.Printf("Failed to get csr: %+v\n", req.NamespacedName)
+ return reconcile.Result{}, err
+ }
+
+ // Find the CRs which track this csr via the labelselector
+ crSelector := returnLabel(csr.GetLabels())
+ if crSelector == nil {
+ log.Println("We should not be here. The predicate should have filtered this Csr")
+ }
+
+ // Get the CRs which have this label and update them all
+ // Ideally, we will have only one CR, but there is nothing
+ // preventing the creation of multiple.
+ // TODO: Consider using an admission validating webook to prevent multiple
+ rbStatusList := &v1alpha1.ResourceBundleStateList{}
+ err = listClusterResources(r.client, crSelector, rbStatusList)
+ if err != nil || len(rbStatusList.Items) == 0 {
+ log.Printf("Did not find any CRs tracking this resource\n")
+ return reconcile.Result{}, nil
+ }
+
+ err = r.updateCRs(rbStatusList, csr)
+ if err != nil {
+ // Requeue the update
+ return reconcile.Result{}, err
+ }
+
+ return reconcile.Result{}, nil
+}
+
+// deleteCsrFromAllCRs deletes csr status from all the CRs when the POD itself has been deleted
+// and we have not handled the updateCRs yet.
+// Since, we don't have the csr's labels, we need to look at all the CRs in this namespace
+func (r *csrReconciler) deleteCsrFromAllCRs(namespacedName types.NamespacedName) error {
+
+ rbStatusList := &v1alpha1.ResourceBundleStateList{}
+ err := listClusterResources(r.client, nil, rbStatusList)
+ if err != nil || len(rbStatusList.Items) == 0 {
+ log.Printf("Did not find any CRs tracking this resource\n")
+ return nil
+ }
+ for _, cr := range rbStatusList.Items {
+ r.deleteFromSingleCR(&cr, namespacedName.Name)
+ }
+
+ return nil
+}
+
+func (r *csrReconciler) updateCRs(crl *v1alpha1.ResourceBundleStateList, csr *certsapi.CertificateSigningRequest) error {
+
+ for _, cr := range crl.Items {
+ // Csr is not scheduled for deletion
+ if csr.DeletionTimestamp == nil {
+ err := r.updateSingleCR(&cr, csr)
+ if err != nil {
+ return err
+ }
+ } else {
+ // Csr is scheduled for deletion
+ r.deleteFromSingleCR(&cr, csr.Name)
+ }
+ }
+
+ return nil
+}
+
+func (r *csrReconciler) deleteFromSingleCR(cr *v1alpha1.ResourceBundleState, name string) error {
+ cr.Status.ResourceCount--
+ length := len(cr.Status.CsrStatuses)
+ for i, rstatus := range cr.Status.CsrStatuses {
+ if rstatus.Name == name {
+ //Delete that status from the array
+ cr.Status.CsrStatuses[i] = cr.Status.CsrStatuses[length-1]
+ cr.Status.CsrStatuses[length-1] = certsapi.CertificateSigningRequest{}
+ cr.Status.CsrStatuses = cr.Status.CsrStatuses[:length-1]
+ return nil
+ }
+ }
+
+ log.Println("Did not find a status for POD in CR")
+ return nil
+}
+
+func (r *csrReconciler) updateSingleCR(cr *v1alpha1.ResourceBundleState, csr *certsapi.CertificateSigningRequest) error {
+
+ // Update status after searching for it in the list of resourceStatuses
+ for i, rstatus := range cr.Status.CsrStatuses {
+ // Look for the status if we already have it in the CR
+ if rstatus.Name == csr.Name {
+ csr.Status.DeepCopyInto(&cr.Status.CsrStatuses[i].Status)
+ err := r.client.Status().Update(context.TODO(), cr)
+ if err != nil {
+ log.Printf("failed to update rbstate: %v\n", err)
+ return err
+ }
+ return nil
+ }
+ }
+
+ // Exited for loop with no status found
+ // Increment the number of tracked resources
+ cr.Status.ResourceCount++
+
+ // Add it to CR
+ cr.Status.CsrStatuses = append(cr.Status.CsrStatuses, certsapi.CertificateSigningRequest{
+ TypeMeta: csr.TypeMeta,
+ ObjectMeta: csr.ObjectMeta,
+ Status: csr.Status,
+ })
+
+ err := r.client.Status().Update(context.TODO(), cr)
+ if err != nil {
+ log.Printf("failed to update rbstate: %v\n", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go b/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go
new file mode 100644
index 00000000..cb83eec1
--- /dev/null
+++ b/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go
@@ -0,0 +1,44 @@
+package resourcebundlestate
+
+import (
+ "sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type csrPredicate struct {
+}
+
+func (p *csrPredicate) Create(evt event.CreateEvent) bool {
+
+ if evt.Meta == nil {
+ return false
+ }
+
+ labels := evt.Meta.GetLabels()
+ return checkLabel(labels)
+}
+
+func (p *csrPredicate) Delete(evt event.DeleteEvent) bool {
+
+ if evt.Meta == nil {
+ return false
+ }
+
+ labels := evt.Meta.GetLabels()
+ return checkLabel(labels)
+}
+
+func (p *csrPredicate) Update(evt event.UpdateEvent) bool {
+
+ if evt.MetaNew == nil {
+ return false
+ }
+
+ labels := evt.MetaNew.GetLabels()
+ return checkLabel(labels)
+}
+
+func (p *csrPredicate) Generic(evt event.GenericEvent) bool {
+
+ labels := evt.Meta.GetLabels()
+ return checkLabel(labels)
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/helpers.go b/src/monitor/pkg/controller/resourcebundlestate/helpers.go
index 9a6538b4..4d6be370 100644
--- a/src/monitor/pkg/controller/resourcebundlestate/helpers.go
+++ b/src/monitor/pkg/controller/resourcebundlestate/helpers.go
@@ -52,3 +52,12 @@ func listResources(cli client.Client, namespace string,
return nil
}
+
+// listClusterResources lists non-namespace resources based
+// on the selectors provided.
+// The data is returned in the pointer to the runtime.Object
+// provided as argument.
+func listClusterResources(cli client.Client,
+ labelSelector map[string]string, returnData runtime.Object) error {
+ return listResources(cli, "", labelSelector, returnData)
+}