summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIgor D.C <igor.duarte.cardoso@intel.com>2020-08-31 22:46:10 +0000
committerIgor D.C <igor.duarte.cardoso@intel.com>2020-09-25 18:43:36 +0000
commit425795c7d4e6ce81932918aca2a1462384d4507f (patch)
tree68f9793b29b3d282e62426ab52669319b4dfd4eb
parent8b5c4236639a46f39cbfe852590f34e64f58a85a (diff)
Introduce Monitor support for CSR resource
These changes allow the Monitor to also track CSR (CertificateSigningResource) resources which will make it possible to know when a certificate has been issued by the K8s cluster signer. In turn, DCM will be able to read, store and use that certificate to generate kubeconfigs. Out-of-tree actions required: - publish monitor's docker image built from this source onto emcov2/monitor:latest Issue-ID: MULTICLOUD-1143 Change-Id: I7facd27bbfe08891151bb3b6a9a19948435e24e4 Signed-off-by: Igor D.C <igor.duarte.cardoso@intel.com>
-rw-r--r--src/monitor/deploy/cluster_role.yaml6
-rw-r--r--src/monitor/pkg/apis/k8splugin/v1alpha1/types.go24
-rw-r--r--src/monitor/pkg/controller/add_resourcebundlestate.go1
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/controller.go32
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/csr_controller.go183
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go44
-rw-r--r--src/monitor/pkg/controller/resourcebundlestate/helpers.go9
7 files changed, 288 insertions, 11 deletions
diff --git a/src/monitor/deploy/cluster_role.yaml b/src/monitor/deploy/cluster_role.yaml
index 0732e8d3..9330a687 100644
--- a/src/monitor/deploy/cluster_role.yaml
+++ b/src/monitor/deploy/cluster_role.yaml
@@ -70,3 +70,9 @@ rules:
- '*'
verbs:
- '*'
+- apiGroups:
+ - certificates.k8s.io
+ resources:
+ - '*'
+ verbs:
+ - '*'
diff --git a/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go b/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
index 6476f0db..fba10bb3 100644
--- a/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
+++ b/src/monitor/pkg/apis/k8splugin/v1alpha1/types.go
@@ -3,6 +3,7 @@ package v1alpha1
import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/batch/v1"
+ certsapi "k8s.io/api/certificates/v1beta1"
corev1 "k8s.io/api/core/v1"
v1beta1 "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,17 +32,18 @@ type ResourceBundleStateSpec struct {
// ResourceBundleStatus defines the observed state of ResourceBundleState
// +k8s:openapi-gen=true
type ResourceBundleStatus struct {
- Ready bool `json:"ready" protobuf:"varint,1,opt,name=ready"`
- ResourceCount int32 `json:"resourceCount" protobuf:"varint,2,opt,name=resourceCount"`
- PodStatuses []corev1.Pod `json:"podStatuses" protobuf:"varint,3,opt,name=podStatuses"`
- ServiceStatuses []corev1.Service `json:"serviceStatuses" protobuf:"varint,4,opt,name=serviceStatuses"`
- ConfigMapStatuses []corev1.ConfigMap `json:"configMapStatuses" protobuf:"varint,5,opt,name=configMapStatuses"`
- DeploymentStatuses []appsv1.Deployment `json:"deploymentStatuses" protobuf:"varint,6,opt,name=deploymentStatuses"`
- SecretStatuses []corev1.Secret `json:"secretStatuses" protobuf:"varint,7,opt,name=secretStatuses"`
- DaemonSetStatuses []appsv1.DaemonSet `json:"daemonSetStatuses" protobuf:"varint,8,opt,name=daemonSetStatuses"`
- IngressStatuses []v1beta1.Ingress `json:"ingressStatuses" protobuf:"varint,11,opt,name=ingressStatuses"`
- JobStatuses []v1.Job `json:"jobStatuses" protobuf:"varint,12,opt,name=jobStatuses"`
- StatefulSetStatuses []appsv1.StatefulSet `json:"statefulSetStatuses" protobuf:"varint,13,opt,name=statefulSetStatuses"`
+ Ready bool `json:"ready" protobuf:"varint,1,opt,name=ready"`
+ ResourceCount int32 `json:"resourceCount" protobuf:"varint,2,opt,name=resourceCount"`
+ PodStatuses []corev1.Pod `json:"podStatuses" protobuf:"varint,3,opt,name=podStatuses"`
+ ServiceStatuses []corev1.Service `json:"serviceStatuses" protobuf:"varint,4,opt,name=serviceStatuses"`
+ ConfigMapStatuses []corev1.ConfigMap `json:"configMapStatuses" protobuf:"varint,5,opt,name=configMapStatuses"`
+ DeploymentStatuses []appsv1.Deployment `json:"deploymentStatuses" protobuf:"varint,6,opt,name=deploymentStatuses"`
+ SecretStatuses []corev1.Secret `json:"secretStatuses" protobuf:"varint,7,opt,name=secretStatuses"`
+ DaemonSetStatuses []appsv1.DaemonSet `json:"daemonSetStatuses" protobuf:"varint,8,opt,name=daemonSetStatuses"`
+ IngressStatuses []v1beta1.Ingress `json:"ingressStatuses" protobuf:"varint,11,opt,name=ingressStatuses"`
+ JobStatuses []v1.Job `json:"jobStatuses" protobuf:"varint,12,opt,name=jobStatuses"`
+ StatefulSetStatuses []appsv1.StatefulSet `json:"statefulSetStatuses" protobuf:"varint,13,opt,name=statefulSetStatuses"`
+ CsrStatuses []certsapi.CertificateSigningRequest `json:"csrStatuses" protobuf:"varint,3,opt,name=csrStatuses"`
}
// PodStatus defines the observed state of ResourceBundleState
diff --git a/src/monitor/pkg/controller/add_resourcebundlestate.go b/src/monitor/pkg/controller/add_resourcebundlestate.go
index ee42f9cf..d06e97d6 100644
--- a/src/monitor/pkg/controller/add_resourcebundlestate.go
+++ b/src/monitor/pkg/controller/add_resourcebundlestate.go
@@ -15,4 +15,5 @@ func init() {
AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddIngressController)
AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddJobController)
AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddStatefulSetController)
+ AddToManagerFuncs = append(AddToManagerFuncs, resourcebundlestate.AddCsrController)
}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/controller.go b/src/monitor/pkg/controller/resourcebundlestate/controller.go
index faee5892..5351ea99 100644
--- a/src/monitor/pkg/controller/resourcebundlestate/controller.go
+++ b/src/monitor/pkg/controller/resourcebundlestate/controller.go
@@ -8,6 +8,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/batch/v1"
+ certsapi "k8s.io/api/certificates/v1beta1"
corev1 "k8s.io/api/core/v1"
v1beta1 "k8s.io/api/extensions/v1beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -119,6 +120,12 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error)
return reconcile.Result{}, err
}
+ err = r.updateCsrs(rbstate, rbstate.Spec.Selector.MatchLabels)
+ if err != nil {
+ log.Printf("Error adding csrStatuses: %v\n", err)
+ return reconcile.Result{}, err
+ }
+
// TODO: Update this based on the statuses of the lower resources
rbstate.Status.Ready = false
err = r.client.Status().Update(context.TODO(), rbstate)
@@ -352,3 +359,28 @@ func (r *reconciler) updateStatefulSets(rbstate *v1alpha1.ResourceBundleState,
return nil
}
+
+func (r *reconciler) updateCsrs(rbstate *v1alpha1.ResourceBundleState,
+ selectors map[string]string) error {
+
+ // Update the CR with the csrs tracked
+ csrList := &certsapi.CertificateSigningRequestList{}
+ err := listResources(r.client, rbstate.Namespace, selectors, csrList)
+ if err != nil {
+ log.Printf("Failed to list csrs: %v", err)
+ return err
+ }
+
+ rbstate.Status.CsrStatuses = []certsapi.CertificateSigningRequest{}
+
+ for _, csr := range csrList.Items {
+ resStatus := certsapi.CertificateSigningRequest{
+ TypeMeta: csr.TypeMeta,
+ ObjectMeta: csr.ObjectMeta,
+ Status: csr.Status,
+ }
+ rbstate.Status.CsrStatuses = append(rbstate.Status.CsrStatuses, resStatus)
+ }
+
+ return nil
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go b/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go
new file mode 100644
index 00000000..918fadfb
--- /dev/null
+++ b/src/monitor/pkg/controller/resourcebundlestate/csr_controller.go
@@ -0,0 +1,183 @@
+package resourcebundlestate
+
+import (
+ "context"
+ "log"
+
+ "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1"
+
+ certsapi "k8s.io/api/certificates/v1beta1"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/controller"
+ "sigs.k8s.io/controller-runtime/pkg/handler"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+ "sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+// AddCsrController the new controller to the controller manager
+func AddCsrController(mgr manager.Manager) error {
+ return addCsrController(mgr, newCsrReconciler(mgr))
+}
+
+func addCsrController(mgr manager.Manager, r *csrReconciler) error {
+ // Create a new controller
+ c, err := controller.New("Csr-controller", mgr, controller.Options{Reconciler: r})
+ if err != nil {
+ return err
+ }
+
+ // Watch for changes to secondary resource Csrs
+ // Predicate filters csrs which don't have the k8splugin label
+ err = c.Watch(&source.Kind{Type: &certsapi.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}, &csrPredicate{})
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func newCsrReconciler(m manager.Manager) *csrReconciler {
+ return &csrReconciler{client: m.GetClient()}
+}
+
+type csrReconciler struct {
+ client client.Client
+}
+
+// Reconcile implements the loop that will update the ResourceBundleState CR
+// whenever we get any updates from all the csrs we watch.
+func (r *csrReconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) {
+ log.Printf("Updating ResourceBundleState for Csr: %+v\n", req)
+
+ csr := &certsapi.CertificateSigningRequest{}
+ err := r.client.Get(context.TODO(), req.NamespacedName, csr)
+ if err != nil {
+ if k8serrors.IsNotFound(err) {
+ log.Printf("Csr not found: %+v. Remove from CR if it is stored there.\n", req.NamespacedName)
+ // Remove the Csr's status from StatusList
+ // This can happen if we get the DeletionTimeStamp event
+ // after the POD has been deleted.
+ r.deleteCsrFromAllCRs(req.NamespacedName)
+ return reconcile.Result{}, nil
+ }
+ log.Printf("Failed to get csr: %+v\n", req.NamespacedName)
+ return reconcile.Result{}, err
+ }
+
+ // Find the CRs which track this csr via the labelselector
+ crSelector := returnLabel(csr.GetLabels())
+ if crSelector == nil {
+ log.Println("We should not be here. The predicate should have filtered this Csr")
+ }
+
+ // Get the CRs which have this label and update them all
+ // Ideally, we will have only one CR, but there is nothing
+ // preventing the creation of multiple.
+ // TODO: Consider using an admission validating webook to prevent multiple
+ rbStatusList := &v1alpha1.ResourceBundleStateList{}
+ err = listClusterResources(r.client, crSelector, rbStatusList)
+ if err != nil || len(rbStatusList.Items) == 0 {
+ log.Printf("Did not find any CRs tracking this resource\n")
+ return reconcile.Result{}, nil
+ }
+
+ err = r.updateCRs(rbStatusList, csr)
+ if err != nil {
+ // Requeue the update
+ return reconcile.Result{}, err
+ }
+
+ return reconcile.Result{}, nil
+}
+
+// deleteCsrFromAllCRs deletes csr status from all the CRs when the POD itself has been deleted
+// and we have not handled the updateCRs yet.
+// Since, we don't have the csr's labels, we need to look at all the CRs in this namespace
+func (r *csrReconciler) deleteCsrFromAllCRs(namespacedName types.NamespacedName) error {
+
+ rbStatusList := &v1alpha1.ResourceBundleStateList{}
+ err := listClusterResources(r.client, nil, rbStatusList)
+ if err != nil || len(rbStatusList.Items) == 0 {
+ log.Printf("Did not find any CRs tracking this resource\n")
+ return nil
+ }
+ for _, cr := range rbStatusList.Items {
+ r.deleteFromSingleCR(&cr, namespacedName.Name)
+ }
+
+ return nil
+}
+
+func (r *csrReconciler) updateCRs(crl *v1alpha1.ResourceBundleStateList, csr *certsapi.CertificateSigningRequest) error {
+
+ for _, cr := range crl.Items {
+ // Csr is not scheduled for deletion
+ if csr.DeletionTimestamp == nil {
+ err := r.updateSingleCR(&cr, csr)
+ if err != nil {
+ return err
+ }
+ } else {
+ // Csr is scheduled for deletion
+ r.deleteFromSingleCR(&cr, csr.Name)
+ }
+ }
+
+ return nil
+}
+
+func (r *csrReconciler) deleteFromSingleCR(cr *v1alpha1.ResourceBundleState, name string) error {
+ cr.Status.ResourceCount--
+ length := len(cr.Status.CsrStatuses)
+ for i, rstatus := range cr.Status.CsrStatuses {
+ if rstatus.Name == name {
+ //Delete that status from the array
+ cr.Status.CsrStatuses[i] = cr.Status.CsrStatuses[length-1]
+ cr.Status.CsrStatuses[length-1] = certsapi.CertificateSigningRequest{}
+ cr.Status.CsrStatuses = cr.Status.CsrStatuses[:length-1]
+ return nil
+ }
+ }
+
+ log.Println("Did not find a status for POD in CR")
+ return nil
+}
+
+func (r *csrReconciler) updateSingleCR(cr *v1alpha1.ResourceBundleState, csr *certsapi.CertificateSigningRequest) error {
+
+ // Update status after searching for it in the list of resourceStatuses
+ for i, rstatus := range cr.Status.CsrStatuses {
+ // Look for the status if we already have it in the CR
+ if rstatus.Name == csr.Name {
+ csr.Status.DeepCopyInto(&cr.Status.CsrStatuses[i].Status)
+ err := r.client.Status().Update(context.TODO(), cr)
+ if err != nil {
+ log.Printf("failed to update rbstate: %v\n", err)
+ return err
+ }
+ return nil
+ }
+ }
+
+ // Exited for loop with no status found
+ // Increment the number of tracked resources
+ cr.Status.ResourceCount++
+
+ // Add it to CR
+ cr.Status.CsrStatuses = append(cr.Status.CsrStatuses, certsapi.CertificateSigningRequest{
+ TypeMeta: csr.TypeMeta,
+ ObjectMeta: csr.ObjectMeta,
+ Status: csr.Status,
+ })
+
+ err := r.client.Status().Update(context.TODO(), cr)
+ if err != nil {
+ log.Printf("failed to update rbstate: %v\n", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go b/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go
new file mode 100644
index 00000000..cb83eec1
--- /dev/null
+++ b/src/monitor/pkg/controller/resourcebundlestate/csr_predicate.go
@@ -0,0 +1,44 @@
+package resourcebundlestate
+
+import (
+ "sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type csrPredicate struct {
+}
+
+func (p *csrPredicate) Create(evt event.CreateEvent) bool {
+
+ if evt.Meta == nil {
+ return false
+ }
+
+ labels := evt.Meta.GetLabels()
+ return checkLabel(labels)
+}
+
+func (p *csrPredicate) Delete(evt event.DeleteEvent) bool {
+
+ if evt.Meta == nil {
+ return false
+ }
+
+ labels := evt.Meta.GetLabels()
+ return checkLabel(labels)
+}
+
+func (p *csrPredicate) Update(evt event.UpdateEvent) bool {
+
+ if evt.MetaNew == nil {
+ return false
+ }
+
+ labels := evt.MetaNew.GetLabels()
+ return checkLabel(labels)
+}
+
+func (p *csrPredicate) Generic(evt event.GenericEvent) bool {
+
+ labels := evt.Meta.GetLabels()
+ return checkLabel(labels)
+}
diff --git a/src/monitor/pkg/controller/resourcebundlestate/helpers.go b/src/monitor/pkg/controller/resourcebundlestate/helpers.go
index 9a6538b4..4d6be370 100644
--- a/src/monitor/pkg/controller/resourcebundlestate/helpers.go
+++ b/src/monitor/pkg/controller/resourcebundlestate/helpers.go
@@ -52,3 +52,12 @@ func listResources(cli client.Client, namespace string,
return nil
}
+
+// listClusterResources lists non-namespace resources based
+// on the selectors provided.
+// The data is returned in the pointer to the runtime.Object
+// provided as argument.
+func listClusterResources(cli client.Client,
+ labelSelector map[string]string, returnData runtime.Object) error {
+ return listResources(cli, "", labelSelector, returnData)
+}