From 8a25158d77311eec27d1fb3dc41e16bfbfceebcc Mon Sep 17 00:00:00 2001 From: "Igor D.C" Date: Fri, 25 Sep 2020 05:28:09 +0000 Subject: Implement Kubeconfig endpoint in DCM The /kubeconfig API path allows a client to retrieve a kubeconfig file for a specified cluster reference of a logical cloud. - includes CA cert, address, user private key and signed cert. This commit includes the "lazy-loading" implementation of certificate retrieval per cluster from Rsync (which happens when clients call). The certificate is read from the cluster status in appcontext. Thus, Monitor and Rsync need to be configured and running. Issue-ID: MULTICLOUD-1143 Change-Id: Ie94cd128e14c8a944861eced2bdc886d95fab6ed Signed-off-by: Igor D.C --- src/dcm/api/api.go | 15 +-- src/dcm/api/clusterHandler.go | 41 +++++++ src/dcm/go.mod | 2 + src/dcm/go.sum | 2 + src/dcm/pkg/module/cluster.go | 214 +++++++++++++++++++++++++++++++++++++ src/dcm/pkg/module/logicalcloud.go | 7 +- 6 files changed, 267 insertions(+), 14 deletions(-) diff --git a/src/dcm/api/api.go b/src/dcm/api/api.go index 0f68a517..cd8589dd 100644 --- a/src/dcm/api/api.go +++ b/src/dcm/api/api.go @@ -71,18 +71,8 @@ func NewRouter( lcRouter.HandleFunc( "/logical-clouds/{logical-cloud-name}/terminate", logicalCloudHandler.terminateHandler).Methods("POST") - // To Do - // get kubeconfig - /*lcRouter.HandleFunc( - "/logical-clouds/{name}/kubeconfig?cluster-reference={cluster}", - logicalCloudHandler.getConfigHandler).Methods("GET") - //get status - lcRouter.HandleFunc( - "/logical-clouds/{name}/cluster-references/", - logicalCloudHandler.associateHandler).Methods("GET")*/ // Set up Cluster API - clusterHandler := clusterHandler{client: clusterClient} clusterRouter := router.PathPrefix("/v2/projects/{project-name}").Subrouter() clusterRouter.HandleFunc( @@ -100,6 +90,10 @@ func NewRouter( clusterRouter.HandleFunc( "/logical-clouds/{logical-cloud-name}/cluster-references/{cluster-reference}", clusterHandler.deleteHandler).Methods("DELETE") + // Get kubeconfig for cluster of logical cloud + clusterRouter.HandleFunc( + "/logical-clouds/{logical-cloud-name}/cluster-references/{cluster-reference}/kubeconfig", + clusterHandler.getConfigHandler).Methods("GET") // Set up User Permission API if userPermissionClient == nil { @@ -121,7 +115,6 @@ func NewRouter( userPermissionHandler.deleteHandler).Methods("DELETE") // Set up Quota API - quotaHandler := quotaHandler{client: quotaClient} quotaRouter := router.PathPrefix("/v2/projects/{project-name}").Subrouter() quotaRouter.HandleFunc( diff --git a/src/dcm/api/clusterHandler.go b/src/dcm/api/clusterHandler.go index d0c1e62c..db110399 100644 --- a/src/dcm/api/clusterHandler.go +++ b/src/dcm/api/clusterHandler.go @@ -168,3 +168,44 @@ func (h clusterHandler) deleteHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } + +// getConfigHandler handles GET operations on kubeconfigs +// Returns a kubeconfig file +func (h clusterHandler) getConfigHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + project := vars["project-name"] + logicalCloud := vars["logical-cloud-name"] + name := vars["cluster-reference"] + var ret interface{} + var err error + + ret, err = h.client.GetCluster(project, logicalCloud, name) + if err != nil { + if err.Error() == "Cluster Reference does not exist" { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + ret, err = h.client.GetClusterConfig(project, logicalCloud, name) + if err != nil { + if err.Error() == "The certificate for this cluster hasn't been issued yet. Please try later." { + http.Error(w, err.Error(), http.StatusAccepted) + } else if err.Error() == "Logical Cloud hasn't been applied yet" { + http.Error(w, err.Error(), http.StatusBadRequest) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + w.Header().Set("Content-Type", "application/yaml") + w.WriteHeader(http.StatusOK) + err = json.NewEncoder(w).Encode(ret) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} diff --git a/src/dcm/go.mod b/src/dcm/go.mod index 1f04ac12..71888287 100644 --- a/src/dcm/go.mod +++ b/src/dcm/go.mod @@ -3,6 +3,8 @@ module github.com/onap/multicloud-k8s/src/dcm require ( github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.3 + github.com/onap/multicloud-k8s/src/clm v0.0.0-20200630152613-7c20f73e7c5d + github.com/onap/multicloud-k8s/src/monitor v0.0.0-20200818155723-a5ffa8aadf49 github.com/onap/multicloud-k8s/src/orchestrator v0.0.0-20200818155723-a5ffa8aadf49 github.com/pkg/errors v0.9.1 github.com/russross/blackfriday/v2 v2.0.1 diff --git a/src/dcm/go.sum b/src/dcm/go.sum index 983ceae2..ad36ad8d 100644 --- a/src/dcm/go.sum +++ b/src/dcm/go.sum @@ -1522,6 +1522,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.16.9 h1:3vCx0WX9qcg1Hv4aQ/G1tiIKectGVuimvPVTJU4VOCA= k8s.io/api v0.16.9/go.mod h1:Y7dZNHs1Xy0mSwSlzL9QShi6qkljnN41yR8oWCRTDe8= +k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= +k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/apiextensions-apiserver v0.16.9 h1:CE+SWS6PM3MDJiyihW5hnDiqsJ/sjMaSMblqzH37J18= k8s.io/apiextensions-apiserver v0.16.9/go.mod h1:j/+KedxOeRSPMkvLNyKMbIT3+saXdTO4jTBplTmXJR4= k8s.io/apimachinery v0.16.10-beta.0 h1:l+qmzwWTMIBtFGlo5OpPYoZKCgGLtpAWvIa8Wcr9luU= diff --git a/src/dcm/pkg/module/cluster.go b/src/dcm/pkg/module/cluster.go index 85b20117..6cb18539 100644 --- a/src/dcm/pkg/module/cluster.go +++ b/src/dcm/pkg/module/cluster.go @@ -17,7 +17,15 @@ package module import ( + "encoding/base64" + "encoding/json" + "strings" + + clm "github.com/onap/multicloud-k8s/src/clm/pkg/cluster" + rb "github.com/onap/multicloud-k8s/src/monitor/pkg/apis/k8splugin/v1alpha1" + log "github.com/onap/multicloud-k8s/src/orchestrator/pkg/infra/logutils" pkgerrors "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) // Cluster contains the parameters needed for a Cluster @@ -37,6 +45,7 @@ type ClusterSpec struct { ClusterProvider string `json:"cluster-provider"` ClusterName string `json:"cluster-name"` LoadBalancerIP string `json:"loadbalancer-ip"` + Certificate string `json:"certificate"` } type ClusterKey struct { @@ -45,6 +54,48 @@ type ClusterKey struct { ClusterReference string `json:"clname"` } +type KubeConfig struct { + ApiVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Clusters []KubeCluster `yaml:"clusters"` + Contexts []KubeContext `yaml:"contexts"` + CurrentContext string `yaml:"current-context` + Preferences map[string]string `yaml:"preferences"` + Users []KubeUser `yaml:"users"` +} + +type KubeCluster struct { + ClusterDef KubeClusterDef `yaml:"cluster"` + ClusterName string `yaml:"name"` +} + +type KubeClusterDef struct { + CertificateAuthorityData string `yaml:"certificate-authority-data"` + Server string `yaml:"server"` +} + +type KubeContext struct { + ContextDef KubeContextDef `yaml:"context"` + ContextName string `yaml:"name"` +} + +type KubeContextDef struct { + Cluster string `yaml:"cluster"` + Namespace string `yaml:"namespace,omitempty"` + User string `yaml:"user"` +} + +type KubeUser struct { + UserName string `yaml:"name"` + UserDef KubeUserDef `yaml:"user"` +} + +type KubeUserDef struct { + ClientCertificateData string `yaml:"client-certificate-data"` + ClientKeyData string `yaml:"client-key-data"` + // client-certificate and client-key are NOT implemented +} + // ClusterManager is an interface that exposes the connection // functionality type ClusterManager interface { @@ -53,6 +104,7 @@ type ClusterManager interface { GetAllClusters(project, logicalCloud string) ([]Cluster, error) DeleteCluster(project, logicalCloud, name string) error UpdateCluster(project, logicalCloud, name string, c Cluster) (Cluster, error) + GetClusterConfig(project, logicalcloud, name string) (string, error) } // ClusterClient implements the ClusterManager @@ -204,3 +256,165 @@ func (v *ClusterClient) UpdateCluster(project, logicalCloud, clusterReference st } return c, nil } + +// Get returns Cluster's kubeconfig for corresponding cluster reference +func (v *ClusterClient) GetClusterConfig(project, logicalCloud, clusterReference string) (string, error) { + lcClient := NewLogicalCloudClient() + context, ctxVal, err := lcClient.GetLogicalCloudContext(logicalCloud) + if err != nil { + return "", pkgerrors.Wrap(err, "Error getting logical cloud context.") + } + if ctxVal == "" { + return "", pkgerrors.New("Logical Cloud hasn't been applied yet") + } + + // private key comes from logical cloud + lckey := LogicalCloudKey{ + Project: project, + LogicalCloudName: logicalCloud, + } + // get logical cloud resource + lc, err := lcClient.Get(project, logicalCloud) + if err != nil { + return "", pkgerrors.Wrap(err, "Failed getting logical cloud") + } + // get user's private key + privateKeyData, err := v.util.DBFind(v.storeName, lckey, "privatekey") + if err != nil { + return "", pkgerrors.Wrap(err, "Failed getting private key from logical cloud") + } + + // get cluster from dcm (need provider/name) + cluster, err := v.GetCluster(project, logicalCloud, clusterReference) + if err != nil { + return "", pkgerrors.Wrap(err, "Failed getting cluster") + } + + // before attempting to generate a kubeconfig, + // check if certificate has been issued and copy it from etcd to mongodb + if cluster.Specification.Certificate == "" { + log.Info("Certificate not yet in MongoDB, checking etcd.", log.Fields{}) + + // access etcd + clusterName := strings.Join([]string{cluster.Specification.ClusterProvider, "+", cluster.Specification.ClusterName}, "") + + // get the app context handle for the status of this cluster (which should contain the certificate inside, if already issued) + statusHandle, err := context.GetClusterStatusHandle("logical-cloud", clusterName) + + if err != nil { + return "", pkgerrors.New("The cluster doesn't contain status, please check if all services are up and running.") + } + statusRaw, err := context.GetValue(statusHandle) + if err != nil { + return "", pkgerrors.Wrap(err, "An error occurred while reading the cluster status.") + } + + var rbstatus rb.ResourceBundleStatus + err = json.Unmarshal([]byte(statusRaw.(string)), &rbstatus) + if err != nil { + return "", pkgerrors.Wrap(err, "An error occurred while parsing the cluster status.") + } + + // validate that we indeed obtained a certificate before persisting it in the database: + approved := false + for _, c := range rbstatus.CsrStatuses[0].Status.Conditions { + if c.Type == "Denied" { + return "", pkgerrors.Wrap(err, "Certificate was denied!") + } + if c.Type == "Failed" { + return "", pkgerrors.Wrap(err, "Certificate issue failed.") + } + if c.Type == "Approved" { + approved = true + } + } + if approved { + //just double-check certificate field contents aren't empty: + cert := rbstatus.CsrStatuses[0].Status.Certificate + if len(cert) > 0 { + cluster.Specification.Certificate = base64.StdEncoding.EncodeToString([]byte(cert)) + } else { + return "", pkgerrors.Wrap(err, "Certificate issued was invalid.") + } + } + + // copy key to MongoDB + // func (v *ClusterClient) + // UpdateCluster(project, logicalCloud, clusterReference string, c Cluster) (Cluster, error) { + _, err = v.UpdateCluster(project, logicalCloud, clusterReference, cluster) + if err != nil { + return "", pkgerrors.Wrap(err, "An error occurred while storing the certificate.") + } + } else { + // certificate is already in MongoDB so just hand it over to create the API response + log.Info("Certificate already in MongoDB, pass it to API.", log.Fields{}) + } + + // contact clm about admins cluster kubeconfig (to retrieve CA cert) + clusterContent, err := clm.NewClusterClient().GetClusterContent(cluster.Specification.ClusterProvider, cluster.Specification.ClusterName) + if err != nil { + return "", pkgerrors.Wrap(err, "Failed getting cluster content from CLM") + } + adminConfig, err := base64.StdEncoding.DecodeString(clusterContent.Kubeconfig) + if err != nil { + return "", pkgerrors.Wrap(err, "Failed decoding CLM kubeconfig from base64") + } + + // unmarshall clm kubeconfig into struct + adminKubeConfig := KubeConfig{} + err = yaml.Unmarshal(adminConfig, &adminKubeConfig) + if err != nil { + return "", pkgerrors.Wrap(err, "Failed parsing CLM kubeconfig yaml") + } + + // all data needed for final kubeconfig: + privateKey := string(privateKeyData[0]) + signedCert := cluster.Specification.Certificate + clusterCert := adminKubeConfig.Clusters[0].ClusterDef.CertificateAuthorityData + clusterAddr := adminKubeConfig.Clusters[0].ClusterDef.Server + namespace := lc.Specification.NameSpace + userName := lc.Specification.User.UserName + contextName := userName + "@" + clusterReference + + kubeconfig := KubeConfig{ + ApiVersion: "v1", + Kind: "Config", + Clusters: []KubeCluster{ + KubeCluster{ + ClusterName: clusterReference, + ClusterDef: KubeClusterDef{ + CertificateAuthorityData: clusterCert, + Server: clusterAddr, + }, + }, + }, + Contexts: []KubeContext{ + KubeContext{ + ContextName: contextName, + ContextDef: KubeContextDef{ + Cluster: clusterReference, + Namespace: namespace, + User: userName, + }, + }, + }, + CurrentContext: contextName, + Preferences: map[string]string{}, + Users: []KubeUser{ + KubeUser{ + UserName: userName, + UserDef: KubeUserDef{ + ClientCertificateData: signedCert, + ClientKeyData: privateKey, + }, + }, + }, + } + + yaml, err := yaml.Marshal(&kubeconfig) + if err != nil { + return "", pkgerrors.Wrap(err, "Failed marshaling user kubeconfig into yaml") + } + + return string(yaml), nil +} diff --git a/src/dcm/pkg/module/logicalcloud.go b/src/dcm/pkg/module/logicalcloud.go index 61d7b7a5..83ff153f 100644 --- a/src/dcm/pkg/module/logicalcloud.go +++ b/src/dcm/pkg/module/logicalcloud.go @@ -103,9 +103,10 @@ type DBService struct{} func NewLogicalCloudClient() *LogicalCloudClient { service := DBService{} return &LogicalCloudClient{ - storeName: "orchestrator", - tagMeta: "logicalcloud", - util: service, + storeName: "orchestrator", + tagMeta: "logicalcloud", + tagContext: "lccontext", + util: service, } } -- cgit 1.2.3-korg