From b75115b0678a1034ffc1d1c8fee40c7f5b995c97 Mon Sep 17 00:00:00 2001 From: Kiran Kamineni Date: Wed, 17 Jul 2019 16:55:00 -0700 Subject: Add custom label to track created resources Create a custom label on created resources Also, create it on pods where pods are being created. This will help us later for filtering and querying pods and resources. Issue-ID: MULTICLOUD-675 Change-Id: I4b4fce7b67f9f27559d99dcca94a9191b96cb7c6 Signed-off-by: Kiran Kamineni --- src/k8splugin/internal/app/client.go | 16 ++++++- src/k8splugin/internal/app/client_test.go | 2 +- src/k8splugin/internal/app/config_backend.go | 4 +- src/k8splugin/internal/app/instance.go | 8 ++-- src/k8splugin/internal/config/config.go | 54 ++++++++++++------------ src/k8splugin/internal/plugin/helpers.go | 58 ++++++++++++++++++++++++++ src/k8splugin/plugins/generic/plugin.go | 15 +++++++ src/k8splugin/plugins/namespace/plugin_test.go | 4 ++ src/k8splugin/plugins/service/plugin.go | 9 ++++ src/k8splugin/plugins/service/plugin_test.go | 7 +++- 10 files changed, 142 insertions(+), 35 deletions(-) (limited to 'src') diff --git a/src/k8splugin/internal/app/client.go b/src/k8splugin/internal/app/client.go index d44f3505..914a8eec 100644 --- a/src/k8splugin/internal/app/client.go +++ b/src/k8splugin/internal/app/client.go @@ -39,6 +39,7 @@ type KubernetesClient struct { dynamicClient dynamic.Interface discoverClient discovery.CachedDiscoveryInterface restMapper meta.RESTMapper + instanceID string } // getKubeConfig uses the connectivity client to get the kubeconfig based on the name @@ -55,11 +56,17 @@ func (k *KubernetesClient) getKubeConfig(cloudregion string) (string, error) { } // init loads the Kubernetes configuation values stored into the local configuration file -func (k *KubernetesClient) init(cloudregion string) error { +func (k *KubernetesClient) init(cloudregion string, iid string) error { if cloudregion == "" { return pkgerrors.New("Cloudregion is empty") } + if iid == "" { + return pkgerrors.New("Instance ID is empty") + } + + k.instanceID = iid + configPath, err := k.getKubeConfig(cloudregion) if err != nil { return pkgerrors.Wrap(err, "Get kubeconfig file") @@ -89,6 +96,7 @@ func (k *KubernetesClient) init(cloudregion string) error { } k.restMapper = restmapper.NewDeferredDiscoveryRESTMapper(k.discoverClient) + return nil } @@ -211,3 +219,9 @@ func (k *KubernetesClient) GetDynamicClient() dynamic.Interface { func (k *KubernetesClient) GetStandardClient() kubernetes.Interface { return k.clientSet } + +//GetInstanceID returns the instanceID that is injected into all the +//resources created by the plugin +func (k *KubernetesClient) GetInstanceID() string { + return k.instanceID +} diff --git a/src/k8splugin/internal/app/client_test.go b/src/k8splugin/internal/app/client_test.go index fd293ab0..7001d9e2 100644 --- a/src/k8splugin/internal/app/client_test.go +++ b/src/k8splugin/internal/app/client_test.go @@ -72,7 +72,7 @@ func TestInit(t *testing.T) { kubeClient := KubernetesClient{} // Refer to the connection via its name - err = kubeClient.init("mock_connection") + err = kubeClient.init("mock_connection", "abcdefg") if err != nil { t.Fatalf("TestGetKubeClient returned an error (%s)", err) } diff --git a/src/k8splugin/internal/app/config_backend.go b/src/k8splugin/internal/app/config_backend.go index b31cbac7..6bc145ee 100644 --- a/src/k8splugin/internal/app/config_backend.go +++ b/src/k8splugin/internal/app/config_backend.go @@ -354,7 +354,7 @@ func scheduleResources(c chan configResourceList) { log.Printf("[scheduleResources]: POST %v %v", data.profile, data.resourceTemplates) for _, inst := range resp { k8sClient := KubernetesClient{} - err = k8sClient.init(inst.Request.CloudRegion) + err = k8sClient.init(inst.Request.CloudRegion, inst.ID) if err != nil { log.Printf("Getting CloudRegion Information: %s", err.Error()) //Move onto the next cloud region @@ -374,7 +374,7 @@ func scheduleResources(c chan configResourceList) { log.Printf("[scheduleResources]: DELETE %v %v", data.profile, data.resourceTemplates) for _, inst := range resp { k8sClient := KubernetesClient{} - err = k8sClient.init(inst.Request.CloudRegion) + err = k8sClient.init(inst.Request.CloudRegion, inst.ID) if err != nil { log.Printf("Getting CloudRegion Information: %s", err.Error()) //Move onto the next cloud region diff --git a/src/k8splugin/internal/app/instance.go b/src/k8splugin/internal/app/instance.go index d28fe799..5cfdaea1 100644 --- a/src/k8splugin/internal/app/instance.go +++ b/src/k8splugin/internal/app/instance.go @@ -127,8 +127,10 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { return InstanceResponse{}, pkgerrors.Wrap(err, "Error resolving helm charts") } + id := generateInstanceID() + k8sClient := KubernetesClient{} - err = k8sClient.init(i.CloudRegion) + err = k8sClient.init(i.CloudRegion, id) if err != nil { return InstanceResponse{}, pkgerrors.Wrap(err, "Getting CloudRegion Information") } @@ -138,8 +140,6 @@ func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) { return InstanceResponse{}, pkgerrors.Wrap(err, "Create Kubernetes Resources") } - id := generateInstanceID() - //Compose the return response resp := InstanceResponse{ ID: id, @@ -292,7 +292,7 @@ func (v *InstanceClient) Delete(id string) error { } k8sClient := KubernetesClient{} - err = k8sClient.init(inst.Request.CloudRegion) + err = k8sClient.init(inst.Request.CloudRegion, inst.ID) if err != nil { return pkgerrors.Wrap(err, "Getting CloudRegion Information") } diff --git a/src/k8splugin/internal/config/config.go b/src/k8splugin/internal/config/config.go index ac653282..23ec401e 100644 --- a/src/k8splugin/internal/config/config.go +++ b/src/k8splugin/internal/config/config.go @@ -26,19 +26,20 @@ import ( // Configuration loads up all the values that are used to configure // backend implementations type Configuration struct { - CAFile string `json:"ca-file"` - ServerCert string `json:"server-cert"` - ServerKey string `json:"server-key"` - Password string `json:"password"` - DatabaseAddress string `json:"database-address"` - DatabaseType string `json:"database-type"` - PluginDir string `json:"plugin-dir"` - EtcdIP string `json:"etcd-ip"` - EtcdCert string `json:"etcd-cert"` - EtcdKey string `json:"etcd-key"` - EtcdCAFile string `json:"etcd-ca-file"` - OVNCentralAddress string `json:"ovn-central-address"` - ServicePort string `json:"service-port"` + CAFile string `json:"ca-file"` + ServerCert string `json:"server-cert"` + ServerKey string `json:"server-key"` + Password string `json:"password"` + DatabaseAddress string `json:"database-address"` + DatabaseType string `json:"database-type"` + PluginDir string `json:"plugin-dir"` + EtcdIP string `json:"etcd-ip"` + EtcdCert string `json:"etcd-cert"` + EtcdKey string `json:"etcd-key"` + EtcdCAFile string `json:"etcd-ca-file"` + OVNCentralAddress string `json:"ovn-central-address"` + ServicePort string `json:"service-port"` + KubernetesLabelName string `json:"kubernetes-label-name"` } // Config is the structure that stores the configuration @@ -74,19 +75,20 @@ func defaultConfiguration() *Configuration { } return &Configuration{ - CAFile: "ca.cert", - ServerCert: "server.cert", - ServerKey: "server.key", - Password: "", - DatabaseAddress: "127.0.0.1", - DatabaseType: "mongo", - PluginDir: cwd, - EtcdIP: "127.0.0.1", - EtcdCert: "etcd.cert", - EtcdKey: "etcd.key", - EtcdCAFile: "etcd-ca.cert", - OVNCentralAddress: "127.0.0.1:6641", - ServicePort: "9015", + CAFile: "ca.cert", + ServerCert: "server.cert", + ServerKey: "server.key", + Password: "", + DatabaseAddress: "127.0.0.1", + DatabaseType: "mongo", + PluginDir: cwd, + EtcdIP: "127.0.0.1", + EtcdCert: "etcd.cert", + EtcdKey: "etcd.key", + EtcdCAFile: "etcd-ca.cert", + OVNCentralAddress: "127.0.0.1:6641", + ServicePort: "9015", + KubernetesLabelName: "k8splugin.io/rb-instance-id", } } diff --git a/src/k8splugin/internal/plugin/helpers.go b/src/k8splugin/internal/plugin/helpers.go index 26e0f467..b5c9109c 100644 --- a/src/k8splugin/internal/plugin/helpers.go +++ b/src/k8splugin/internal/plugin/helpers.go @@ -17,14 +17,18 @@ package plugin import ( + "encoding/json" "log" "strings" utils "github.com/onap/multicloud-k8s/src/k8splugin/internal" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/config" "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" pkgerrors "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -45,6 +49,9 @@ type KubernetesConnector interface { // GetStandardClient returns the standard client that can be used to handle // standard kubernetes kinds GetStandardClient() kubernetes.Interface + + //GetInstanceID returns the InstanceID for tracking during creation + GetInstanceID() string } // Reference is the interface that is implemented @@ -90,3 +97,54 @@ func GetPluginByKind(kind string) (Reference, error) { return pluginImpl, nil } + +// TagPodsIfPresent finds the PodTemplateSpec from any workload +// object that contains it and changes the spec to include the tag label +func TagPodsIfPresent(unstruct *unstructured.Unstructured, tag string) { + + spec, ok := unstruct.Object["spec"].(map[string]interface{}) + if !ok { + log.Println("Error converting spec to map") + return + } + template, ok := spec["template"].(map[string]interface{}) + if !ok { + log.Println("Error converting template to map") + return + } + + data, err := json.Marshal(template) + if err != nil { + log.Println("Error Marshaling Podspec") + return + } + + //Attempt to convert the template to a podtemplatespec. + //This is to check if we have any pods being created. + podTemplateSpec := &corev1.PodTemplateSpec{} + _, err = podTemplateSpec.MarshalTo(data) + if err != nil { + log.Println("Did not find a podTemplateSpec" + err.Error()) + return + } + + //At this point, we know that the data contains a PodTemplateSpec + metadata, ok := template["metadata"].(map[string]interface{}) + if !ok { + log.Println("Error converting metadata to map") + return + } + + //Get the labels map + labels, ok := metadata["labels"].(map[string]string) + if !ok { + log.Println("Error converting labels to map") + return + } + + //Check if labels exist for this object + if labels == nil { + labels = map[string]string{} + } + labels[config.GetConfiguration().KubernetesLabelName] = tag +} diff --git a/src/k8splugin/plugins/generic/plugin.go b/src/k8splugin/plugins/generic/plugin.go index cc5fcb7a..0711466f 100644 --- a/src/k8splugin/plugins/generic/plugin.go +++ b/src/k8splugin/plugins/generic/plugin.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utils "github.com/onap/multicloud-k8s/src/k8splugin/internal" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/config" "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" "github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin" ) @@ -57,6 +58,20 @@ func (g genericPlugin) Create(yamlFilePath string, namespace string, client plug return "", pkgerrors.Wrap(err, "Mapping kind to resource error") } + //Add the tracking label to all resources created here + labels := unstruct.GetLabels() + //Check if labels exist for this object + if labels == nil { + labels = map[string]string{} + } + labels[config.GetConfiguration().KubernetesLabelName] = client.GetInstanceID() + unstruct.SetLabels(labels) + + // This checks if the resource we are creating has a podSpec in it + // Eg: Deployment, StatefulSet, Job etc.. + // If a PodSpec is found, the label will be added to it too. + plugin.TagPodsIfPresent(unstruct, client.GetInstanceID()) + gvr := mapping.Resource var createdObj *unstructured.Unstructured diff --git a/src/k8splugin/plugins/namespace/plugin_test.go b/src/k8splugin/plugins/namespace/plugin_test.go index 489ac096..c1944a40 100644 --- a/src/k8splugin/plugins/namespace/plugin_test.go +++ b/src/k8splugin/plugins/namespace/plugin_test.go @@ -46,6 +46,10 @@ func (t TestKubernetesConnector) GetStandardClient() kubernetes.Interface { return fake.NewSimpleClientset(t.object) } +func (t TestKubernetesConnector) GetInstanceID() string { + return "" +} + func TestCreateNamespace(t *testing.T) { testCases := []struct { label string diff --git a/src/k8splugin/plugins/service/plugin.go b/src/k8splugin/plugins/service/plugin.go index 136a1343..4c1f37be 100644 --- a/src/k8splugin/plugins/service/plugin.go +++ b/src/k8splugin/plugins/service/plugin.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utils "github.com/onap/multicloud-k8s/src/k8splugin/internal" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/config" "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" "github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin" ) @@ -52,6 +53,14 @@ func (p servicePlugin) Create(yamlFilePath string, namespace string, client plug } service.Namespace = namespace + labels := service.GetLabels() + //Check if labels exist for this object + if labels == nil { + labels = map[string]string{} + } + labels[config.GetConfiguration().KubernetesLabelName] = client.GetInstanceID() + service.SetLabels(labels) + result, err := client.GetStandardClient().CoreV1().Services(namespace).Create(service) if err != nil { return "", pkgerrors.Wrap(err, "Create Service error") diff --git a/src/k8splugin/plugins/service/plugin_test.go b/src/k8splugin/plugins/service/plugin_test.go index aa0bcc29..1cef5027 100644 --- a/src/k8splugin/plugins/service/plugin_test.go +++ b/src/k8splugin/plugins/service/plugin_test.go @@ -14,11 +14,12 @@ limitations under the License. package main import ( - "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" "reflect" "strings" "testing" + "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm" + coreV1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,6 +46,10 @@ func (t TestKubernetesConnector) GetStandardClient() kubernetes.Interface { return fake.NewSimpleClientset(t.object) } +func (t TestKubernetesConnector) GetInstanceID() string { + return "" +} + func TestCreateService(t *testing.T) { name := "mock-service" testCases := []struct { -- cgit 1.2.3-korg