From 2b5b2e0ff77608cfdfc8a949076860672b38b93f Mon Sep 17 00:00:00 2001 From: Pawel Wieczorek Date: Tue, 6 Aug 2019 16:04:53 +0200 Subject: k8s: Add support for RKE-deployed clusters RKE is used as a Kubernetes cluster deployment method from ONAP Dublin release. RKE cluster definition is used to get access to necessary information. Issue-ID: SECCOM-235 Change-Id: I588598011ea746b5f7ba327a48f1cea605e56d31 Signed-off-by: Pawel Wieczorek --- test/security/k8s/src/check/cmd/check/check.go | 25 ++++- test/security/k8s/src/check/config/config.go | 59 ++++++++++ test/security/k8s/src/check/raw/raw.go | 149 +++++++++++++++++++++++++ 3 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 test/security/k8s/src/check/config/config.go create mode 100644 test/security/k8s/src/check/raw/raw.go diff --git a/test/security/k8s/src/check/cmd/check/check.go b/test/security/k8s/src/check/cmd/check/check.go index c4dd39870..3c005f77c 100644 --- a/test/security/k8s/src/check/cmd/check/check.go +++ b/test/security/k8s/src/check/cmd/check/check.go @@ -5,12 +5,35 @@ import ( "log" "check/rancher" + "check/raw" "check/validators/master" ) +var ( + ranchercli = flag.Bool("ranchercli", false, "use rancher utility for accessing cluster nodes") + rke = flag.Bool("rke", true, "use RKE cluster definition and ssh for accessing cluster nodes (default)") +) + func main() { flag.Parse() - k8sParams, err := rancher.GetK8sParams() + if *ranchercli && *rke { + log.Fatal("Not supported.") + } + + var ( + k8sParams []string + err error + ) + + switch { + case *ranchercli: + k8sParams, err = rancher.GetK8sParams() + case *rke: + k8sParams, err = raw.GetK8sParams() + default: + log.Fatal("Missing cluster access method.") + } + if err != nil { log.Fatal(err) } diff --git a/test/security/k8s/src/check/config/config.go b/test/security/k8s/src/check/config/config.go new file mode 100644 index 000000000..dade6a67a --- /dev/null +++ b/test/security/k8s/src/check/config/config.go @@ -0,0 +1,59 @@ +// Package config reads relevant SSH access information from cluster config declaration. +package config + +import ( + "io/ioutil" + + v3 "github.com/rancher/types/apis/management.cattle.io/v3" + "gopkg.in/yaml.v2" +) + +const ( + defaultConfigFile = "cluster.yml" +) + +// NodeInfo contains role and SSH access information for a single cluster node. +type NodeInfo struct { + Role []string + User string + Address string + Port string + SSHKeyPath string +} + +// GetNodesInfo returns nodes' roles and SSH access information for a whole cluster. +func GetNodesInfo() ([]NodeInfo, error) { + config, err := readConfig(defaultConfigFile) + if err != nil { + return []NodeInfo{}, err + } + + cluster, err := parseConfig(config) + if err != nil { + return []NodeInfo{}, err + } + + var nodes []NodeInfo + for _, node := range cluster.Nodes { + nodes = append(nodes, NodeInfo{ + node.Role, node.User, node.Address, node.Port, node.SSHKeyPath, + }) + } + return nodes, nil +} + +func readConfig(configFile string) (string, error) { + config, err := ioutil.ReadFile(configFile) + if err != nil { + return "", err + } + return string(config), nil +} + +func parseConfig(config string) (*v3.RancherKubernetesEngineConfig, error) { + var rkeConfig v3.RancherKubernetesEngineConfig + if err := yaml.Unmarshal([]byte(config), &rkeConfig); err != nil { + return nil, err + } + return &rkeConfig, nil +} diff --git a/test/security/k8s/src/check/raw/raw.go b/test/security/k8s/src/check/raw/raw.go new file mode 100644 index 000000000..4efa1d4f8 --- /dev/null +++ b/test/security/k8s/src/check/raw/raw.go @@ -0,0 +1,149 @@ +// Package raw wraps SSH commands necessary for K8s inspection. +package raw + +import ( + "bytes" + "errors" + "io/ioutil" + "os/user" + "path/filepath" + + "golang.org/x/crypto/ssh" + kh "golang.org/x/crypto/ssh/knownhosts" + + "check/config" +) + +const ( + controlplane = "controlplane" + etcd = "etcd" + worker = "worker" + + k8sProcess = "kube-apiserver" + dockerInspectCmd = "docker inspect " + k8sProcess + " --format {{.Args}}" + + knownHostsFile = "~/.ssh/known_hosts" +) + +// GetK8sParams returns parameters of running Kubernetes API servers. +// It queries only cluster nodes with "controlplane" role. +func GetK8sParams() ([]string, error) { + nodes, err := config.GetNodesInfo() + if err != nil { + return []string{}, err + } + + for _, node := range nodes { + if isControlplaneNode(node.Role) { + cmd, err := getK8sCmd(node) + if err != nil { + return []string{}, err + } + + if len(cmd) > 0 { + i := bytes.Index(cmd, []byte(k8sProcess)) + if i == -1 { + return []string{}, errors.New("missing " + k8sProcess + " command") + } + return btos(cmd[i+len(k8sProcess):]), nil + } + } + } + + return []string{}, nil +} + +func isControlplaneNode(roles []string) bool { + for _, role := range roles { + if role == controlplane { + return true + } + } + return false +} + +func getK8sCmd(node config.NodeInfo) ([]byte, error) { + path, err := expandPath(node.SSHKeyPath) + if err != nil { + return nil, err + } + + pubKey, err := parsePublicKey(path) + if err != nil { + return nil, err + } + + khPath, err := expandPath(knownHostsFile) + if err != nil { + return nil, err + } + + hostKeyCallback, err := kh.New(khPath) + if err != nil { + return nil, err + } + + config := &ssh.ClientConfig{ + User: node.User, + Auth: []ssh.AuthMethod{pubKey}, + HostKeyCallback: hostKeyCallback, + } + + conn, err := ssh.Dial("tcp", node.Address+":"+node.Port, config) + if err != nil { + return nil, err + } + defer conn.Close() + + out, err := runCommand(dockerInspectCmd, conn) + if err != nil { + return nil, err + } + return out, nil +} + +func expandPath(path string) (string, error) { + if len(path) == 0 || path[0] != '~' { + return path, nil + } + + usr, err := user.Current() + if err != nil { + return "", err + } + return filepath.Join(usr.HomeDir, path[1:]), nil +} + +func parsePublicKey(path string) (ssh.AuthMethod, error) { + key, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, err + } + return ssh.PublicKeys(signer), nil +} + +func runCommand(cmd string, conn *ssh.Client) ([]byte, error) { + sess, err := conn.NewSession() + if err != nil { + return nil, err + } + defer sess.Close() + out, err := sess.Output(cmd) + if err != nil { + return nil, err + } + return out, nil +} + +// btos converts slice of bytes to slice of strings split by white space characters. +func btos(in []byte) []string { + var out []string + for _, b := range bytes.Fields(in) { + out = append(out, string(b)) + } + return out +} -- cgit 1.2.3-korg