diff options
Diffstat (limited to 'test/security')
64 files changed, 4566 insertions, 624 deletions
diff --git a/test/security/check_certificates/MANIFEST.in b/test/security/check_certificates/MANIFEST.in new file mode 100644 index 000000000..02c7aaf32 --- /dev/null +++ b/test/security/check_certificates/MANIFEST.in @@ -0,0 +1 @@ +include check_certificates/templates/*.j2 diff --git a/test/security/check_certificates/check_certificates/__init__.py b/test/security/check_certificates/check_certificates/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/test/security/check_certificates/check_certificates/__init__.py diff --git a/test/security/check_certificates/check_certificates/check_certificates_validity.py b/test/security/check_certificates/check_certificates/check_certificates_validity.py new file mode 100644 index 000000000..5d19a7390 --- /dev/null +++ b/test/security/check_certificates/check_certificates/check_certificates_validity.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2020 Orange, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# Check all the kubernetes pods, evaluate the certificate and build a +# certificate dashboard. +# +# Dependencies: +# See requirements.txt +# The dashboard is based on bulma framework +# +# Environment: +# This script should be run on the local machine which has network access to +# the onap K8S cluster. +# It requires k8s cluster config file on local machine +# It requires also the ONAP IP provided through an env variable ONAP_IP +# ONAP_NAMESPACE env variable is also considered +# if not set we set it to onap +# Example usage: +# python check_certificates_validity.py +# the summary html page will be generated where the script is launched +""" +Check ONAP certificates +""" +import argparse +import logging +import os +import ssl +import sys +import OpenSSL +from datetime import datetime +from kubernetes import client, config +from jinja2 import Environment, FileSystemLoader, select_autoescape +from socket import * # pylint: disable=W0614 + +# Set SSL timeout +setdefaulttimeout(10) + +# Logger +LOG_LEVEL = 'INFO' +logging.basicConfig() +LOGGER = logging.getLogger("Gating-Index") +LOGGER.setLevel(LOG_LEVEL) +CERT_MODES = ['nodeport', 'ingress', 'internal'] +EXP_CRITERIA_MIN = 30 +EXP_CRITERIA_MAX = 389 +EXPECTED_CERT_STRING = "C=US;O=ONAP;OU=OSAAF;CN=intermediateCA_9" +EXPECTED_STRIMZI_CA_CERT_STRING = "O=io.strimzi;CN=cluster-ca v0" +RESULT_PATH = "." + + +# Get arguments +parser = argparse.ArgumentParser() +parser.add_argument( + '-m', + '--mode', + choices=CERT_MODES, + help='Mode (nodeport, ingress, internal). If not set all modes are tried', + default='nodeport') +parser.add_argument( + '-i', + '--ip', + help='ONAP IP needed (for nodeport mode)', + default=os.environ.get('ONAP_IP')) +parser.add_argument( + '-n', + '--namespace', + help='ONAP namespace', + default='onap') +parser.add_argument( + '-d', + '--dir', + help='Result directory', + default=RESULT_PATH) + +args = parser.parse_args() + +# Get the ONAP namespace +onap_namespace = args.namespace +LOGGER.info("Verification of the %s certificates started", onap_namespace) + +# Create the target dir (in case it does not exist) +if os.pardir not in args.dir: + os.makedirs(args.dir, exist_ok=True) + +# Nodeport specific section +# Retrieve the kubernetes IP for mode nodeport +if args.mode == "nodeport": + if args.ip is None: + LOGGER.error( + "In nodeport mode, the IP of the ONAP cluster is required." + + "The value can be set using -i option " + + "or retrieved from the ONAP_IP env variable") + exit(parser.print_usage()) + try: + nodeports_xfail_list = [] + with open('nodeports_xfail.txt', 'r') as f: + first_line = f.readline() + for line in f: + nodeports_xfail_list.append( + line.split(" ", 1)[0].strip('\n')) + LOGGER.info("nodeports xfail list: %s", + nodeports_xfail_list) + except KeyError: + LOGGER.error("Please set the environment variable ONAP_IP") + sys.exit(1) + except FileNotFoundError: + LOGGER.warning("Nodeport xfail list not found") + +# Kubernetes section +# retrieve the candidate ports first +if args.mode == "internal": + k8s_config = config.load_incluster_config() +else: + k8s_config = config.load_kube_config() + +core = client.CoreV1Api() +api_instance = client.NetworkingV1Api( + client.ApiClient(k8s_config)) +k8s_services = core.list_namespaced_service(onap_namespace).items +k8s_ingress = api_instance.list_namespaced_ingress(onap_namespace).items + + +def get_certifificate_info(host, port): + LOGGER.debug("Host: %s", host) + LOGGER.debug("Port: %s", port) + cert = ssl.get_server_certificate( + (host, port)) + LOGGER.debug("get certificate") + x509 = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, cert) + + LOGGER.debug("get certificate") + exp_date = datetime.strptime( + x509.get_notAfter().decode('ascii'), '%Y%m%d%H%M%SZ') + LOGGER.debug("Expiration date retrieved %s", exp_date) + issuer = x509.get_issuer().get_components() + + issuer_info = '' + # format issuer nicely + for issuer_info_key, issuer_info_val in issuer: + issuer_info += (issuer_info_key.decode('utf-8') + "=" + + issuer_info_val.decode('utf-8') + ";") + cert_validity = False + if issuer_info[:-1] in [EXPECTED_CERT_STRING, EXPECTED_STRIMZI_CA_CERT_STRING]: + cert_validity = True + + return {'expiration_date': exp_date, + 'issuer': issuer_info[:-1], + 'validity': cert_validity} + + +def test_services(k8s_services, mode): + success_criteria = True # success criteria per scan + # looks for the certificates + node_ports_list = [] + node_ports_ssl_error_list = [] + node_ports_connection_error_list = [] + node_ports_type_error_list = [] + node_ports_reset_error_list = [] + + # for node ports and internal we consider the services + # for the ingress we consider the ingress + for service in k8s_services: + try: + for port in service.spec.ports: + # For nodeport mode, we consider + # - the IP of the cluster + # - spec.port.node_port + # + # For internal mode, we consider + # - spec.selector.app + # - spec.port.port + test_name = service.metadata.name + test_port = None + error_waiver = False # waiver per port + if mode == 'nodeport': + test_url = args.ip + test_port = port.node_port + + # Retrieve the nodeport xfail list + # to consider SECCOM waiver if needed + if test_port in nodeports_xfail_list: + error_waiver = True + else: # internal mode + test_port = port.port + test_url = '' + # in Internal mode there are 2 types + # app + # app.kubernetes.io/name + try: + test_url = service.spec.selector['app'] + except KeyError: + test_url = service.spec.selector['app.kubernetes.io/name'] + + if test_port is not None: + LOGGER.info( + "Look for certificate %s (%s:%s)", + test_name, + test_url, + test_port) + cert_info = get_certifificate_info(test_url, test_port) + exp_date = cert_info['expiration_date'] + LOGGER.info("Expiration date retrieved %s", exp_date) + # calculate the remaining time + delta_time = (exp_date - datetime.now()).days + + # Test criteria + if error_waiver: + LOGGER.info("Port found in the xfail list," + + "do not consider it for success criteria") + else: + if (delta_time < EXP_CRITERIA_MIN or + delta_time > EXP_CRITERIA_MAX): + success_criteria = False + if cert_info['validity'] is False: + success_criteria = False + # add certificate to the list + node_ports_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'expiration_date': str(exp_date), + 'remaining_days': delta_time, + 'cluster_ip': service.spec.cluster_ip, + 'issuer': cert_info['issuer'], + 'validity': cert_info['validity']}) + else: + LOGGER.debug("Port value retrieved as None") + except ssl.SSLError as e: + LOGGER.exception("Bad certificate for port %s" % port) + node_ports_ssl_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + except ConnectionRefusedError as e: + LOGGER.exception("ConnectionrefusedError for port %s" % port) + node_ports_connection_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + except TypeError as e: + LOGGER.exception("Type Error for port %s" % port) + node_ports_type_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + except ConnectionResetError as e: + LOGGER.exception("ConnectionResetError for port %s" % port) + node_ports_reset_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + except: + LOGGER.error("Unknown error") + + # Create html summary + jinja_env = Environment( + autoescape=select_autoescape(['html']), + loader=FileSystemLoader('./templates')) + if args.mode == 'nodeport': + jinja_env.get_template('cert-nodeports.html.j2').stream( + node_ports_list=node_ports_list, + node_ports_ssl_error_list=node_ports_ssl_error_list, + node_ports_connection_error_list=node_ports_connection_error_list, + node_ports_type_error_list=node_ports_type_error_list, + node_ports_reset_error_list=node_ports_reset_error_list).dump( + '{}/certificates.html'.format(args.dir)) + else: + jinja_env.get_template('cert-internal.html.j2').stream( + node_ports_list=node_ports_list, + node_ports_ssl_error_list=node_ports_ssl_error_list, + node_ports_connection_error_list=node_ports_connection_error_list, + node_ports_type_error_list=node_ports_type_error_list, + node_ports_reset_error_list=node_ports_reset_error_list).dump( + '{}/certificates.html'.format(args.dir)) + + return success_criteria + + +def test_ingress(k8s_ingress, mode): + LOGGER.debug('Test %s mode', mode) + for ingress in k8s_ingress: + LOGGER.debug(ingress) + return True + + +# *************************************************************************** +# *************************************************************************** +# start of the test +# *************************************************************************** +# *************************************************************************** +test_status = True +if args.mode == "ingress": + test_routine = test_ingress + test_param = k8s_ingress +else: + test_routine = test_services + test_param = k8s_services + +LOGGER.info(">>>> Test certificates: mode = %s", args.mode) +if test_routine(test_param, args.mode): + LOGGER.warning(">>>> Test PASS") +else: + LOGGER.warning(">>>> Test FAIL") + test_status = False + +if test_status: + LOGGER.info(">>>> Test Check certificates PASS") +else: + LOGGER.error(">>>> Test Check certificates FAIL") + sys.exit(1) diff --git a/test/security/check_certificates/check_certificates/templates/base.html.j2 b/test/security/check_certificates/check_certificates/templates/base.html.j2 new file mode 100644 index 000000000..cbb4e4428 --- /dev/null +++ b/test/security/check_certificates/check_certificates/templates/base.html.j2 @@ -0,0 +1,231 @@ +{% macro color(failing, total) %} +{% if failing == 0 %} +is-success +{% else %} +{% if (failing / total) <= 0.1 %} +is-warning +{% else %} +is-danger +{% endif %} +{% endif %} +{% endmacro %} + +{% macro percentage(failing, total) %} +{{ ((total - failing) / total) | round }} +{% endmacro %} + +{% macro statistic(resource_name, failing, total) %} +{% set success = total - failing %} +<div class="level-item has-text-centered"> + <div> + <p class="heading">{{ resource_name | capitalize }}</p> + <p class="title">{{ success }}/{{ total }}</p> + <progress class="progress {{ color(failing, total) }}" value="{{ success }}" max="{{ total }}">{{ percentage(failing, total) }}</progress> + </div> + </div> +{% endmacro %} + +{% macro pods_table(pods) %} +<div id="pods" class="table-container"> + <table class="table is-fullwidth is-striped is-hoverable"> + <thead> + <tr> + <th>Name</th> + <th>Ready</th> + <th>Status</th> + <th>Reason</th> + <th>Restarts</th> + </tr> + </thead> + <tbody> + {% for pod in pods %} + <tr> + <td><a href="./pod-{{ pod.name }}.html" title="{{ pod.name }}">{{ pod.k8s.metadata.name }}</a></td> + {% if pod.init_done %} + <td>{{ pod.running_containers }}/{{ (pod.containers | length) }}</td> + {% else %} + <td>Init:{{ pod.runned_init_containers }}/{{ (pod.init_containers | length) }}</td> + {% endif %} + <td>{{ pod.k8s.status.phase }}</td> + <td>{{ pod.k8s.status.reason }}</td> + {% if pod.init_done %} + <td>{{ pod.restart_count }}</td> + {% else %} + <td>{{ pod.init_restart_count }}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> + </table> +</div> +{% endmacro %} + +{% macro key_value_description_list(title, dict) %} +<dt><strong>{{ title | capitalize }}:</strong></dt> +<dd> + {% if dict %} + {% for key, value in dict.items() %} + {% if loop.first %} + <dl> + {% endif %} + <dt>{{ key }}:</dt> + <dd>{{ value }}</dd> + {% if loop.last %} + </dl> + {% endif %} + {% endfor %} + {% endif %} +</dd> +{% endmacro %} + +{% macro description(k8s) %} +<div class="container"> + <h1 class="title is-1">Description</h1> + <div class="content"> + <dl> + {% if k8s.spec.type %} + <dt><strong>Type:</strong></dt> + <dd>{{ k8s.spec.type }}</dd> + {% if (k8s.spec.type | lower) == "clusterip" %} + <dt><strong>Headless:</strong></dt> + <dd>{% if (k8s.spec.cluster_ip | lower) == "none" %}Yes{% else %}No{% endif %}</dd> + {% endif %} + {% endif %} + {{ key_value_description_list('Labels', k8s.metadata.labels) | indent(width=6) }} + {{ key_value_description_list('Annotations', k8s.metadata.annotations) | indent(width=6) }} + {% if k8s.spec.selector %} + {% if k8s.spec.selector.match_labels %} + {{ key_value_description_list('Selector', k8s.spec.selector.match_labels) | indent(width=6) }} + {% else %} + {{ key_value_description_list('Selector', k8s.spec.selector) | indent(width=6) }} + {% endif %} + {% endif %} + {% if k8s.phase %} + <dt><strong>Status:</strong></dt> + <dd>{{ k8s.phase }}</dd> + {% endif %} + {% if k8s.metadata.owner_references %} + <dt><strong>Controlled By:</strong></dt> + <dd>{{ k8s.metadata.owner_references[0].kind }}/{{ k8s.metadata.owner_references[0].name }}</dd> + {% endif %} + </dl> + </div> +</div> +{% endmacro %} + +{% macro pods_container(pods, parent, has_title=True) %} +<div class="container"> + {% if has_title %} + <h1 class="title is-1">Pods</h1> + {% endif %} + {% if (pods | length) > 0 %} + {{ pods_table(pods) | indent(width=2) }} + {% else %} + <div class="notification is-warning">{{ parent }} has no pods!</div> + {% endif %} +</div> +{% endmacro %} + +{% macro two_level_breadcrumb(title, name) %} +<section class="section"> + <div class="container"> + <nav class="breadcrumb" aria-label="breadcrumbs"> + <ul> + <li><a href="./index.html">Summary</a></li> + <li class="is-active"><a href="#" aria-current="page">{{ title | capitalize }} {{ name }}</a></li> + </ul> + </nav> + </div> +</section> +{% endmacro %} + +{% macro pod_parent_summary(title, name, failed_pods, pods) %} +{{ summary(title, name, [{'title': 'Pod', 'failing': failed_pods, 'total': (pods | length)}]) }} +{% endmacro %} + +{% macro number_ok(number, none_value, total=None) %} +{% if number %} +{% if total and number < total %} +<span class="tag is-warning">{{ number }}</span> +{% else %} +{{ number }} +{% endif %} +{% else %} +<span class="tag is-warning">{{ none_value }}</span> +{% endif %} +{% endmacro %} + +{% macro summary(title, name, statistics) %} +<section class="hero is-light"> + <div class="hero-body"> + <div class="container"> + <h1 class="title is-1"> + {{ title | capitalize }} {{ name }} Summary + </h1> + <nav class="level"> + {% for stat in statistics %} + {% if stat.total > 0 %} + {{ statistic(stat.title, stat.failing, stat.total) | indent(width=8) }} + {% endif %} + {% endfor %} + </nav> + </div> + </div> +</section> +{% endmacro %} + +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Tests results - {% block title %}{% endblock %}</title> + <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.0/css/bulma.min.css"> + <script defer src="https://use.fontawesome.com/releases/v5.3.1/js/all.js"></script> + {% block more_head %}{% endblock %} + </head> + <body> + <nav class="navbar" role="navigation" aria-label="main navigation"> + <div class="navbar-brand"> + <a class="navbar-item" href="https://www.onap.org"> + <img src="https://www.onap.org/wp-content/uploads/sites/20/2017/02/logo_onap_2017.png" width="234" height="50"> + </a> + + <a role="button" class="navbar-burger burger" aria-label="menu" aria-expanded="false" data-target="navbarBasicExample"> + <span aria-hidden="true"></span> + <span aria-hidden="true"></span> + <span aria-hidden="true"></span> + </a> + </div> + + <div id="navbarBasicExample" class="navbar-menu"> + <div class="navbar-start"> + <a class="navbar-item"> + Summary + </a> + </div> + </div> + </nav> + + {% block content %}{% endblock %} + + <footer class="footer"> + <div class="container"> + <div class="columns"> + <div class="column"> + <p class="has-text-grey-light"> + <a href="https://bulma.io/made-with-bulma/"> + <img src="https://bulma.io/images/made-with-bulma.png" alt="Made with Bulma" width="128" height="24"> + </a> + </div> + <div class="column"> + <a class="has-text-grey" href="https://gitlab.com/Orange-OpenSource/lfn/tools/kubernetes-status" style="border-bottom: 1px solid currentColor;"> + Improve this page on Gitlab + </a> + </p> + </div> + </div> + </div> + </footer> + </body> +</html> diff --git a/test/security/check_certificates/check_certificates/templates/cert-internal.html.j2 b/test/security/check_certificates/check_certificates/templates/cert-internal.html.j2 new file mode 100644 index 000000000..f9049807a --- /dev/null +++ b/test/security/check_certificates/check_certificates/templates/cert-internal.html.j2 @@ -0,0 +1,129 @@ +{% extends "base.html.j2" %} +{% block title %}ONAP Certificates expiration page{% endblock %} + +{% block content %} +<h1 class="title is-1">ONAP Certificates</h1> +<section class="section"> + <div class="container"> + <h3 class="subtitle">Cluster Internal Ports</h3> + +<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Expected Expiration Date</th> + <th>Remaining Days</th> + <th>Root CA</th> + <th>Root CA Validity</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_list %} + <tr {% if cert.remaining_days < 0 %} class="has-background-danger" {%elif cert.remaining_days < 30 %} class="has-background-warning" {%elif cert.remaining_days < 60 %} class="has-background-warning-light " {%elif cert.remaining_days > 389 %} class="has-background-warning-light" {%elif cert.remaining_days == 364 and cert.validity %} class="has-background-success-light" {% endif %}> + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.expiration_date }}</td> + <td>{{ cert.remaining_days }}</td> + <td>{{ cert.issuer }}</td> + <td>{% if cert.validity %} + <span class="icon is-large has-text-success"> + <i class="fas fa-check-square"></i> + </span> + {% else %} + <span class="icon is-large has-text-danger"> + <i class="fas fa-ban"></i> + </span> + {% endif %}</td> + </tr> + {% endfor %} + </tbody> + </table> + + {% if node_ports_ssl_error_list|length > 0 %} + <h3 class="subtitle">Cluster Internal Port SSL errors</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_ssl_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> + </table> +{% endif %} + +{% if node_ports_connection_error_list|length > 0 %} + <h3 class="subtitle">Cluster Internal Ports Connection errors</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_connection_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> + </table> +{% endif %} + +{% if node_ports_list_type_error_list|length > 0 %} + <h3 class="subtitle">Cluster Internal Ports ports Type Error</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_list_type_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> + </table> +{% endif %} + +{% if node_ports_reset_error_list|length > 0 %} + <h3 class="subtitle">Cluster Internal Ports Connections Error</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_reset_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> +</table> +{% endif %} + +{% endblock %} +</div> +</section> diff --git a/test/security/check_certificates/check_certificates/templates/cert-nodeports.html.j2 b/test/security/check_certificates/check_certificates/templates/cert-nodeports.html.j2 new file mode 100644 index 000000000..df37c3da9 --- /dev/null +++ b/test/security/check_certificates/check_certificates/templates/cert-nodeports.html.j2 @@ -0,0 +1,129 @@ +{% extends "base.html.j2" %} +{% block title %}ONAP Certificates expiration page{% endblock %} + +{% block content %} +<h1 class="title is-1">ONAP Certificates</h1> +<section class="section"> + <div class="container"> + <h3 class="subtitle">Node ports</h3> + +<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Expected Expiration Date</th> + <th>Remaining Days</th> + <th>Root CA</th> + <th>Root CA Validity</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_list %} + <tr {% if cert.remaining_days < 0 %} class="has-background-danger" {%elif cert.remaining_days < 30 %} class="has-background-warning" {%elif cert.remaining_days < 60 %} class="has-background-warning-light " {%elif cert.remaining_days > 389 %} class="has-background-warning-light" {%elif cert.remaining_days == 364 and cert.validity %} class="has-background-success-light" {% endif %}> + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.expiration_date }}</td> + <td>{{ cert.remaining_days }}</td> + <td>{{ cert.issuer }}</td> + <td>{% if cert.validity %} + <span class="icon is-large has-text-success"> + <i class="fas fa-check-square"></i> + </span> + {% else %} + <span class="icon is-large has-text-danger"> + <i class="fas fa-ban"></i> + </span> + {% endif %}</td> + </tr> + {% endfor %} + </tbody> + </table> + + {% if node_ports_ssl_error_list|length > 0 %} + <h3 class="subtitle">Node ports SSL errors</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_ssl_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> + </table> +{% endif %} + +{% if node_ports_connection_error_list|length > 0 %} + <h3 class="subtitle">Node ports Connection errors</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_connection_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> + </table> +{% endif %} + +{% if node_ports_list_type_error_list|length > 0 %} + <h3 class="subtitle">Node ports Type Error</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_list_type_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> + </table> +{% endif %} + +{% if node_ports_reset_error_list|length > 0 %} + <h3 class="subtitle">Node ports Connections Error</h3> + <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Port</th> + <th>Error Details</th> + </tr> + </thead> + <tbody> + {% for cert in node_ports_reset_error_list %} + <td>{{ cert.pod_name }}</td> + <td>{{ cert.pod_port }}</td> + <td>{{ cert.error_details }}</td> + </tr> + {% endfor %} + </tbody> +</table> +{% endif %} + +{% endblock %} +</div> +</section> diff --git a/test/security/check_certificates/requirements.txt b/test/security/check_certificates/requirements.txt new file mode 100644 index 000000000..15d50c44c --- /dev/null +++ b/test/security/check_certificates/requirements.txt @@ -0,0 +1,3 @@ +pyopenssl +kubernetes +jinja2 diff --git a/test/security/check_certificates/setup.cfg b/test/security/check_certificates/setup.cfg new file mode 100644 index 000000000..72966f837 --- /dev/null +++ b/test/security/check_certificates/setup.cfg @@ -0,0 +1,3 @@ +[metadata] +name = check_certificates +version = 0.1 diff --git a/test/security/check_certificates/setup.py b/test/security/check_certificates/setup.py new file mode 100644 index 000000000..f5154f282 --- /dev/null +++ b/test/security/check_certificates/setup.py @@ -0,0 +1,5 @@ +import setuptools +setuptools.setup( + setup_requires=['pbr', 'setuptools'], + pbr=True, + include_package_data=True) diff --git a/test/security/check_certificates/test-requirements.txt b/test/security/check_certificates/test-requirements.txt new file mode 100644 index 000000000..a0679b703 --- /dev/null +++ b/test/security/check_certificates/test-requirements.txt @@ -0,0 +1,6 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +coverage!=4.4,>=4.0 # Apache-2.0 +flake8 # MIT +pylint # GPLv2 diff --git a/test/security/check_certificates/tox.ini b/test/security/check_certificates/tox.ini new file mode 100644 index 000000000..2172bbc96 --- /dev/null +++ b/test/security/check_certificates/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py3 + +[testenv] +deps = + -r{toxinidir}/requirements.txt + +[testenv:py3] +commands = python {toxinidir}/setup.py test diff --git a/test/security/check_for_http_endpoints.sh b/test/security/check_for_http_endpoints.sh deleted file mode 100755 index 5c2ba20c3..000000000 --- a/test/security/check_for_http_endpoints.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2019 Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -# Check all ports exposed outside of kubernetes cluster looking for plain http -# endpoints. -# -# Dependencies: -# nmap -# kubectl + config -# -# Return value: Number of discovered http ports -# Output: List of pods exposing http endpoints -# - -#Prerequisities commands list -REQ_APPS=(kubectl nmap awk column sort paste grep wc) - -# Check for prerequisites apps -for cmd in "${REQ_APPS[@]}"; do - if ! [ -x "$(command -v "$cmd")" ]; then - echo "Error: command $cmd is not installed" - exit 1 - fi -done - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 <k8s-namespace>" - exit 1 -fi - -K8S_NAMESPACE=$1 - -# Get both values on single call as this may get slow -PORTS_SVCS=`kubectl get svc --namespace=$K8S_NAMESPACE -o go-template='{{range $item := .items}}{{range $port := $item.spec.ports}}{{if .nodePort}}{{.nodePort}}{{"\t"}}{{$item.metadata.name}}{{"\n"}}{{end}}{{end}}{{end}}' | column -t | sort -n` - -# Split port number and service name -PORTS=`awk '{print $1}' <<<"$PORTS_SVCS"` -SVCS=`awk '{print $2}' <<<"$PORTS_SVCS"` - -# Create a list in nmap-compatible format -PORT_LIST=`tr "\\n" "," <<<"$PORTS" | sed 's/,$//'; echo ''` - -# Get IP addres of some cluster node -K8S_NODE=`kubectl describe nodes \`kubectl get nodes | grep -v NAME | head -n 1 | awk '{print $1}'\` | grep external-ip | awk '{print $2}'` - -# perform scan -SCAN_RESULT=`nmap $K8S_NODE -sV -p $PORT_LIST 2>/dev/null | grep \tcp` - -# Concatenate scan result with service name -RESULTS=`paste <(printf %s "$SVCS") <(printf %s "$SCAN_RESULT") | column -t` - -# Find all plain http ports -HTTP_PORTS=`grep http <<< "$RESULTS" | grep -v ssl/http` - -# Count them -N_HTTP=`wc -l <<<"$HTTP_PORTS"` - -if [ "$N_HTTP" -gt 0 ]; then - echo "$HTTP_PORTS" -fi - -exit $N_HTTP diff --git a/test/security/check_for_ingress_and_nodeports.py b/test/security/check_for_ingress_and_nodeports.py new file mode 100755 index 000000000..e7950a0d7 --- /dev/null +++ b/test/security/check_for_ingress_and_nodeports.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2020 Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# Check all node ports exposed outside of kubernetes cluster looking for plain http and https. +# Check all ingress controller services exposed outside of the kubernetes cluster +# looking for plain http and https. This script looks for K8S NodePorts and ingress services declared +# in the K8S cluster configurations and check if service is alive or not. +# Automatic detect nodeport or ingress protocol HTTP or HTTPS it also detect if particular service uses HTTPS +# with self signed certificate (HTTPU). +# Verbose option retrives HTTP header and prints it for each service +# +# To setup runtime environment execute: +# +# $ tox +# $ source .tox/security/bin/activate +# +# Environment: +# This script should be run on on a host with access to the Onap K8S cluster API. +# It requires k8s cluster config file on local machine. +# +# Example usage: +# Display exposed nodeport and ingress resources declared in the K8S cluster without scanning: +# check_for_ingress_and_nodeports.py +# Scan declared nodeports: +# check_for_ingress_and_nodeports.py --scan-nodeport +# Scan declared exposed ingress resources: +# check_for_ingress_and_nodeports.py --scan-ingress + +from kubernetes import client, config +import http.client +import ssl +import socket +from enum import Enum +import argparse +import sys +import colorama +from colorama import Fore +import urllib.parse +from os import path + +""" List all nodeports """ +def list_nodeports(v1): + ret = {} + svc = v1.list_namespaced_service(K8S_NAMESPACE) + for i in svc.items: + if i.spec.ports: + ports = [ j.node_port for j in i.spec.ports if j.node_port ] + if ports: + ret[i.metadata.name] = ports + return ret + +# Class enum for returning current http mode +class ScanMode(Enum): + HTTPS = 0 #Safe https + HTTPU = 1 #Unsafe https + HTTP = 2 #Pure http + def __str__(self): + return self.name + +#Read the ingress controller http and https ports from the kubernetes cluster +def find_ingress_ports(v1): + svc = v1.list_namespaced_service(K8S_INGRESS_NS) + http_port = 0 + https_port = 0 + for item in svc.items: + if item.metadata.name == K8S_INGRESS_NS: + for pinfo in item.spec.ports: + if pinfo and pinfo.name == 'http': + http_port = pinfo.node_port + elif pinfo and pinfo.name == 'https': + https_port = pinfo.node_port + + return http_port,https_port + else: return(80,443) + +# List all ingress devices +def list_ingress(xv1b): + SSL_ANNOTATION = 'nginx.ingress.kubernetes.io/ssl-redirect' + inglist = xv1b.list_namespaced_ingress(K8S_NAMESPACE) + svc_list = {} + for ing in inglist.items: + svc_name = ing.metadata.labels['app'] + arr = [] + annotations = ing.metadata.annotations + for host in ing.spec.rules: + arr.append(host.host) + if (SSL_ANNOTATION in annotations) and annotations[SSL_ANNOTATION]=="true": + smode = ScanMode.HTTPS + else: smode = ScanMode.HTTP + svc_list[svc_name] = [ arr, smode ] + return svc_list + +# Scan single port +def scan_single_port(host,port,scanmode): + ssl_unverified = ssl._create_unverified_context() + if scanmode==ScanMode.HTTP: + conn = http.client.HTTPConnection(host,port,timeout=10) + elif scanmode==ScanMode.HTTPS: + conn = http.client.HTTPSConnection(host,port,timeout=10) + elif scanmode==ScanMode.HTTPU: + conn = http.client.HTTPSConnection(host,port,timeout=10,context=ssl_unverified) + outstr = None + retstatus = False + try: + conn.request("GET","/") + outstr = conn.getresponse() + except http.client.BadStatusLine as exc: + outstr = "Non HTTP proto" + str(exc) + retstatus = exc + except ConnectionRefusedError as exc: + outstr = "Connection refused" + str(exc) + retstatus = exc + except ConnectionResetError as exc: + outstr = "Connection reset" + str(exc) + retstatus = exc + except socket.timeout as exc: + outstr = "Connection timeout" + str(exc) + retstatus = exc + except ssl.SSLError as exc: + outstr = "SSL error" + str(exc) + retstatus = exc + except OSError as exc: + outstr = "OS error" + str(exc) + retstatus = exc + conn.close() + return retstatus,outstr + +# Scan port +def scan_portn(port): + host = urllib.parse.urlsplit(v1c.host).hostname + for mode in ScanMode: + retstatus, out = scan_single_port(host,port,mode) + if not retstatus: + result = port, mode, out.getcode(), out.read().decode('utf-8'),mode + break + else: + result = port, retstatus, out, None,mode + return result + + +def scan_port(host, http, https, mode): + if mode==ScanMode.HTTP: + retstatus, out = scan_single_port(host,http,ScanMode.HTTP) + if not retstatus: + return host, ScanMode.HTTP, out.getcode(), out.read().decode('utf-8'), mode + else: + return host, retstatus, out, None, mode + elif mode==ScanMode.HTTPS: + retstatus, out = scan_single_port(host,https,ScanMode.HTTPS) + if not retstatus: + return host, ScanMode.HTTPS, out.getcode(), out.read().decode('utf-8'), mode + else: + retstatus, out = scan_single_port(host,https,ScanMode.HTTPU) + if not retstatus: + return host, ScanMode.HTTPU, out.getcode(), out.read().decode('utf-8'), mode + else: + return host, retstatus, out, None, mode + + +# Visualise scan result +def console_visualisation(cname, name, retstatus, httpcode, out, mode, httpcodes = None): + if httpcodes is None: httpcodes=[] + print(Fore.YELLOW,end='') + print( cname,name, end='\t',sep='\t') + if isinstance(retstatus,ScanMode): + if httpcode in httpcodes: estr = Fore.RED + '[ERROR ' + else: estr = Fore.GREEN + '[OK ' + print( estr, retstatus, str(httpcode)+ ']'+Fore.RESET,end='') + if VERBOSE: print( '\t',str(out) ) + else: print() + else: + if not out: out = str(retstatus) + print( Fore.RED, '[ERROR ' +str(mode) +']', Fore.RESET,'\t', str(out)) + +# Visualize compare results +def console_compare_visualisation(cname,d1,d2): + print(Fore.YELLOW,end='') + print(cname, end='\t',sep='\t') + if d1!=d2: + print(Fore.RED + '[ERROR] '+ Fore.RESET) + if d1[0]!=d2[0]: + print('\tCode:',d1[0],'!=',d2[0]) + if d1[1]!=d2[1]: + print('\t******** Response #1 ********\n',d1[1]) + print('\t******** Response #2 ********\n',d2[1]) + else: + print(Fore.GREEN + '[OK ',d1[0],']', Fore.RESET,sep='') + if VERBOSE and d1[1]: + print(d1[1]) + + +# Port detector type +def check_onap_ports(): + print("Scanning onap NodePorts") + check_list = list_nodeports(v1) + if not check_list: + print(Fore.RED + 'Unable to find any declared node port in the K8S cluster', Fore.RESET) + for k,v in check_list.items(): + for port in v: + console_visualisation(k,*scan_portn(port) ) + +#Check ONAP ingress +def check_onap_ingress(): + print("Scanning onap ingress services") + ihttp,ihttps = find_ingress_ports(v1) + check_list = list_ingress(v1b) + if not check_list: + print(Fore.RED+ 'Unable to find any declared ingress service in the K8S cluster', Fore.RESET) + for k,v in check_list.items(): + for host in v[0]: + console_visualisation(k,*scan_port(host,ihttp,ihttps,v[1]),httpcodes=[404]) + +#Print onap all ingress ports and node ports +def onap_list_all(): + ihttp,ihttps = find_ingress_ports(v1) + host = urllib.parse.urlsplit(v1c.host).hostname + print( 'Cluster IP' + Fore.YELLOW, host, Fore.RESET ) + print('Ingress ' + Fore.RED + 'HTTP' + Fore.RESET + ' port:',Fore.YELLOW, ihttp, Fore.RESET) + print('Ingress ' + Fore.RED + 'HTTPS' + Fore.RESET + ' port:',Fore.YELLOW, ihttps, Fore.RESET) + print(Fore.YELLOW+"Onap NodePorts list:",Fore.RESET) + check_list = list_nodeports(v1) + for name,ports in check_list.items(): + print(Fore.GREEN, name,Fore.RESET,":", *ports) + print(Fore.YELLOW+"Onap ingress controler services list:",Fore.RESET) + check_list = list_ingress(v1b) + for name,hosts in check_list.items(): + print(Fore.GREEN, name + Fore.RESET,":", *hosts[0], Fore.RED+':', hosts[1],Fore.RESET) + +#Scan and compare nodeports and ingress check for results +def compare_nodeports_and_ingress(): + ihttp,ihttps = find_ingress_ports(v1) + print('Scanning nodeport services ...') + check_list = list_nodeports(v1) + if not check_list: + print(Fore.RED + 'Unable to find any declared node port in the K8S cluster', Fore.RESET) + valid_results = {} + for k,v in check_list.items(): + for port in v: + nodeport_results = scan_portn(port) + if isinstance(nodeport_results[1],ScanMode) and nodeport_results[2] != 404: + valid_results[k] = nodeport_results + if VERBOSE: console_visualisation(k,*nodeport_results) + check_list = list_ingress(v1b) + if not check_list: + print(Fore.RED+ 'Unable to find any declared ingress service in the K8S cluster', Fore.RESET) + print('Scanning ingress services ...') + ing_valid_results = {} + for k,v in check_list.items(): + for host in v[0]: + ingress_results = scan_port(host,ihttp,ihttps,v[1]) + if isinstance(ingress_results[1],ScanMode) and ingress_results[2]!=404: + ing_valid_results[k] = ingress_results + if VERBOSE: console_visualisation(k,*ingress_results,httpcodes=[404]) + ks1 = set(valid_results.keys()) + ks2 = set(ing_valid_results.keys()) + diff_keys = (ks1 - ks2) | (ks2 - ks1) + common_keys = ks1 & ks2 + if VERBOSE and diff_keys: + print(Fore.BLUE + '[WARNING] Non matching nodes and ingress list:') + for key in diff_keys: print(key,sep='\t') + print(Fore.RESET + 'Please check is it correct.') + print('Matching ingress and nodeport host scan results:') + for scan_key in common_keys: + s1 = valid_results[scan_key][2:4] + s2 = ing_valid_results[scan_key][2:4] + num_failures = 0 + if s1!=s2: ++num_failures + console_compare_visualisation(scan_key,s1,s2) + return num_failures + +def kube_config_exists(conf): + try: + assert path.exists(conf) + except AssertionError: + raise argparse.ArgumentTypeError(f'Fatal! K8S config {conf} does not exist') + else: + return conf + +if __name__ == "__main__": + colorama.init() + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + command_group = parser.add_mutually_exclusive_group() + command_group.add_argument("--scan-nodeport", + default=False, action='store_true', + help='Scan onap for node services' + ) + command_group.add_argument("--scan-ingress", + default=False, action='store_true', + help='Scan onap for ingress services' + ) + command_group.add_argument("--scan-and-compare", + default=False, action='store_true', + help='Scan nodeports and ingress and compare results' + ) + parser.add_argument( "--namespace", + default='onap', action='store', + help = 'kubernetes onap namespace' + ) + parser.add_argument( "--ingress-namespace", + default='ingress-nginx', action='store', + help = 'kubernetes ingress namespace' + ) + parser.add_argument( "--conf", + default='~/.kube/config', action='store', + help = 'kubernetes config file', + type = kube_config_exists + ) + parser.add_argument("--verbose", + default=False, action='store_true', + help='Verbose output' + ) + args = parser.parse_args() + K8S_NAMESPACE = args.namespace + K8S_INGRESS_NS = args.ingress_namespace + VERBOSE = args.verbose + config.load_kube_config(config_file=args.conf) + v1 = client.CoreV1Api() + v1b = client.ExtensionsV1beta1Api() + v1c = client.Configuration() + if args.scan_nodeport: check_onap_ports() + elif args.scan_ingress: check_onap_ingress() + elif args.scan_and_compare: sys.exit(compare_nodeports_and_ingress()) + else: onap_list_all() diff --git a/test/security/check_for_jdwp.sh b/test/security/check_for_jdwp.sh index e79f712bf..ec5b5cb16 100755 --- a/test/security/check_for_jdwp.sh +++ b/test/security/check_for_jdwp.sh @@ -28,68 +28,122 @@ # Return value: Number of discovered JDWP ports # Output: List of pods and exposing JDWP interface # +usage() { + cat <<EOF +Usage: $(basename $0) <k8s-namespace> [-l <white list file>] + -l: jdpw white list ports file +EOF + exit ${1:-0} +} if [ "$#" -lt 1 ]; then - echo "Usage: $0 <k8s-namespace>" + usage exit 1 fi K8S_NAMESPACE=$1 LOCAL_PORT=12543 +FILTERED_PORTS_LIST=$(mktemp jdpw_ports_XXXXXX) +WL_RAW_FILE_PATH=$(mktemp raw_filtered_ports_XXXXXX) + +manage_white_list() { + # init filtered port list file + if [ ! -f $WL_FILE_PATH ];then + echo "File not found" + usage + fi + grep -o '^[^#]*' $WL_FILE_PATH > $WL_RAW_FILE_PATH +} + +### getopts +while : +do + case $2 in + -h|--help|help) usage;; + -l) WL_FILE_PATH=$3;manage_white_list;shift;; + -*) usage 1 ;; + *) break ;; + esac +done list_pods() { - kubectl get po --namespace=$K8S_NAMESPACE | grep Running | awk '{print $1}' | grep -v NAME + kubectl get po --namespace=$K8S_NAMESPACE | grep Running | awk '{print $1}' | grep -v NAME } do_jdwp_handshake() { - local ip="127.0.0.1" - local port=$1 - local jdwp_challenge="JDWP-Handshake\n" - local jdwp_response="JDWP-Handshake" - - # 10s timeout to avoid hangs when service doesn't answer at all - local response=`nc -w 10 $ip $port <<<$jdwp_challenge | tr '\0' '\n'` - local n_response_lines=`echo "$response" | wc -l` - if [[ "$n_response_lines" -le 1 ]] && [[ $response == *"$jdwp_response"* ]]; then - return 0 - fi - - return 1 + local ip="127.0.0.1" + local port=$1 + local jdwp_challenge="JDWP-Handshake\n" + local jdwp_response="JDWP-Handshake" + + # 10s timeout to avoid hangs when service doesn't answer at all + local response=`nc -w 10 $ip $port <<<$jdwp_challenge | tr '\0' '\n'` + local n_response_lines=`echo "$response" | wc -l` + if [[ "$n_response_lines" -le 1 ]] && [[ $response == *"$jdwp_response"* ]]; then + return 0 + fi + + return 1 } # get open ports from procfs as netstat is not always available get_open_ports_on_pod() { - local pod=$1 - local open_ports_hex=`kubectl exec --namespace=$K8S_NAMESPACE $pod cat /proc/net/tcp 2>/dev/null| grep -v "local_address" | awk '{ print $2" "$4 }' | grep '0A$' | tr ":" " " | awk '{ print $2 }' | sort | uniq` - for hex_port in $open_ports_hex; do - echo $((16#$hex_port)) - done + local pod=$1 + local open_ports_hex=`kubectl exec --namespace=$K8S_NAMESPACE $pod cat /proc/net/tcp 2>/dev/null| grep -v "local_address" | awk '{ print $2" "$4 }' | grep '0A$' | tr ":" " " | awk '{ print $2 }' | sort | uniq` + for hex_port in $open_ports_hex; do + echo $((16#$hex_port)) + done } +echo "------------------------------------------------------------------------" +# Display the waivers +if [ -s $XL_FILE_PATH ]; then + echo "-------------------- *** WARNING XFail List *** ------------------------" + cat $WL_FILE_PATH + echo "------------------------------------------------------------------------" +fi + N_PORTS=0 # go through all pods for pod in `list_pods`; do - open_ports=`get_open_ports_on_pod $pod` - # if there is no open ports just go to next pod - if [ -z "$open_ports" ]; then - continue - fi - - # let's setup a proxy and check every open port - for port in $open_ports; do - # run proxy - kubectl port-forward --namespace=$K8S_NAMESPACE $pod $LOCAL_PORT:$port &>/dev/null & - sleep 1 - proxy_pid=$! - - do_jdwp_handshake $LOCAL_PORT - if [ $? -eq 0 ]; then - echo $pod $port - ((++N_PORTS)) - fi - kill $proxy_pid 2>/dev/null - wait $proxy_pid 2>/dev/null - done + open_ports=`get_open_ports_on_pod $pod` + # if there is no open ports just go to next pod + if [ -z "$open_ports" ]; then + continue + fi + + # let's setup a proxy and check every open port + for port in $open_ports; do + # run proxy + kubectl port-forward --namespace=$K8S_NAMESPACE $pod $LOCAL_PORT:$port &>/dev/null & + sleep 1 + proxy_pid=$! + + do_jdwp_handshake $LOCAL_PORT + if [ $? -eq 0 ]; then + echo $pod $port | tee $FILTERED_PORTS_LIST + ((++N_PORTS)) + fi + kill $proxy_pid 2>/dev/null + wait $proxy_pid 2>/dev/null + done done -exit $N_PORTS +while IFS= read -r line; do + # for each line we test if it is in the white list with a regular expression + while IFS= read -r wl_line; do + wl_name=$(echo $wl_line | awk {'print $1'}) + wl_port=$(echo $wl_line | awk {'print $2'}) + if grep -e $wl_name.*$wl_port <<< "$line";then + # Found in white list, exclude it + sed -i "/$line/d" $FILTERED_PORTS_LIST + fi + done < $WL_RAW_FILE_PATH +done < $FILTERED_PORTS_LIST + +N_FILTERED_PORTS_LIST=$(cat $FILTERED_PORTS_LIST |wc -l) +echo "------------------------------------" +echo "Nb error pod(s): $N_FILTERED_PORTS_LIST" +cat $FILTERED_PORTS_LIST + +exit $N_FILTERED_PORTS_LIST diff --git a/test/security/check_for_nonssl_endpoints.sh b/test/security/check_for_nonssl_endpoints.sh new file mode 100755 index 000000000..446792dea --- /dev/null +++ b/test/security/check_for_nonssl_endpoints.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2019 Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# Check all ports exposed outside of kubernetes cluster looking for non-SSL +# endpoints. +# +# Dependencies: +# nmap +# kubectl + config +# +# Return value: Number of discovered non-SSL ports +# Output: List of pods exposing non-SSL endpoints +# + +usage() { + cat <<EOF +Usage: $(basename $0) <k8s-namespace> [-l <list of non-SSL endpoints expected to fail this test>] + -l: list of non-SSL endpoints expected to fail this test +EOF + exit ${1:-0} +} + +#Prerequisities commands list +REQ_APPS=(kubectl nmap awk column sort paste grep wc mktemp sed cat) + +# Check for prerequisites apps +for cmd in "${REQ_APPS[@]}"; do + if ! [ -x "$(command -v "$cmd")" ]; then + echo "Error: command $cmd is not installed" + exit 1 + fi +done + +if [ "$#" -lt 1 ]; then + usage 1 +fi + +K8S_NAMESPACE=$1 +FILTERED_PORTS_LIST=$(mktemp nonssl_endpoints_XXXXXX) +XF_RAW_FILE_PATH=$(mktemp raw_filtered_nonssl_endpoints_XXXXXX) + +strip_white_list() { + if [ ! -f $XF_FILE_PATH ]; then + echo "File not found" + usage 1 + fi + grep -o '^[^#]*' $XF_FILE_PATH > $XF_RAW_FILE_PATH +} + +### getopts +while : +do + case $2 in + -h|--help|help) usage ;; + -l) XF_FILE_PATH=$3; strip_white_list; shift ;; + -*) usage 1 ;; + *) break ;; + esac +done + +echo "------------------------------------------------------------------------" +# Display the waivers +if [ -s $XF_FILE_PATH ]; then + echo "-------------------- *** WARNING XFail List *** ------------------------" + cat $XF_FILE_PATH + echo "------------------------------------------------------------------------" +fi + +# Get both values on single call as this may get slow +PORTS_SVCS=`kubectl get svc --namespace=$K8S_NAMESPACE -o go-template='{{range $item := .items}}{{range $port := $item.spec.ports}}{{if .nodePort}}{{.nodePort}}{{"\t"}}{{$item.metadata.name}}{{"\n"}}{{end}}{{end}}{{end}}' | column -t | sort -n` + +# Split port number and service name +PORTS=`awk '{print $1}' <<<"$PORTS_SVCS"` +SVCS=`awk '{print $2}' <<<"$PORTS_SVCS"` + +# Create a list in nmap-compatible format +PORT_LIST=`tr "\\n" "," <<<"$PORTS" | sed 's/,$//'; echo ''` + +# Get IP address of some cluster node (both "external-ip" and "ExternalIP" labels are matched) +K8S_NODE=`kubectl describe nodes \`kubectl get nodes | grep -v NAME | head -n 1 | awk '{print $1}'\` | grep "external-ip\|ExternalIP" | awk '{print $2}'` + +# perform scan +SCAN_RESULT=`nmap $K8S_NODE -sV -p $PORT_LIST 2>/dev/null | grep \tcp` + +# Concatenate scan result with service name +RESULTS=`paste <(printf %s "$SVCS") <(printf %s "$SCAN_RESULT") | column -t` + +# Find all non-SSL ports +HTTP_PORTS=`grep -v ssl <<< "$RESULTS" | grep open | tee "$FILTERED_PORTS_LIST"` + +# Filter out whitelisted endpoints +while IFS= read -r line; do + # for each line we test if it is in the white list with a regular expression + while IFS= read -r wl_line; do + wl_name=$(echo $wl_line | awk {'print $1'}) + wl_port=$(echo $wl_line | awk {'print $2'}) + if grep -e $wl_name.*$wl_port <<< "$line"; then + # Found in white list, exclude it + sed -i "/^$wl_name.*$wl_port/d" $FILTERED_PORTS_LIST + fi + done < $XF_RAW_FILE_PATH +done < $FILTERED_PORTS_LIST + +# Count them +N_FILTERED_PORTS_LIST=$(cat $FILTERED_PORTS_LIST | wc -l) +echo "------------------------------------" +echo "Nb error pod(s): $N_FILTERED_PORTS_LIST" +cat $FILTERED_PORTS_LIST +exit $N_FILTERED_PORTS_LIST diff --git a/test/security/check_versions/.gitignore b/test/security/check_versions/.gitignore new file mode 100644 index 000000000..2b574f8c0 --- /dev/null +++ b/test/security/check_versions/.gitignore @@ -0,0 +1,4 @@ +.pytest_cache/ +__pycache__/ +/temp/ +/.tox/ diff --git a/test/security/check_versions/README.md b/test/security/check_versions/README.md new file mode 100644 index 000000000..399d10443 --- /dev/null +++ b/test/security/check_versions/README.md @@ -0,0 +1,92 @@ +# Kubernetes Binaries Versions Inspector + +**Kubernetes Binaries Versions Inspector** (`k8s_bin_versions_inspector`) is a +python module for verifying versions of CPython and OpenJDK binaries installed +in the kubernetes cluster containers. + +## Commands + +### Install dependencies + +To install dependencies for normal usage of script, run this command. + +```bash +pip3 install -r requirements.txt +``` + +### Code formatting + +```bash +black src tests +``` + +### Code static analysis + +```bash +pylint -d C0330 src +``` + +### Automatic tests + +To running the automated tests is required to have properly configured +kubernetes cluster, which is in the virtual machine, that is containing +development environment. + +```bash +PYTHONPATH=src pytest -vv -s tests +``` + +### Removing caches + +```bash +find -name __pycache__ -exec rm -Rf {} + +find -name .pytest_cache -exec rm -Rf {} + +``` + +## Acceptable format + +Example of the acceptable file format: + +```yaml +python: + - 3.6.9 + - 3.7.3 +java: + - 11.0.7 +``` + +## Paths research + +Commands to research for the paths +of the software binaries in multiple docker images: + +```bash +docker run --entrypoint /bin/sh python:buster -c "which python" +docker run --entrypoint /bin/sh python:alpine -c "which python" +docker run --entrypoint /bin/sh python:slim -c "which python" +docker run --entrypoint /bin/sh python:2-buster -c "which python" +docker run --entrypoint /bin/sh python:2-alpine -c "which python" +docker run --entrypoint /bin/sh python:2-slim -c "which python" +docker run --entrypoint /bin/sh ubuntu:bionic -c "apt-get update && apt-get install -y python && which python" +docker run --entrypoint /bin/sh ubuntu:bionic -c "apt-get update && apt-get install -y python3 && which python3" +docker run --entrypoint /bin/sh openjdk -c "type java" +``` + +## Todo + +List of features, that should be implemented: + +- Complete license and copyrights variables. +- Find a way, to safe searching of the container files from Kubernetes API. +- Parallelization of executing binaries on the single container. +- Parallelization of versions determination in multiple containers. +- Support for determination the old versions of OpenJDK (attribute `-version`). +- Deleting namespace from cluster in development environment (for example, + during cluster reset), cause hanging in namespace terminating state. +- Find a nicer way to extracting exit code from execution result. + +## Links + +- <https://github.com/kubernetes-client/python> +- <https://github.com/kubernetes-client/python/issues/812> +- <https://success.docker.com/article/kubernetes-namespace-stuck-in-terminating> diff --git a/test/security/check_versions/pyproject.toml b/test/security/check_versions/pyproject.toml new file mode 100644 index 000000000..2c235c7b8 --- /dev/null +++ b/test/security/check_versions/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "check_versions" +readme = "README.md" +version = "1.0" +requires-python = ">=3.7" +dependencies = [ + "kubernetes", + "jinja2", + "xtesting", + "tabulate", + "cerberus", + "packaging", + "wget" +] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project.entry-points."xtesting.testcase"] +versions = "versions.k8s_bin_versions_inspector_test_case:Inspector" + +[tool.setuptools.package-data] +versions = ["templates/*.j2"] diff --git a/test/security/check_versions/requirements.txt b/test/security/check_versions/requirements.txt new file mode 100644 index 000000000..8e46a3acf --- /dev/null +++ b/test/security/check_versions/requirements.txt @@ -0,0 +1,7 @@ +kubernetes +jinja2 +xtesting +tabulate +cerberus +packaging +wget diff --git a/test/security/check_versions/tests/conftest.py b/test/security/check_versions/tests/conftest.py new file mode 100644 index 000000000..7c3e2e171 --- /dev/null +++ b/test/security/check_versions/tests/conftest.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 + +import pytest + + +def pod_name_trimmer_fun(pod_name): + return "-".join(pod_name.split("-")[:-2]) + + +@pytest.fixture +def pod_name_trimmer(): + return pod_name_trimmer_fun diff --git a/test/security/check_versions/tests/test_gather_containers_informations.py b/test/security/check_versions/tests/test_gather_containers_informations.py new file mode 100644 index 000000000..63401721e --- /dev/null +++ b/test/security/check_versions/tests/test_gather_containers_informations.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import k8s_bin_versions_inspector as kbvi +import kubernetes + + +def test_gather_containers_informations(pod_name_trimmer): + kubernetes.config.load_kube_config() + api = kubernetes.client.CoreV1Api() + containers = kbvi.gather_containers_informations(api, "", False) + data = [ + ( + c.namespace, + pod_name_trimmer(c.pod), + c.container, + c.versions.python, + c.versions.java, + ) + for c in containers + ] + sorted_data = sorted(data) + assert sorted_data == [ + ("default", "kbvi-test-java-keycloak", "keycloak", [], ["11.0.8"]), + ("default", "kbvi-test-java-keycloak-old", "keycloak-old", [], ["11.0.5"]), + ( + "default", + "kbvi-test-java-keycloak-very-old", + "keycloak-very-old", + ["2.7.5"], + [], + ), # TODO + ("default", "kbvi-test-python-jupyter", "jupyter", ["3.8.4"], []), + ("default", "kbvi-test-python-jupyter-old", "jupyter-old", ["3.6.6"], []), + ("default", "kbvi-test-python-stderr-filebeat", "filebeat", ["2.7.5"], []), + ("default", "kbvi-test-terminated", "python", [], []), # TODO + ("ingress-nginx", "kbvi-test-ingress-nginx", "echo-server", [], []), + ("kube-system", "kbvi-test-kube-system", "echo-server", [], []), + ] diff --git a/test/security/check_versions/tests/test_list_all_containers.py b/test/security/check_versions/tests/test_list_all_containers.py new file mode 100644 index 000000000..4178077c3 --- /dev/null +++ b/test/security/check_versions/tests/test_list_all_containers.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import k8s_bin_versions_inspector as kbvi +import kubernetes + + +def exec_list_all_containers(pod_name_trimmer, field_selector): + kubernetes.config.load_kube_config() + api = kubernetes.client.CoreV1Api() + containers = kbvi.list_all_containers(api, field_selector) + extracted = ((c.namespace, c.pod, c.container) for c in containers) + trimmed = ((n, pod_name_trimmer(p), c) for n, p, c in extracted) + result = sorted(trimmed) + return result + + +def test_list_all_containers(pod_name_trimmer): + result = exec_list_all_containers(pod_name_trimmer, "") + assert result == [ + ("default", "kbvi-test-java-keycloak", "keycloak"), + ("default", "kbvi-test-java-keycloak-old", "keycloak-old"), + ("default", "kbvi-test-java-keycloak-very-old", "keycloak-very-old"), + ("default", "kbvi-test-python-jupyter", "jupyter"), + ("default", "kbvi-test-python-jupyter-old", "jupyter-old"), + ("default", "kbvi-test-python-stderr-filebeat", "filebeat"), + ("default", "kbvi-test-terminated", "python"), + ("ingress-nginx", "kbvi-test-ingress-nginx", "echo-server"), + ("kube-system", "kbvi-test-kube-system", "echo-server"), + ] + + +def test_list_all_containers_not_default(pod_name_trimmer): + field_selector = "metadata.namespace!=default" + result = exec_list_all_containers(pod_name_trimmer, field_selector) + assert result == [ + ("ingress-nginx", "kbvi-test-ingress-nginx", "echo-server"), + ("kube-system", "kbvi-test-kube-system", "echo-server"), + ] + + +def test_list_all_containers_conjunction(pod_name_trimmer): + field_selector = "metadata.namespace!=kube-system,metadata.namespace!=ingress-nginx" + result = exec_list_all_containers(pod_name_trimmer, field_selector) + assert result == [ + ("default", "kbvi-test-java-keycloak", "keycloak"), + ("default", "kbvi-test-java-keycloak-old", "keycloak-old"), + ("default", "kbvi-test-java-keycloak-very-old", "keycloak-very-old"), + ("default", "kbvi-test-python-jupyter", "jupyter"), + ("default", "kbvi-test-python-jupyter-old", "jupyter-old"), + ("default", "kbvi-test-python-stderr-filebeat", "filebeat"), + ("default", "kbvi-test-terminated", "python"), + ] diff --git a/test/security/check_versions/tests/test_main.py b/test/security/check_versions/tests/test_main.py new file mode 100644 index 000000000..37ad45ee3 --- /dev/null +++ b/test/security/check_versions/tests/test_main.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import k8s_bin_versions_inspector as kbvi +import json +import tempfile +import yaml + + +def exec_main(pod_name_trimmer, acceptable_data): + with tempfile.NamedTemporaryFile() as output_temp, tempfile.NamedTemporaryFile() as acceptable_temp: + with open(acceptable_temp.name, "w") as stream: + yaml.safe_dump(acceptable_data, stream) + + result = kbvi.main( + [ + "--quiet", + "--output-file", + output_temp.name, + "--output-format", + "json", + "--acceptable", + acceptable_temp.name, + ] + ) + + with open(output_temp.name, "r") as stream: + output_data = json.load(stream) + output_extracted = ( + ( + item["namespace"], + pod_name_trimmer(item["pod"]), + item["container"], + item["versions"]["python"], + item["versions"]["java"], + ) + for item in output_data + ) + output_sorted = sorted(output_extracted) + + assert output_sorted == [ + ("default", "kbvi-test-java-keycloak", "keycloak", [], ["11.0.8"]), + ("default", "kbvi-test-java-keycloak-old", "keycloak-old", [], ["11.0.5"]), + ( + "default", + "kbvi-test-java-keycloak-very-old", + "keycloak-very-old", + ["2.7.5"], + [], + ), + ("default", "kbvi-test-python-jupyter", "jupyter", ["3.8.4"], []), + ("default", "kbvi-test-python-jupyter-old", "jupyter-old", ["3.6.6"], []), + ("default", "kbvi-test-python-stderr-filebeat", "filebeat", ["2.7.5"], []), + ("default", "kbvi-test-terminated", "python", [], []), + ("ingress-nginx", "kbvi-test-ingress-nginx", "echo-server", [], []), + ("kube-system", "kbvi-test-kube-system", "echo-server", [], []), + ] + + return result + + +def test_main(pod_name_trimmer): + acceptable_data = { + "python": ["2.7.5", "3.6.6", "3.8.4"], + "java": ["11.0.5", "11.0.8"], + } + + result = exec_main(pod_name_trimmer, acceptable_data) + + assert result == 0 + + +def test_main_neg(pod_name_trimmer): + acceptable_data = { + "python": ["3.6.6", "3.8.4"], + "java": ["11.0.5", "11.0.8"], + } + + result = exec_main(pod_name_trimmer, acceptable_data) + + assert result == 1 diff --git a/test/security/check_versions/tests/test_sync_post_namespaced_pod_exec.py b/test/security/check_versions/tests/test_sync_post_namespaced_pod_exec.py new file mode 100644 index 000000000..50620d3a7 --- /dev/null +++ b/test/security/check_versions/tests/test_sync_post_namespaced_pod_exec.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import k8s_bin_versions_inspector as kbvi +import kubernetes + + +def exec_sync_post_namespaced_pod_exec(pod, command): + kubernetes.config.load_kube_config() + api = kubernetes.client.CoreV1Api() + containers = kbvi.list_all_containers(api, "") + container = next(c for c in containers if c.pod.startswith(pod)) + result = kbvi.sync_post_namespaced_pod_exec(api, container, command) + return result + + +def test_sync_post_namespaced_pod_exec(): + pod = "kbvi-test-python-jupyter" + result = exec_sync_post_namespaced_pod_exec(pod, "id") + assert result == { + "stdout": "uid=1000(jovyan) gid=100(users) groups=100(users)\n", + "stderr": "", + "error": {"status": "Success", "metadata": {}}, + "code": 0, + } + + +def test_sync_post_namespaced_pod_exec_not_running(): + pod = "kbvi-test-terminated" + result = exec_sync_post_namespaced_pod_exec(pod, "id") + assert result == {"stdout": "", "stderr": "", "error": {}, "code": -1} + + +def test_sync_post_namespaced_pod_exec_not_found(): + pod = "kbvi-test-python-jupyter" + command = "/command/not/found" + result = exec_sync_post_namespaced_pod_exec(pod, command) + assert result["stdout"] == "" + assert result["stderr"] == "" + assert result["error"]["status"] == "Failure" + assert result["error"]["reason"] == "InternalError" + assert result["code"] == -2 + + +def test_sync_post_namespaced_pod_exec_exit_code(): + pod = "kbvi-test-python-jupyter" + command = ["python3", "--invalid-attribute"] + result = exec_sync_post_namespaced_pod_exec(pod, command) + assert result == { + "stdout": "", + "stderr": "unknown option --invalid-attribute\n" + "usage: python3 [option] ... [-c cmd | -m mod | file | -] [arg] ...\n" + "Try `python -h' for more information.\n", + "error": { + "status": "Failure", + "reason": "NonZeroExitCode", + "message": "command terminated with non-zero exit code: error " + "executing command [python3 --invalid-attribute], exit code 2", + "details": {"causes": [{"message": "2", "reason": "ExitCode"}]}, + "metadata": {}, + }, + "code": 2, + } + + +def test_sync_post_namespaced_pod_exec_stderr(): + pod = "kbvi-test-python-stderr-filebeat" + command = ["python", "--version"] + result = exec_sync_post_namespaced_pod_exec(pod, command) + assert result == { + "stdout": "", + "stderr": "Python 2.7.5\n", + "error": {"status": "Success", "metadata": {}}, + "code": 0, + } diff --git a/test/security/check_versions/tests/test_verify_versions_acceptability.py b/test/security/check_versions/tests/test_verify_versions_acceptability.py new file mode 100644 index 000000000..1cb931679 --- /dev/null +++ b/test/security/check_versions/tests/test_verify_versions_acceptability.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import k8s_bin_versions_inspector as kbvi +import yaml +import tempfile +import pathlib + + +def exec_verify_versions_acceptability(containers): + config = { + "python": ["1.1.1", "2.2.2"], + "java": ["3.3.3"], + } + + with tempfile.NamedTemporaryFile() as temp: + with open(temp.name, "w") as stream: + yaml.safe_dump(config, stream) + acceptable = pathlib.Path(temp.name) + result = kbvi.verify_versions_acceptability(containers, acceptable, True) + + return result + + +def test_verify_versions_acceptability(): + containers = [ + kbvi.ContainerInfo("a", "b", "c", None, kbvi.ContainerVersions([], [])), + kbvi.ContainerInfo( + "a", "b", "c", None, kbvi.ContainerVersions(["1.1.1"], ["3.3.3"]) + ), + ] + + result = exec_verify_versions_acceptability(containers) + + assert result == 0 + + +def test_verify_versions_acceptability_neg_1(): + containers = [ + kbvi.ContainerInfo("a", "b", "c", None, kbvi.ContainerVersions(["3.3.3"], [])) + ] + + result = exec_verify_versions_acceptability(containers) + + assert result == 1 + + +def test_verify_versions_acceptability_neg_2(): + containers = [ + kbvi.ContainerInfo("a", "b", "c", None, kbvi.ContainerVersions([], ["1.1.1"])) + ] + + result = exec_verify_versions_acceptability(containers) + + assert result == 1 diff --git a/test/security/check_versions/tox.ini b/test/security/check_versions/tox.ini new file mode 100644 index 000000000..d2a007160 --- /dev/null +++ b/test/security/check_versions/tox.ini @@ -0,0 +1,19 @@ +[tox] +envlist = black, pylint, pytest +skipsdist = true + +[testenv] +basepython = python3.8 +deps = -r{toxinidir}/requirements.txt + +[testenv:black] +commands = black {toxinidir}/versions tests +deps = black + +[testenv:pylint] +commands = pylint -d C0330,W0511 {toxinidir}/versions +deps= pylint + +[testenv:pytest] +setenv = PYTHONPATH = {toxinidir}/src +commands = pytest -vv -s tests diff --git a/test/security/check_versions/versions/__init__.py b/test/security/check_versions/versions/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/test/security/check_versions/versions/__init__.py diff --git a/test/security/check_versions/versions/k8s_bin_versions_inspector.py b/test/security/check_versions/versions/k8s_bin_versions_inspector.py new file mode 100644 index 000000000..bd3041d63 --- /dev/null +++ b/test/security/check_versions/versions/k8s_bin_versions_inspector.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2020 Samsung Electronics Co., Ltd. +# Copyright 2023 Deutsche Telekom AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +""" +k8s_bin_versions_inspector is a module for verifying versions of CPython and +OpenJDK binaries installed in the kubernetes cluster containers. +""" + +__title__ = "k8s_bin_versions_inspector" +__summary__ = ( + "Module for verifying versions of CPython and OpenJDK binaries installed" + " in the kubernetes cluster containers." +) +__version__ = "0.1.0" +__author__ = "kkkk.k@samsung.com" +__license__ = "Apache-2.0" +__copyright__ = "Copyright 2020 Samsung Electronics Co., Ltd." + +from typing import Iterable, List, Optional, Pattern, Union + +import argparse +import dataclasses +import itertools +import json +import logging +import pathlib +import pprint +import re +import string +import sys +from typing import Iterable, List, Optional, Pattern, Union +import tabulate +import yaml + +import kubernetes + +RECOMMENDED_VERSIONS_FILE = "/tmp/recommended_versions.yaml" +WAIVER_LIST_FILE = "/tmp/versions_xfail.txt" + +# Logger +logging.basicConfig() +LOGGER = logging.getLogger("onap-versions-status-inspector") +LOGGER.setLevel("INFO") + + +def parse_argv(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Function for parsing command line arguments. + + Args: + argv: Unparsed list of command line arguments. + + Returns: + Namespace with values from parsed arguments. + """ + + epilog = ( + f"Author: {__author__}\n" + f"License: {__license__}\n" + f"Copyright: {__copyright__}\n" + ) + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + prog=__title__, + description=__summary__, + epilog=epilog, + add_help=False, + ) + + parser.add_argument("-c", "--config-file", help="Name of the kube-config file.") + + parser.add_argument( + "-s", + "--field-selector", + default="", + help="Kubernetes field selector, to filter out containers objects.", + ) + + parser.add_argument( + "-o", + "--output-file", + type=pathlib.Path, + help="Path to file, where output will be saved.", + ) + + parser.add_argument( + "-f", + "--output-format", + choices=("tabulate", "pprint", "json"), + default="tabulate", + help="Format of the output file (tabulate, pprint, json).", + ) + + parser.add_argument( + "-i", + "--ignore-empty", + action="store_true", + help="Ignore containers without any versions.", + ) + + parser.add_argument( + "-a", + "--acceptable", + type=pathlib.Path, + help="Path to YAML file, with list of acceptable software versions.", + ) + + parser.add_argument( + "-n", + "--namespace", + help="Namespace to use to list pods." + "If empty pods are going to be listed from all namespaces", + ) + + parser.add_argument( + "--check-istio-sidecar", + action="store_true", + help="Add if you want to check istio sidecars also", + ) + + parser.add_argument( + "--istio-sidecar-name", + default="istio-proxy", + help="Name of istio sidecar to filter out", + ) + + parser.add_argument( + "-d", + "--debug", + action="store_true", + help="Enable debugging mode in the k8s API.", + ) + + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="Suppress printing text on standard output.", + ) + + parser.add_argument( + "-w", + "--waiver", + type=pathlib.Path, + help="Path of the waiver xfail file.", + ) + + parser.add_argument( + "-V", + "--version", + action="version", + version=f"{__title__} {__version__}", + help="Display version information and exit.", + ) + + parser.add_argument( + "-h", "--help", action="help", help="Display this help text and exit." + ) + + args = parser.parse_args(argv) + + return args + + +@dataclasses.dataclass +class ContainerExtra: + "Data class, to storage extra informations about container." + + running: bool + image: str + identifier: str + + +@dataclasses.dataclass +class ContainerVersions: + "Data class, to storage software versions from container." + + python: list + java: list + + +@dataclasses.dataclass +class ContainerInfo: + "Data class, to storage multiple informations about container." + + namespace: str + pod: str + container: str + extra: ContainerExtra + versions: ContainerVersions = None + + +def is_container_running( + status: kubernetes.client.models.v1_container_status.V1ContainerStatus, +) -> bool: + """Function to determine if k8s cluster container is in running state. + + Args: + status: Single item from container_statuses list, that represents container status. + + Returns: + If container is in running state. + """ + + if status.state.terminated: + return False + + if status.state.waiting: + return False + + if not status.state.running: + return False + + return True + + +def list_all_containers( + api: kubernetes.client.api.core_v1_api.CoreV1Api, + field_selector: str, + namespace: Union[None, str], + check_istio_sidecars: bool, + istio_sidecar_name: str, +) -> Iterable[ContainerInfo]: + """Get list of all containers names. + + Args: + api: Client of the k8s cluster API. + field_selector: Kubernetes field selector, to filter out containers objects. + namespace: Namespace to limit reading pods from + check_istio_sidecars: Flag to enable/disable istio sidecars check. + Default to False + istio_sidecar_name: If checking istio sidecars is disabled the name to filter + containers out + + Yields: + Objects for all containers in k8s cluster. + """ + + if namespace: + pods = api.list_namespaced_pod(namespace, field_selector=field_selector).items + else: + pods = api.list_pod_for_all_namespaces(field_selector=field_selector).items + + # Filtering to avoid testing integration or replica pods + pods = [ + pod + for pod in pods + if "replica" not in pod.metadata.name and "integration" not in pod.metadata.name + ] + + containers_statuses = ( + (pod.metadata.namespace, pod.metadata.name, pod.status.container_statuses) + for pod in pods + if pod.status.container_statuses + ) + + containers_status = ( + itertools.product([namespace], [pod], statuses) + for namespace, pod, statuses in containers_statuses + ) + + containers_chained = itertools.chain.from_iterable(containers_status) + + containers_fields = ( + ( + namespace, + pod, + status.name, + is_container_running(status), + status.image, + status.container_id, + ) + for namespace, pod, status in containers_chained + ) + + container_items = ( + ContainerInfo( + namespace, pod, container, ContainerExtra(running, image, identifier) + ) + for namespace, pod, container, running, image, identifier in containers_fields + ) + + if not check_istio_sidecars: + container_items = filter( + lambda container: container.container != istio_sidecar_name, container_items + ) + + yield from container_items + + +def sync_post_namespaced_pod_exec( + api: kubernetes.client.api.core_v1_api.CoreV1Api, + container: ContainerInfo, + command: Union[List[str], str], +) -> dict: + """Function to execute command on selected container. + + Args: + api: Client of the k8s cluster API. + container: Object, that represents container in k8s cluster. + command: Command to execute as a list of arguments or single string. + + Returns: + Dictionary that store informations about command execution. + * stdout - Standard output captured from execution. + * stderr - Standard error captured from execution. + * error - Error object that was received from kubernetes API. + * code - Exit code returned by executed process + or -1 if container is not running + or -2 if other failure occurred. + """ + + stdout = "" + stderr = "" + error = {} + code = -1 + LOGGER.debug("sync_post_namespaced_pod_exec container= %s", container.pod) + try: + client_stream = kubernetes.stream.stream( + api.connect_post_namespaced_pod_exec, + namespace=container.namespace, + name=container.pod, + container=container.container, + command=command, + stderr=True, + stdin=False, + stdout=True, + tty=False, + _request_timeout=1.0, + _preload_content=False, + ) + client_stream.run_forever(timeout=5) + stdout = client_stream.read_stdout() + stderr = client_stream.read_stderr() + error = yaml.safe_load( + client_stream.read_channel(kubernetes.stream.ws_client.ERROR_CHANNEL) + ) + + code = ( + 0 + if error["status"] == "Success" + else -2 + if error["reason"] != "NonZeroExitCode" + else int(error["details"]["causes"][0]["message"]) + ) + except ( + kubernetes.client.rest.ApiException, + kubernetes.client.exceptions.ApiException, + ): + LOGGER.debug("Discard unexpected k8s client Error..") + except TypeError: + LOGGER.debug("Type Error, no error status") + pass + + return { + "stdout": stdout, + "stderr": stderr, + "error": error, + "code": code, + } + + +def generate_python_binaries() -> List[str]: + """Function to generate list of names and paths for CPython binaries. + + Returns: + List of names and paths, to CPython binaries. + """ + + dirnames = ["", "/usr/bin/", "/usr/local/bin/"] + + majors_minors = [ + f"{major}.{minor}" for major, minor in itertools.product("23", string.digits) + ] + + suffixes = ["", "2", "3"] + majors_minors + + basenames = [f"python{suffix}" for suffix in suffixes] + + binaries = [f"{dir}{base}" for dir, base in itertools.product(dirnames, basenames)] + + return binaries + + +def generate_java_binaries() -> List[str]: + """Function to generate list of names and paths for OpenJDK binaries. + + Returns: + List of names and paths, to OpenJDK binaries. + """ + + binaries = [ + "java", + "/usr/bin/java", + "/usr/local/bin/java", + "/etc/alternatives/java", + "/usr/java/openjdk-14/bin/java", + ] + + return binaries + + +def determine_versions_abstraction( + api: kubernetes.client.api.core_v1_api.CoreV1Api, + container: ContainerInfo, + binaries: List[str], + extractor: Pattern, +) -> List[str]: + """Function to determine list of software versions, that are installed in + given container. + + Args: + api: Client of the k8s cluster API. + container: Object, that represents container in k8s cluster. + binaries: List of names and paths to the abstract software binaries. + extractor: Pattern to extract the version string from the output of the binary execution. + + Returns: + List of installed software versions. + """ + + commands = ([binary, "--version"] for binary in binaries) + commands_old = ([binary, "-version"] for binary in binaries) + commands_all = itertools.chain(commands, commands_old) + + # TODO: This list comprehension should be parallelized + results = ( + sync_post_namespaced_pod_exec(api, container, command) + for command in commands_all + ) + + successes = ( + f"{result['stdout']}{result['stderr']}" + for result in results + if result["code"] == 0 + ) + + extractions = (extractor.search(success) for success in successes) + + versions = sorted( + set(extraction.group(1) for extraction in extractions if extraction) + ) + + return versions + + +def determine_versions_of_python( + api: kubernetes.client.api.core_v1_api.CoreV1Api, container: ContainerInfo +) -> List[str]: + """Function to determine list of CPython versions, + that are installed in given container. + + Args: + api: Client of the k8s cluster API. + container: Object, that represents container in k8s cluster. + + Returns: + List of installed CPython versions. + """ + + extractor = re.compile("Python ([0-9.]+)") + + binaries = generate_python_binaries() + + versions = determine_versions_abstraction(api, container, binaries, extractor) + + return versions + + +def determine_versions_of_java( + api: kubernetes.client.api.core_v1_api.CoreV1Api, container: ContainerInfo +) -> List[str]: + """Function to determine list of OpenJDK versions, + that are installed in given container. + + Args: + api: Client of the k8s cluster API. + container: Object, that represents container in k8s cluster. + + Returns: + List of installed OpenJDK versions. + """ + + extractor = re.compile('openjdk [version" ]*([0-9._]+)') + + binaries = generate_java_binaries() + + versions = determine_versions_abstraction(api, container, binaries, extractor) + + return versions + + +def gather_containers_informations( + api: kubernetes.client.api.core_v1_api.CoreV1Api, + field_selector: str, + ignore_empty: bool, + namespace: Union[None, str], + check_istio_sidecars: bool, + istio_sidecar_name: str, +) -> List[ContainerInfo]: + """Get list of all containers names. + + Args: + api: Client of the k8s cluster API. + field_selector: Kubernetes field selector, to filter out containers objects. + ignore_empty: Determines, if containers with empty versions should be ignored. + namespace: Namespace to limit reading pods from + check_istio_sidecars: Flag to enable/disable istio sidecars check. + Default to False + istio_sidecar_name: If checking istio sidecars is disabled the name to filter + containers out + + Returns: + List of initialized objects for containers in k8s cluster. + """ + + containers = list( + list_all_containers( + api, field_selector, namespace, check_istio_sidecars, istio_sidecar_name + ) + ) + LOGGER.info("List of containers: %s", containers) + + # TODO: This loop should be parallelized + for container in containers: + LOGGER.info("Container -----------------> %s", container) + python_versions = determine_versions_of_python(api, container) + java_versions = determine_versions_of_java(api, container) + container.versions = ContainerVersions(python_versions, java_versions) + LOGGER.info("Container versions: %s", container.versions) + + if ignore_empty: + containers = [c for c in containers if c.versions.python or c.versions.java] + + return containers + + +def generate_output_tabulate(containers: Iterable[ContainerInfo]) -> str: + """Function for generate output string in tabulate format. + + Args: + containers: List of items, that represents containers in k8s cluster. + + Returns: + Output string formatted by tabulate module. + """ + + headers = [ + "Namespace", + "Pod", + "Container", + "Running", + "CPython", + "OpenJDK", + ] + + rows = [ + [ + container.namespace, + container.pod, + container.container, + container.extra.running, + " ".join(container.versions.python), + " ".join(container.versions.java), + ] + for container in containers + ] + + output = tabulate.tabulate(rows, headers=headers) + + return output + + +def generate_output_pprint(containers: Iterable[ContainerInfo]) -> str: + """Function for generate output string in pprint format. + + Args: + containers: List of items, that represents containers in k8s cluster. + + Returns: + Output string formatted by pprint module. + """ + + output = pprint.pformat(containers) + + return output + + +def generate_output_json(containers: Iterable[ContainerInfo]) -> str: + """Function for generate output string in JSON format. + + Args: + containers: List of items, that represents containers in k8s cluster. + + Returns: + Output string formatted by json module. + """ + + data = [ + { + "namespace": container.namespace, + "pod": container.pod, + "container": container.container, + "extra": { + "running": container.extra.running, + "image": container.extra.image, + "identifier": container.extra.identifier, + }, + "versions": { + "python": container.versions.python, + "java": container.versions.java, + }, + } + for container in containers + ] + + output = json.dumps(data, indent=4) + + return output + + +def generate_and_handle_output( + containers: List[ContainerInfo], + output_format: str, + output_file: pathlib.Path, + quiet: bool, +) -> None: + """Generate and handle the output of the containers software versions. + + Args: + containers: List of items, that represents containers in k8s cluster. + output_format: String that will determine output format (tabulate, pprint, json). + output_file: Path to file, where output will be save. + quiet: Determines if output should be printed, to stdout. + """ + + output_generators = { + "tabulate": generate_output_tabulate, + "pprint": generate_output_pprint, + "json": generate_output_json, + } + LOGGER.debug("output_generators: %s", output_generators) + + output = output_generators[output_format](containers) + + if output_file: + try: + output_file.write_text(output) + except AttributeError: + LOGGER.error("Not possible to write_text") + + if not quiet: + LOGGER.info(output) + + +def verify_versions_acceptability( + containers: List[ContainerInfo], acceptable: pathlib.Path, quiet: bool +) -> bool: + """Function for verification of software versions installed in containers. + + Args: + containers: List of items, that represents containers in k8s cluster. + acceptable: Path to the YAML file, with the software verification parameters. + quiet: Determines if output should be printed, to stdout. + + Returns: + 0 if the verification was successful or 1 otherwise. + """ + + if not acceptable: + return 0 + + try: + acceptable.is_file() + except AttributeError: + LOGGER.error("No acceptable file found") + return -1 + + if not acceptable.is_file(): + raise FileNotFoundError( + "File with configuration for acceptable does not exists!" + ) + + with open(acceptable) as stream: + data = yaml.safe_load(stream) + + python_acceptable = data.get("python3", []) + java_acceptable = data.get("java11", []) + + python_not_acceptable = [ + (container, "python3", version) + for container in containers + for version in container.versions.python + if version not in python_acceptable + ] + + java_not_acceptable = [ + (container, "java11", version) + for container in containers + for version in container.versions.java + if version not in java_acceptable + ] + + if not python_not_acceptable and not java_not_acceptable: + return 0 + + if quiet: + return 1 + + LOGGER.error("List of not acceptable versions") + pprint.pprint(python_not_acceptable) + pprint.pprint(java_not_acceptable) + + return 1 + + +def main(argv: Optional[List[str]] = None) -> str: + """Main entrypoint of the module for verifying versions of CPython and + OpenJDK installed in k8s cluster containers. + + Args: + argv: List of command line arguments. + """ + + args = parse_argv(argv) + + kubernetes.config.load_kube_config(args.config_file) + + api = kubernetes.client.CoreV1Api() + api.api_client.configuration.debug = args.debug + + containers = gather_containers_informations( + api, + args.field_selector, + args.ignore_empty, + args.namespace, + args.check_istio_sidecar, + args.istio_sidecar_name, + ) + + generate_and_handle_output( + containers, args.output_format, args.output_file, args.quiet + ) + + code = verify_versions_acceptability(containers, args.acceptable, args.quiet) + + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/security/check_versions/versions/k8s_bin_versions_inspector_test_case.py b/test/security/check_versions/versions/k8s_bin_versions_inspector_test_case.py new file mode 100644 index 000000000..30e46cad5 --- /dev/null +++ b/test/security/check_versions/versions/k8s_bin_versions_inspector_test_case.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2020 Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +import logging +import pathlib +import time +import os +import wget +from kubernetes import client, config +from xtesting.core import testcase # pylint: disable=import-error + +import versions.reporting as Reporting +from versions.k8s_bin_versions_inspector import ( + gather_containers_informations, + generate_and_handle_output, + verify_versions_acceptability, +) + +RECOMMENDED_VERSIONS_FILE = "/tmp/recommended_versions.yaml" +WAIVER_LIST_FILE = "/tmp/versions_xfail.txt" + +# Logger +logging.basicConfig() +LOGGER = logging.getLogger("onap-versions-status-inspector") +LOGGER.setLevel("INFO") + + +class Inspector(testcase.TestCase): + """Inspector CLass.""" + + def __init__(self, **kwargs): + """Init the testcase.""" + if "case_name" not in kwargs: + kwargs["case_name"] = "check_versions" + super().__init__(**kwargs) + + version = os.getenv("ONAP_VERSION", "master") + base_url = "https://git.onap.org/integration/seccom/plain" + + self.namespace = "onap" + # if no Recommended file found, download it + if pathlib.Path(RECOMMENDED_VERSIONS_FILE).is_file(): + self.acceptable = pathlib.Path(RECOMMENDED_VERSIONS_FILE) + else: + self.acceptable = wget.download( + base_url + "/recommended_versions.yaml?h=" + version, + out=RECOMMENDED_VERSIONS_FILE, + ) + self.output_file = "/tmp/versions.json" + # if no waiver file found, download it + if pathlib.Path(WAIVER_LIST_FILE).is_file(): + self.waiver = pathlib.Path(WAIVER_LIST_FILE) + else: + self.waiver = wget.download( + base_url + "/waivers/versions/versions_xfail.txt?h=" + version, + out=WAIVER_LIST_FILE, + ) + self.result = 0 + self.start_time = None + self.stop_time = None + + def run(self): + """Execute the version Inspector.""" + self.start_time = time.time() + config.load_kube_config() + api = client.CoreV1Api() + + field_selector = "metadata.namespace==onap" + + containers = gather_containers_informations(api, field_selector, True, None, False, "istio-proxy") + LOGGER.info("gather_containers_informations") + LOGGER.info(containers) + LOGGER.info("---------------------------------") + + generate_and_handle_output( + containers, "json", pathlib.Path(self.output_file), True + ) + LOGGER.info("generate_and_handle_output in %s", self.output_file) + LOGGER.info("---------------------------------") + + code = verify_versions_acceptability(containers, self.acceptable, True) + LOGGER.info("verify_versions_acceptability") + LOGGER.info(code) + LOGGER.info("---------------------------------") + + # Generate reporting + test = Reporting.OnapVersionsReporting(result_file=self.output_file) + LOGGER.info("Prepare reporting") + self.result = test.generate_reporting(self.output_file) + LOGGER.info("Reporting generated") + + self.stop_time = time.time() + if self.result >= 90: + return testcase.TestCase.EX_OK + return testcase.TestCase.EX_TESTCASE_FAILED + + def set_namespace(self, namespace): + """Set namespace.""" + self.namespace = namespace diff --git a/test/security/check_versions/versions/reporting.py b/test/security/check_versions/versions/reporting.py new file mode 100644 index 000000000..9053600c2 --- /dev/null +++ b/test/security/check_versions/versions/reporting.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Orange, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Generate result page +""" +import logging +import pathlib +import json +from dataclasses import dataclass +import os +import statistics +import wget +import yaml + +from packaging.version import Version + +from jinja2 import ( # pylint: disable=import-error + Environment, + select_autoescape, + PackageLoader, +) + +# Logger +LOG_LEVEL = "INFO" +logging.basicConfig() +LOGGER = logging.getLogger("onap-versions-status-reporting") +LOGGER.setLevel(LOG_LEVEL) + +REPORTING_FILE = "/var/lib/xtesting/results/versions_reporting.html" +# REPORTING_FILE = "/tmp/versions_reporting.html" +RESULT_FILE = "/tmp/versions.json" +RECOMMENDED_VERSIONS_FILE = "/tmp/recommended_versions.yaml" +WAIVER_LIST_FILE = "/tmp/versions_xfail.txt" + + +@dataclass +class TestResult: + """Test results retrieved from xtesting.""" + + pod_name: str + container: str + image: str + python_version: str + python_status: int + java_version: str + java_status: int + + +@dataclass +class SerieResult: + """Serie of tests.""" + + serie_id: str + success_rate: int = 0 + min: int = 0 + max: int = 0 + mean: float = 0.0 + median: float = 0.0 + nb_occurences: int = 0 + + +class OnapVersionsReporting: + """Build html summary page.""" + + def __init__(self, result_file) -> None: + """Initialization of the report.""" + version = os.getenv("ONAP_VERSION", "master") + base_url = "https://git.onap.org/integration/seccom/plain" + if pathlib.Path(WAIVER_LIST_FILE).is_file(): + self._waiver_file = pathlib.Path(WAIVER_LIST_FILE) + else: + self._waiver_file = wget.download( + base_url + "/waivers/versions/versions_xfail.txt?h=" + version, + out=WAIVER_LIST_FILE, + ) + if pathlib.Path(RECOMMENDED_VERSIONS_FILE).is_file(): + self._recommended_versions_file = pathlib.Path(RECOMMENDED_VERSIONS_FILE) + else: + self._recommended_versions_file = wget.download( + base_url + "/recommended_versions.yaml?h=" + version, + out=RECOMMENDED_VERSIONS_FILE, + ) + + def get_versions_scan_results(self, result_file, waiver_list): + """Get all the versions from the scan.""" + testresult = [] + # Get the recommended version list for java and python + min_java_version = self.get_recommended_version( + RECOMMENDED_VERSIONS_FILE, "java11" + ) + min_python_version = self.get_recommended_version( + RECOMMENDED_VERSIONS_FILE, "python3" + ) + + LOGGER.info("Min Java recommended version: %s", min_java_version) + LOGGER.info("Min Python recommended version: %s", min_python_version) + + with open(result_file) as json_file: + data = json.load(json_file) + LOGGER.info("Number of pods: %s", len(data)) + for component in data: + if component["container"] not in waiver_list: + testresult.append( + TestResult( + pod_name=component["pod"], + container=component["container"], + image=component["extra"]["image"], + python_version=component["versions"]["python"], + java_version=component["versions"]["java"], + python_status=self.get_version_status( + component["versions"]["python"], min_python_version[0] + ), + java_status=self.get_version_status( + component["versions"]["java"], min_java_version[0] + ), + ) + ) + LOGGER.info("Nb of pods (after waiver filtering) %s", len(testresult)) + return testresult + + @staticmethod + def get_version_status(versions, min_version): + """Based on the min version set the status of the component version.""" + # status_code + # 0: only recommended version found + # 1: recommended version found but not alone + # 2: recommended version not found but not far + # 3: recommended version not found but not far but not alone + # 4: recommended version not found + # we assume that versions are given accordign to usual java way + # X.Y.Z + LOGGER.debug("Version = %s", versions) + LOGGER.debug("Min Version = %s", min_version) + nb_versions_found = len(versions) + status_code = -1 + LOGGER.debug("Nb versions found :%s", nb_versions_found) + # if no version found retrieved -1 + if nb_versions_found > 0: + for version in versions: + clean_version = Version(version.replace("_", ".")) + min_version_ok = str(min_version) + + if clean_version >= Version(min_version_ok): + if nb_versions_found < 2: + status_code = 0 + else: + status_code = 2 + elif clean_version.major >= Version(min_version_ok).major: + if nb_versions_found < 2: + status_code = 1 + else: + status_code = 3 + else: + status_code = 4 + LOGGER.debug("Version status code = %s", status_code) + return status_code + + @staticmethod + def get_recommended_version(recommended_versions_file, component): + """Retrieve data from the json file.""" + with open(recommended_versions_file) as stream: + data = yaml.safe_load(stream) + try: + recommended_version = data[component]["recommended_versions"] + except KeyError: + recommended_version = None + return recommended_version + + @staticmethod + def get_waiver_list(waiver_file_path): + """Get the waiver list.""" + pods_to_be_excluded = [] + with open(waiver_file_path) as waiver_list: + for line in waiver_list: + line = line.strip("\n") + line = line.strip("\t") + if not line.startswith("#"): + pods_to_be_excluded.append(line) + return pods_to_be_excluded + + @staticmethod + def get_score(component_type, scan_res): + # Look at the java and python results + # 0 = recommended version + # 1 = acceptable version + nb_good_versions = 0 + nb_results = 0 + + for res in scan_res: + if component_type == "java": + if res.java_status >= 0: + nb_results += 1 + if res.java_status < 2: + nb_good_versions += 1 + elif component_type == "python": + if res.python_status >= 0: + nb_results += 1 + if res.python_status < 2: + nb_good_versions += 1 + try: + return round(nb_good_versions * 100 / nb_results, 1) + except ZeroDivisionError: + LOGGER.error("Impossible to calculate the success rate") + return 0 + + def generate_reporting(self, result_file): + """Generate HTML reporting page.""" + LOGGER.info("Generate versions HTML report.") + + # Get the waiver list + waiver_list = self.get_waiver_list(self._waiver_file) + LOGGER.info("Waiver list: %s", waiver_list) + + # Get the Versions results + scan_res = self.get_versions_scan_results(result_file, waiver_list) + + LOGGER.info("scan_res: %s", scan_res) + + # Evaluate result + status_res = {"java": 0, "python": 0} + for component_type in "java", "python": + status_res[component_type] = self.get_score(component_type, scan_res) + + LOGGER.info("status_res: %s", status_res) + + # Calculate the average score + numbers = [status_res[key] for key in status_res] + mean_ = statistics.mean(numbers) + + # Create reporting page + jinja_env = Environment( + autoescape=select_autoescape(["html"]), + loader=PackageLoader("versions"), + ) + page_info = { + "title": "ONAP Integration versions reporting", + "success_rate": status_res, + "mean": mean_, + } + jinja_env.get_template("versions.html.j2").stream( + info=page_info, data=scan_res + ).dump("{}".format(REPORTING_FILE)) + + return mean_ + + +if __name__ == "__main__": + test = OnapVersionsReporting( + RESULT_FILE, WAIVER_LIST_FILE, RECOMMENDED_VERSIONS_FILE + ) + test.generate_reporting(RESULT_FILE) diff --git a/test/security/check_versions/versions/templates/base.html.j2 b/test/security/check_versions/versions/templates/base.html.j2 new file mode 100644 index 000000000..025c0ad25 --- /dev/null +++ b/test/security/check_versions/versions/templates/base.html.j2 @@ -0,0 +1,232 @@ +{% macro color(failing, total) %} +{% if failing == 0 %} +is-success +{% else %} +{% if (failing / total) <= 0.1 %} +is-warning +{% else %} +is-danger +{% endif %} +{% endif %} +{% endmacro %} + +{% macro percentage(failing, total) %} +{{ ((total - failing) / total) | round }} +{% endmacro %} + +{% macro statistic(resource_name, failing, total) %} +{% set success = total - failing %} +<div class="level-item has-text-centered"> + <div> + <p class="heading">{{ resource_name | capitalize }}</p> + <p class="title">{{ success }}/{{ total }}</p> + <progress class="progress {{ color(failing, total) }}" value="{{ success }}" max="{{ total }}">{{ percentage(failing, total) }}</progress> + </div> + </div> +{% endmacro %} + +{% macro pods_table(pods) %} +<div id="pods" class="table-container"> + <table class="table is-fullwidth is-striped is-hoverable"> + <thead> + <tr> + <th>Name</th> + <th>Ready</th> + <th>Status</th> + <th>Reason</th> + <th>Restarts</th> + </tr> + </thead> + <tbody> + {% for pod in pods %} + <tr> + <td><a href="./pod-{{ pod.name }}.html" title="{{ pod.name }}">{{ pod.k8s.metadata.name }}</a></td> + {% if pod.init_done %} + <td>{{ pod.running_containers }}/{{ (pod.containers | length) }}</td> + {% else %} + <td>Init:{{ pod.runned_init_containers }}/{{ (pod.init_containers | length) }}</td> + {% endif %} + <td>{{ pod.k8s.status.phase }}</td> + <td>{{ pod.k8s.status.reason }}</td> + {% if pod.init_done %} + <td>{{ pod.restart_count }}</td> + {% else %} + <td>{{ pod.init_restart_count }}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> + </table> +</div> +{% endmacro %} + +{% macro key_value_description_list(title, dict) %} +<dt><strong>{{ title | capitalize }}:</strong></dt> +<dd> + {% if dict %} + {% for key, value in dict.items() %} + {% if loop.first %} + <dl> + {% endif %} + <dt>{{ key }}:</dt> + <dd>{{ value }}</dd> + {% if loop.last %} + </dl> + {% endif %} + {% endfor %} + {% endif %} +</dd> +{% endmacro %} + +{% macro description(k8s) %} +<div class="container"> + <h1 class="title is-1">Description</h1> + <div class="content"> + <dl> + {% if k8s.spec.type %} + <dt><strong>Type:</strong></dt> + <dd>{{ k8s.spec.type }}</dd> + {% if (k8s.spec.type | lower) == "clusterip" %} + <dt><strong>Headless:</strong></dt> + <dd>{% if (k8s.spec.cluster_ip | lower) == "none" %}Yes{% else %}No{% endif %}</dd> + {% endif %} + {% endif %} + {{ key_value_description_list('Labels', k8s.metadata.labels) | indent(width=6) }} + {{ key_value_description_list('Annotations', k8s.metadata.annotations) | indent(width=6) }} + {% if k8s.spec.selector %} + {% if k8s.spec.selector.match_labels %} + {{ key_value_description_list('Selector', k8s.spec.selector.match_labels) | indent(width=6) }} + {% else %} + {{ key_value_description_list('Selector', k8s.spec.selector) | indent(width=6) }} + {% endif %} + {% endif %} + {% if k8s.phase %} + <dt><strong>Status:</strong></dt> + <dd>{{ k8s.phase }}</dd> + {% endif %} + {% if k8s.metadata.owner_references %} + <dt><strong>Controlled By:</strong></dt> + <dd>{{ k8s.metadata.owner_references[0].kind }}/{{ k8s.metadata.owner_references[0].name }}</dd> + {% endif %} + </dl> + </div> +</div> +{% endmacro %} + +{% macro pods_container(pods, parent, has_title=True) %} +<div class="container"> + {% if has_title %} + <h1 class="title is-1">Pods</h1> + {% endif %} + {% if (pods | length) > 0 %} + {{ pods_table(pods) | indent(width=2) }} + {% else %} + <div class="notification is-warning">{{ parent }} has no pods!</div> + {% endif %} +</div> +{% endmacro %} + +{% macro two_level_breadcrumb(title, name) %} +<section class="section"> + <div class="container"> + <nav class="breadcrumb" aria-label="breadcrumbs"> + <ul> + <li><a href="./index.html">Summary</a></li> + <li class="is-active"><a href="#" aria-current="page">{{ title | capitalize }} {{ name }}</a></li> + </ul> + </nav> + </div> +</section> +{% endmacro %} + +{% macro pod_parent_summary(title, name, failed_pods, pods) %} +{{ summary(title, name, [{'title': 'Pod', 'failing': failed_pods, 'total': (pods | length)}]) }} +{% endmacro %} + +{% macro number_ok(number, none_value, total=None) %} +{% if number %} +{% if total and number < total %} +<span class="tag is-warning">{{ number }}</span> +{% else %} +{{ number }} +{% endif %} +{% else %} +<span class="tag is-warning">{{ none_value }}</span> +{% endif %} +{% endmacro %} + +{% macro summary(title, name, statistics) %} +<section class="hero is-light"> + <div class="hero-body"> + <div class="container"> + <h1 class="title is-1"> + {{ title | capitalize }} {{ name }} Summary + </h1> + <nav class="level"> + {% for stat in statistics %} + {% if stat.total > 0 %} + {{ statistic(stat.title, stat.failing, stat.total) | indent(width=8) }} + {% endif %} + {% endfor %} + </nav> + </div> + </div> +</section> +{% endmacro %} + +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Tests results - {% block title %}{% endblock %}</title> + <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.0/css/bulma.min.css"> + <script defer src="https://use.fontawesome.com/releases/v5.3.1/js/all.js"></script> + {% block more_head %}{% endblock %} + </head> + <body> + <nav class="navbar" role="navigation" aria-label="main navigation"> + <div class="navbar-brand"> + <a class="navbar-item" href="https://www.onap.org"> + <img src="https://www.onap.org/wp-content/uploads/sites/20/2017/02/logo_onap_2017.png" width="234" height="50"> + </a> + + <a role="button" class="navbar-burger burger" aria-label="menu" aria-expanded="false" data-target="navbarBasicExample"> + <span aria-hidden="true"></span> + <span aria-hidden="true"></span> + <span aria-hidden="true"></span> + </a> + </div> + + <div id="navbarBasicExample" class="navbar-menu"> + <div class="navbar-start"> + <a class="navbar-item"> + Summary + </a> + </div> + </div> + </nav> + + {% block content %}{% endblock %} + + <footer class="footer"> + <div class="container"> + <div class="columns"> + <div class="column"> + <p class="has-text-grey-light"> + <a href="https://bulma.io/made-with-bulma/"> + <img src="https://bulma.io/images/made-with-bulma.png" alt="Made with Bulma" width="128" height="24"> + </a> + </div> + <div class="column"> + <a class="has-text-grey" href="https://gitlab.com/Orange-OpenSource/lfn/tools/kubernetes-status" style="border-bottom: 1px solid currentColor;"> + Improve this page on Gitlab + </a> + </p> + </div> + </div> + </div> + </footer> + </body> +</html> + diff --git a/test/security/check_versions/versions/templates/versions.html.j2 b/test/security/check_versions/versions/templates/versions.html.j2 new file mode 100644 index 000000000..4860a72da --- /dev/null +++ b/test/security/check_versions/versions/templates/versions.html.j2 @@ -0,0 +1,85 @@ +{% extends "base.html.j2" %} +{% block title %}ONAPTEST Bench{% endblock %} + +{% block content %} +<h1 class="title is-1">{{ info.title }}</h1> + +<div class="container"> + +<article class="message"> +<div class="message-header"> + <p>Results</p> +</div> +<div class="message-body"> +SECCOM recommended versions (global success rate: {{ info.mean }}): + <ul> + <li>Java: {{ info.success_rate.java }}% </li> + <li>Python: {{ info.success_rate.python }}%</li> + </ul> +</div> +</article> + +<article class="message"> + <div class="message-header"> + <p>Legend</p> + </div> + <div class="message-body"> + <div class="has-background-success">SECCOM recommended version</div> + <div class="has-background-success-light">Not the recommended version but at least the major version</div> + <div class="has-background-warning-light">Ambiguous versions but at least 1 is the SECCOM recommended version</div> + <div class="has-background-warning">Ambiguous versions but at least 1 is the major recommended version</div> + <div class="has-background-danger">Wrong Versions</div> + </div> +</article> +<br> + +<h2 class="title is-1">JAVA versions</h2> + +<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Versions</th> + </tr> + </thead> + <tbody> + {% for component in data %} + <tr {% if component.java_status == 4 %} class="has-background-danger" {%elif component.java_status == 0 %} class="has-background-success" {%elif component.java_status == 1 %} class="has-background-success-light" {%elif component.java_status == 2 %} class="has-background-warning-light" {%elif component.java_status == 3 %} class="has-background-warning" {% endif %}> + + {% if component.java_version is defined and component.java_version|length > 0 %} + <td>{{ component.container }}</td> + <td>{{ component.java_version}}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> + </table> +</div> +<br> + +<div class="container"> +<h2 class="title is-1">Python versions</h2> + +<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth"> + <thead> + <tr> + <th>Component</th> + <th>Versions</th> + </tr> + </thead> + <tbody> + {% for component in data %} + <tr {% if component.python_status == 4 %} class="has-background-danger" {%elif component.python_status == 0 %} class="has-background-success" {%elif component.python_status == 1 %} class="has-background-success-light" {%elif component.python_status == 2 %} class="has-background-warning-light" {%elif component.python_status == 3 %} class="has-background-warning" {% endif %}> + {% if component.python_version is defined and component.python_version|length > 0 %} + <td>{{ component.container }}</td> + <td>{{ component.python_version}}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> + </table> +</div> + +{% endblock %} +</div> +</section> diff --git a/test/security/k8s/README b/test/security/k8s/README deleted file mode 100644 index fdb7e4c11..000000000 --- a/test/security/k8s/README +++ /dev/null @@ -1,56 +0,0 @@ -############################## -K8s secure configuration check -############################## - -Utility for checking if Kubernetes cluster configuration follows security recommendations. - -*************** -Getting started -*************** - -Prerequisites -============= - -Build ------ - -- make -- go_ - -.. _go: https://golang.org/doc/install - -Run ---- - -.. note:: Below applies to Rancher-based clusters (e.g. Casablanca) - -- `Rancher CLI`_ -- Docker_ (required to perform selected Rancher CLI calls) - -.. _`Rancher CLI`: https://rancher.com/docs/rancher/v1.6/en/cli -.. _Docker: https://docs.docker.com/install - -Test ----- - -- Ginkgo_ - -.. _Ginkgo: https://onsi.github.io/ginkgo/#getting-ginkgo - -Running -======= - -Calling:: - - make run - -will build and run configuration check executable. It is the default target. - -Testing -======= - -Calling:: - - make test - -will run tests. diff --git a/test/security/k8s/README.rst b/test/security/k8s/README.rst index 100b93820..b9e2dd5d2 120000..100644 --- a/test/security/k8s/README.rst +++ b/test/security/k8s/README.rst @@ -1 +1,45 @@ -README
\ No newline at end of file +############################## +K8s secure configuration check +############################## + +Utility for checking if Kubernetes cluster configuration follows security recommendations. + +*************** +Getting started +*************** + +Prerequisites +============= + +Build +----- + +- make +- go_ + +.. _go: https://golang.org/doc/install + +Test +---- + +- Ginkgo_ + +.. _Ginkgo: https://onsi.github.io/ginkgo/#getting-ginkgo + +Running +======= + +Calling:: + + make run + +will build and run configuration check executable. It is the default target. + +Testing +======= + +Calling:: + + make test + +will run tests. diff --git a/test/security/k8s/src/check/check.go b/test/security/k8s/src/check/check.go index cf412c112..91e9e5fd2 100644 --- a/test/security/k8s/src/check/check.go +++ b/test/security/k8s/src/check/check.go @@ -8,6 +8,8 @@ type Informer interface { GetSchedulerParams() ([]string, error) // GetControllerManagerParams returns controller manager parameters. GetControllerManagerParams() ([]string, error) + // GetEtcdParams returns etcd parameters. + GetEtcdParams() ([]string, error) } // Command represents commands run on cluster. @@ -20,6 +22,8 @@ const ( SchedulerProcess // ControllerManagerProcess represents controller manager command ("kube-controller-manager"). ControllerManagerProcess + // EtcdProcess represents controller manager service ("etcd"). + EtcdProcess ) func (c Command) String() string { @@ -27,35 +31,11 @@ func (c Command) String() string { "kube-apiserver", "kube-scheduler", "kube-controller-manager", + "etcd", } - if c < APIProcess || c > ControllerManagerProcess { + if c < APIProcess || c > EtcdProcess { return "exit" } return names[c] } - -// Service represents services run on Rancher-based cluster. -type Service int - -const ( - // APIService represents API server service ("kubernetes/kubernetes"). - APIService Service = iota - // SchedulerService represents scheduler service ("kubernetes/scheduler"). - SchedulerService - // ControllerManagerService represents controller manager service ("kubernetes/controller-manager"). - ControllerManagerService -) - -func (s Service) String() string { - names := [...]string{ - "kubernetes/kubernetes", - "kubernetes/scheduler", - "kubernetes/controller-manager", - } - - if s < APIService || s > ControllerManagerService { - return "" - } - return names[s] -} diff --git a/test/security/k8s/src/check/cmd/check/check.go b/test/security/k8s/src/check/cmd/check/check.go index d7176170a..42e9903e6 100644 --- a/test/security/k8s/src/check/cmd/check/check.go +++ b/test/security/k8s/src/check/cmd/check/check.go @@ -5,32 +5,23 @@ import ( "log" "check" - "check/rancher" "check/raw" "check/validators/master" ) var ( - ranchercli = flag.Bool("ranchercli", false, "use rancher utility for accessing cluster nodes") - rke = flag.Bool("rke", false, "use RKE cluster definition and ssh for accessing cluster nodes (default)") + rke = flag.Bool("rke", true, "use RKE cluster definition and ssh for accessing cluster nodes (default)") ) func main() { flag.Parse() - if *ranchercli && *rke { + if !(*rke) { log.Fatal("Not supported.") } - // Use default cluster access method if none was declared explicitly. - if !(*ranchercli || *rke) { - *rke = true - } - var info check.Informer switch { - case *ranchercli: - info = &rancher.Rancher{} case *rke: info = &raw.Raw{} default: @@ -54,4 +45,14 @@ func main() { log.Fatal(err) } master.CheckControllerManager(controllerManagerParams) + + _, err = info.GetEtcdParams() + if err != nil { + switch err { + case check.ErrNotImplemented: + log.Print(err) // Fail softly. + default: + log.Fatal(err) + } + } } diff --git a/test/security/k8s/src/check/errors.go b/test/security/k8s/src/check/errors.go new file mode 100644 index 000000000..d657c1827 --- /dev/null +++ b/test/security/k8s/src/check/errors.go @@ -0,0 +1,10 @@ +package check + +import ( + "errors" +) + +var ( + // ErrNotImplemented is returned when function is not implemented yet. + ErrNotImplemented = errors.New("function not implemented") +) diff --git a/test/security/k8s/src/check/rancher/rancher.go b/test/security/k8s/src/check/rancher/rancher.go deleted file mode 100644 index b5e382221..000000000 --- a/test/security/k8s/src/check/rancher/rancher.go +++ /dev/null @@ -1,118 +0,0 @@ -// Package rancher wraps Rancher commands necessary for K8s inspection. -package rancher - -import ( - "bytes" - "fmt" - "os/exec" - - "check" -) - -const ( - bin = "rancher" - paramHost = "--host" - cmdHosts = "hosts" - cmdHostsParams = "--quiet" - cmdDocker = "docker" - cmdDockerCmdPs = "ps" - cmdDockerCmdPsParams = "--no-trunc" - cmdDockerCmdPsFilter = "--filter" - cmdDockerCmdPsFilterArgs = "label=io.rancher.stack_service.name=" - cmdDockerCmdPsFormat = "--format" - cmdDockerCmdPsFormatArgs = "{{.Command}}" -) - -// Rancher implements Informer interface. -type Rancher struct { - check.Informer -} - -// GetAPIParams returns parameters of running Kubernetes API server. -// It queries default environment set in configuration file. -func (r *Rancher) GetAPIParams() ([]string, error) { - return getProcessParams(check.APIProcess, check.APIService) -} - -// GetSchedulerParams returns parameters of running Kubernetes scheduler. -// It queries default environment set in configuration file. -func (r *Rancher) GetSchedulerParams() ([]string, error) { - return getProcessParams(check.SchedulerProcess, check.SchedulerService) -} - -// GetControllerManagerParams returns parameters of running Kubernetes scheduler. -// It queries default environment set in configuration file. -func (r *Rancher) GetControllerManagerParams() ([]string, error) { - return getProcessParams(check.ControllerManagerProcess, check.ControllerManagerService) -} - -func getProcessParams(process check.Command, service check.Service) ([]string, error) { - hosts, err := listHosts() - if err != nil { - return []string{}, err - } - - for _, host := range hosts { - cmd, err := getPsCmdOutput(host, service) - if err != nil { - return []string{}, err - } - - cmd = trimOutput(cmd) // TODO: improve `docker ps` query format. - if len(cmd) > 0 { - i := bytes.Index(cmd, []byte(process.String())) - if i == -1 { - return []string{}, fmt.Errorf("missing %s command", process) - } - return btos(cmd[i+len(process.String()):]), nil - } - } - return []string{}, nil -} - -// listHosts lists IDs of active hosts. -// It queries default environment set in configuration file. -func listHosts() ([]string, error) { - cmd := exec.Command(bin, cmdHosts, cmdHostsParams) - out, err := cmd.Output() - if err != nil { - return nil, err - } - return btos(out), nil -} - -// getPsCmdOutput returns running Kubernetes service command with its parameters. -// It queries default environment set in configuration file. -func getPsCmdOutput(host string, service check.Service) ([]byte, error) { - // Following is equivalent to: - // $ rancher --host $HOST \ - // docker ps --no-trunc \ - // --filter "label=io.rancher.stack_service.name=$SERVICE" \ - // --format "{{.Command}}" - cmd := exec.Command(bin, paramHost, host, - cmdDocker, cmdDockerCmdPs, cmdDockerCmdPsParams, - cmdDockerCmdPsFilter, cmdDockerCmdPsFilterArgs+service.String(), - cmdDockerCmdPsFormat, cmdDockerCmdPsFormatArgs) - out, err := cmd.Output() - if err != nil { - return nil, err - } - return out, nil -} - -// trimOutput removes trailing new line and brackets from output. -func trimOutput(b []byte) []byte { - b = bytes.TrimSpace(b) - b = bytes.TrimPrefix(b, []byte("[")) - b = bytes.TrimSuffix(b, []byte("]")) - return b -} - -// btos converts slice of bytes to slice of strings split by white space characters. -func btos(in []byte) []string { - var out []string - for _, b := range bytes.Fields(in) { - out = append(out, string(b)) - } - return out -} diff --git a/test/security/k8s/src/check/raw/raw.go b/test/security/k8s/src/check/raw/raw.go index 555115950..91237ba82 100644 --- a/test/security/k8s/src/check/raw/raw.go +++ b/test/security/k8s/src/check/raw/raw.go @@ -46,6 +46,12 @@ func (r *Raw) GetControllerManagerParams() ([]string, error) { return getProcessParams(check.ControllerManagerProcess) } +// GetEtcdParams returns parameters of running etcd. +// It queries only cluster nodes with "controlplane" role. +func (r *Raw) GetEtcdParams() ([]string, error) { + return getProcessParams(check.EtcdProcess) +} + func getProcessParams(process check.Command) ([]string, error) { nodes, err := config.GetNodesInfo() if err != nil { @@ -61,6 +67,10 @@ func getProcessParams(process check.Command) ([]string, error) { cmd = trimOutput(cmd) // TODO: improve `docker inspect` query format. if len(cmd) > 0 { + if process == check.EtcdProcess { // etcd process name is not included in its argument list. + return btos(cmd), nil + } + i := bytes.Index(cmd, []byte(process.String())) if i == -1 { return []string{}, fmt.Errorf("missing %s command", process) diff --git a/test/security/k8s/src/check/validators/master/api/api_test.go b/test/security/k8s/src/check/validators/master/api/api_test.go index 4ba5070a8..01fe9b1c6 100644 --- a/test/security/k8s/src/check/validators/master/api/api_test.go +++ b/test/security/k8s/src/check/validators/master/api/api_test.go @@ -45,39 +45,7 @@ var _ = Describe("Api", func() { "TLS_RSA_WITH_AES_128_GCM_SHA256", } - // kubeApiServerCasablanca was obtained from virtual environment for testing - // (introduced in Change-Id: I57f9f3caac0e8b391e9ed480f6bebba98e006882). - kubeApiServerCasablanca = []string{ - "--storage-backend=etcd2", - "--storage-media-type=application/json", - "--service-cluster-ip-range=10.43.0.0/16", - "--etcd-servers=https://etcd.kubernetes.rancher.internal:2379", - "--insecure-bind-address=0.0.0.0", - "--insecure-port=0", - "--cloud-provider=rancher", - "--allow-privileged=true", - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount," + - "PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota", - "--client-ca-file=/etc/kubernetes/ssl/ca.pem", - "--tls-cert-file=/etc/kubernetes/ssl/cert.pem", - "--tls-private-key-file=/etc/kubernetes/ssl/key.pem", - "--kubelet-client-certificate=/etc/kubernetes/ssl/cert.pem", - "--kubelet-client-key=/etc/kubernetes/ssl/key.pem", - "--runtime-config=batch/v2alpha1", - "--anonymous-auth=false", - "--authentication-token-webhook-config-file=/etc/kubernetes/authconfig", - "--runtime-config=authentication.k8s.io/v1beta1=true", - "--external-hostname=kubernetes.kubernetes.rancher.internal", - "--etcd-cafile=/etc/kubernetes/etcd/ca.pem", - "--etcd-certfile=/etc/kubernetes/etcd/cert.pem", - "--etcd-keyfile=/etc/kubernetes/etcd/key.pem", - "--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," + - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305," + - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," + - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", - } - - // kubeApiServerCasablanca was obtained from virtual environment for testing + // kubeApiServerDublin was obtained from virtual environment for testing // (introduced in Change-Id: I54ada5fade3b984dedd1715f20579e3ce901faa3). kubeApiServerDublin = []string{ "--requestheader-group-headers=X-Remote-Group", @@ -130,7 +98,6 @@ var _ = Describe("Api", func() { }, Entry("Is not absent on insecure cluster", []string{"--insecure-allow-any-token"}, false), Entry("Should be absent on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be absent on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be absent on Dublin cluster", kubeApiServerDublin, true), ) @@ -140,7 +107,6 @@ var _ = Describe("Api", func() { }, Entry("Is not set on insecure cluster", []string{}, false), Entry("Should be set to false on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be set to false on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be set to false on Dublin cluster", kubeApiServerDublin, true), ) @@ -150,7 +116,6 @@ var _ = Describe("Api", func() { }, Entry("Is not set on insecure cluster", []string{}, false), Entry("Is explicitly enabled on insecure cluster", []string{"--profiling=true"}, false), - Entry("Is not set on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should be set to false on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should be set to false on Dublin cluster", kubeApiServerDublin, true), ) @@ -161,7 +126,6 @@ var _ = Describe("Api", func() { }, Entry("Is explicitly disabled on insecure cluster", []string{"--kubelet-https=false"}, false), Entry("Should be absent or set to true on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be absent or set to true on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be absent or set to true on Dublin cluster", kubeApiServerDublin, true), ) @@ -171,7 +135,6 @@ var _ = Describe("Api", func() { }, Entry("Is not set on insecure cluster", []string{}, false), Entry("Is explicitly enabled on insecure cluster", []string{"--repair-malformed-updates=true"}, false), - Entry("Is not set on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should be set to false on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should be set to false on Dublin cluster", kubeApiServerDublin, true), ) @@ -182,7 +145,6 @@ var _ = Describe("Api", func() { }, Entry("Is not set on insecure cluster", []string{}, false), Entry("Is explicitly disabled on insecure cluster", []string{"--service-account-lookup=false"}, false), - Entry("Is not set on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should be set to true on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should be set to true on Dublin cluster", kubeApiServerDublin, true), ) @@ -195,7 +157,6 @@ var _ = Describe("Api", func() { }, Entry("Is not absent on insecure cluster", []string{"--basic-auth-file=/path/to/file"}, false), Entry("Should be absent on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be absent on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be absent on Dublin cluster", kubeApiServerDublin, true), ) @@ -205,7 +166,6 @@ var _ = Describe("Api", func() { }, Entry("Is not absent on insecure cluster", []string{"--token-auth-file=/path/to/file"}, false), Entry("Should be absent on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be absent on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be absent on Dublin cluster", kubeApiServerDublin, true), ) @@ -215,7 +175,6 @@ var _ = Describe("Api", func() { }, Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--audit-log-path="}, false), - Entry("Is absent on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is absent on Dublin cluster", kubeApiServerDublin, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -226,7 +185,6 @@ var _ = Describe("Api", func() { }, Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--kubelet-certificate-authority="}, false), - Entry("Is absent on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is absent on Dublin cluster", kubeApiServerDublin, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -238,7 +196,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--client-ca-file="}, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be present on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) @@ -249,7 +206,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"-etcd-cafile="}, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be present on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) @@ -259,7 +215,6 @@ var _ = Describe("Api", func() { }, Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--service-account-key-file="}, false), - Entry("Is absent on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) @@ -271,7 +226,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--kubelet-client-certificate= --kubelet-client-key="}, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be present on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) @@ -282,7 +236,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--etcd-certfile= --etcd-keyfile="}, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be present on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) @@ -293,7 +246,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--tls-cert-file= --tls-private-key-file="}, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be present on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) }) @@ -304,7 +256,6 @@ var _ = Describe("Api", func() { Expect(IsInsecureBindAddressAbsentOrLoopback(params)).To(Equal(expected)) }, Entry("Is not absent on insecure cluster", []string{"--insecure-bind-address=1.2.3.4"}, false), - Entry("Is not absent nor set to loopback on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should be absent or set to loopback on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should be absent or set to loopback on Dublin cluster", kubeApiServerDublin, true), ) @@ -316,7 +267,6 @@ var _ = Describe("Api", func() { Entry("Is not set on insecure cluster", []string{}, false), Entry("Is explicitly enabled on insecure cluster", []string{"--insecure-port=1234"}, false), Entry("Should be set to 0 on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be set to 0 on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be set to 0 on Dublin cluster", kubeApiServerDublin, true), ) @@ -326,7 +276,6 @@ var _ = Describe("Api", func() { }, Entry("Is explicitly disabled on insecure cluster", []string{"--secure-port=0"}, false), Entry("Should be absent or set to valid port on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be absent or set to valid port on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be absent or set to valid port on Dublin cluster", kubeApiServerDublin, true), ) }) @@ -339,7 +288,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--audit-log-maxage="}, false), Entry("Is insufficient on insecure cluster", []string{"--audit-log-maxage=5"}, false), - Entry("Is absent on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is absent on Dublin cluster", kubeApiServerDublin, false), Entry("Should be set appropriately on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -351,7 +299,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--audit-log-maxbackup="}, false), Entry("Is insufficient on insecure cluster", []string{"--audit-log-maxbackup=2"}, false), - Entry("Is absent on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is absent on Dublin cluster", kubeApiServerDublin, false), Entry("Should be set appropriately on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -363,7 +310,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--audit-log-maxsize="}, false), Entry("Is insufficient on insecure cluster", []string{"--audit-log-maxsize=5"}, false), - Entry("Is absent on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is absent on Dublin cluster", kubeApiServerDublin, false), Entry("Should be set appropriately on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -375,7 +321,6 @@ var _ = Describe("Api", func() { Entry("Is empty on insecure cluster", []string{"--request-timeout="}, false), Entry("Is too high on insecure cluster", []string{"--request-timeout=600"}, false), Entry("Should be set only if needed on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be set only if needed on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be set only if needed on Dublin cluster", kubeApiServerDublin, true), ) }) @@ -388,7 +333,6 @@ var _ = Describe("Api", func() { Entry("Is not absent on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar,AlwaysAdmit,Baz,Quuz"}, false), Entry("Is not absent on insecure deprecated cluster", []string{"--admission-control=Foo,Bar,AlwaysAdmit,Baz,Quuz"}, false), Entry("Should be absent on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be absent on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be absent on Dublin cluster", kubeApiServerDublin, true), ) @@ -398,7 +342,6 @@ var _ = Describe("Api", func() { }, Entry("Is not present on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar"}, false), Entry("Is not present on insecure deprecated cluster", []string{"--admission-control=Foo,Bar"}, false), - Entry("Is not present on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is not present on Dublin cluster", kubeApiServerDublin, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -409,7 +352,6 @@ var _ = Describe("Api", func() { }, Entry("Is not present on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar"}, false), Entry("Is not present on insecure deprecated cluster", []string{"--admission-control=Foo,Bar"}, false), - Entry("Is not present on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is not present on Dublin cluster", kubeApiServerDublin, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -420,7 +362,6 @@ var _ = Describe("Api", func() { }, Entry("Is not present on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar"}, false), Entry("Is not present on insecure deprecated cluster", []string{"--admission-control=Foo,Bar"}, false), - Entry("Is not present on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is not present on Dublin cluster", kubeApiServerDublin, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -431,7 +372,6 @@ var _ = Describe("Api", func() { }, Entry("Is not present on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar"}, false), Entry("Is not present on insecure deprecated cluster", []string{"--admission-control=Foo,Bar"}, false), - Entry("Is not present on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is not present on Dublin cluster", kubeApiServerDublin, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -443,7 +383,6 @@ var _ = Describe("Api", func() { Entry("Is not present on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar"}, false), Entry("Is not present on insecure deprecated cluster", []string{"--admission-control=Foo,Bar"}, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should be present on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) @@ -453,7 +392,6 @@ var _ = Describe("Api", func() { }, Entry("Is not present on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar"}, false), Entry("Is not present on insecure deprecated cluster", []string{"--admission-control=Foo,Bar"}, false), - Entry("Is not present on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should be present on Dublin cluster", kubeApiServerDublin, true), ) @@ -464,7 +402,6 @@ var _ = Describe("Api", func() { }, Entry("Is not present on insecure cluster", []string{"--enable-admission-plugins=Foo,Bar"}, false), Entry("Is not present on insecure deprecated cluster", []string{"--admission-control=Foo,Bar"}, false), - Entry("Is not present on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is not present on Dublin cluster", kubeApiServerDublin, false), Entry("Should be present on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) @@ -475,7 +412,6 @@ var _ = Describe("Api", func() { }, Entry("Is explicitly disabled on insecure cluster", []string{"--disable-admission-plugins=Foo,Bar,NamespaceLifecycle,Baz,Quuz"}, false), Entry("Should not be disabled on CIS-compliant cluster", kubeApiServerCISCompliant, true), - Entry("Should not be disabled on Casablanca cluster", kubeApiServerCasablanca, true), Entry("Should not be disabled on Dublin cluster", kubeApiServerDublin, true), ) @@ -485,7 +421,6 @@ var _ = Describe("Api", func() { }, Entry("Is not explicitly disabled on insecure cluster", []string{}, false), Entry("Is not absent on insecure cluster", []string{"--authorization-mode=Foo,Bar,AlwaysAllow,Baz,Quuz"}, false), - Entry("Is not explicitly disabled on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should be absent on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should be absent on Dublin cluster", kubeApiServerDublin, true), ) @@ -496,7 +431,6 @@ var _ = Describe("Api", func() { }, Entry("Is not explicitly enabled on insecure cluster", []string{}, false), Entry("Is not present on insecure cluster", []string{"--authorization-mode=Foo,Bar"}, false), - Entry("Is not explicitly enabled on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Should present on CIS-compliant cluster", kubeApiServerCISCompliant, true), Entry("Should present on Dublin cluster", kubeApiServerDublin, true), ) @@ -510,7 +444,6 @@ var _ = Describe("Api", func() { Entry("Is absent on insecure cluster", []string{}, false), Entry("Is empty on insecure cluster", []string{"--tls-cipher-suites="}, false), Entry("Is incomplete on insecure cluster", []string{"--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"}, false), - Entry("Is incomplete on Casablanca cluster", kubeApiServerCasablanca, false), Entry("Is incomplete on Dublin cluster", kubeApiServerDublin, false), Entry("Should be complete on CIS-compliant cluster", kubeApiServerCISCompliant, true), ) diff --git a/test/security/k8s/src/check/validators/master/controllermanager/controllermanager.go b/test/security/k8s/src/check/validators/master/controllermanager/controllermanager.go index f1dd0fe49..4629ad86f 100644 --- a/test/security/k8s/src/check/validators/master/controllermanager/controllermanager.go +++ b/test/security/k8s/src/check/validators/master/controllermanager/controllermanager.go @@ -25,3 +25,18 @@ func IsInsecureBindAddressAbsentOrLoopback(params []string) bool { return boolean.IsFlagAbsent("--address=", params) || args.HasSingleFlagArgument("--address=", "127.0.0.1", params) } + +// IsTerminatedPodGcThresholdValid validates terminated pod garbage collector threshold is set and it has non-empty argument. +func IsTerminatedPodGcThresholdValid(params []string) bool { + return args.HasSingleFlagNonemptyArgument("--terminated-pod-gc-threshold", params) +} + +// IsServiceAccountPrivateKeyFileSet validates service account private key is set and it has non-empty argument. +func IsServiceAccountPrivateKeyFileSet(params []string) bool { + return args.HasSingleFlagNonemptyArgument("--service-account-private-key-file", params) +} + +// IsRootCertificateAuthoritySet validates root certificate authority is set and it has non-empty argument. +func IsRootCertificateAuthoritySet(params []string) bool { + return args.HasSingleFlagNonemptyArgument("--root-ca-file", params) +} diff --git a/test/security/k8s/src/check/validators/master/controllermanager/controllermanager_test.go b/test/security/k8s/src/check/validators/master/controllermanager/controllermanager_test.go index 7fd8b5d53..05e3cae7e 100644 --- a/test/security/k8s/src/check/validators/master/controllermanager/controllermanager_test.go +++ b/test/security/k8s/src/check/validators/master/controllermanager/controllermanager_test.go @@ -16,21 +16,12 @@ var _ = Describe("Controllermanager", func() { "--profiling=false", "--use-service-account-credentials=true", "--feature-gates=RotateKubeletServerCertificate=true", + "--terminated-pod-gc-threshold=10", + "--service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem", + "--root-ca-file=/etc/kubernetes/ssl/kube-ca.pem", } - // kubeControllerManagerCasablanca was obtained from virtual environment for testing - // (introduced in Change-Id: I57f9f3caac0e8b391e9ed480f6bebba98e006882). - kubeControllerManagerCasablanca = []string{ - "--kubeconfig=/etc/kubernetes/ssl/kubeconfig", - "--address=0.0.0.0", - "--root-ca-file=/etc/kubernetes/ssl/ca.pem", - "--service-account-private-key-file=/etc/kubernetes/ssl/key.pem", - "--allow-untagged-cloud", - "--cloud-provider=rancher", - "--horizontal-pod-autoscaler-use-rest-clients=false", - } - - // kubeControllerManagerCasablanca was obtained from virtual environment for testing + // kubeControllerManagerDublin was obtained from virtual environment for testing // (introduced in Change-Id: I54ada5fade3b984dedd1715f20579e3ce901faa3). kubeControllerManagerDublin = []string{ "--kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml", @@ -61,7 +52,6 @@ var _ = Describe("Controllermanager", func() { }, Entry("Is not set on insecure cluster", []string{}, false), Entry("Is explicitly enabled on insecure cluster", []string{"--profiling=true"}, false), - Entry("Is not set on Casablanca cluster", kubeControllerManagerCasablanca, false), Entry("Should be set to false on CIS-compliant cluster", kubeControllerManagerCISCompliant, true), Entry("Should be set to false on Dublin cluster", kubeControllerManagerDublin, true), ) @@ -72,24 +62,56 @@ var _ = Describe("Controllermanager", func() { }, Entry("Is not set on insecure cluster", []string{}, false), Entry("Is explicitly disabled on insecure cluster", []string{"--use-service-account-credentials=false"}, false), - Entry("Is not set on Casablanca cluster", kubeControllerManagerCasablanca, false), Entry("Should be set to true on CIS-compliant cluster", kubeControllerManagerCISCompliant, true), Entry("Should be set to true on Dublin cluster", kubeControllerManagerDublin, true), ) }) + Describe("File path flags", func() { + DescribeTable("Service account private key", + func(params []string, expected bool) { + Expect(IsServiceAccountPrivateKeyFileSet(params)).To(Equal(expected)) + }, + Entry("Is absent on insecure cluster", []string{""}, false), + Entry("Is empty on insecure cluster", []string{"--service-account-private-key-file="}, false), + Entry("Should be explicitly set on CIS-compliant cluster", kubeControllerManagerCISCompliant, true), + Entry("Should be explicitly set on Dublin cluster", kubeControllerManagerDublin, true), + ) + + DescribeTable("Root certificate authority", + func(params []string, expected bool) { + Expect(IsRootCertificateAuthoritySet(params)).To(Equal(expected)) + }, + Entry("Is absent on insecure cluster", []string{""}, false), + Entry("Is empty on insecure cluster", []string{"--root-ca-file="}, false), + Entry("Should be explicitly set on CIS-compliant cluster", kubeControllerManagerCISCompliant, true), + Entry("Should be explicitly set on Dublin cluster", kubeControllerManagerDublin, true), + ) + }) + Describe("Address flag", func() { DescribeTable("Bind address", func(params []string, expected bool) { Expect(IsInsecureBindAddressAbsentOrLoopback(params)).To(Equal(expected)) }, Entry("Is not absent on insecure cluster", []string{"--address=1.2.3.4"}, false), - Entry("Is not absent nor set to loopback on Casablanca cluster", kubeControllerManagerCasablanca, false), Entry("Is not absent nor set to loopback on Dublin cluster", kubeControllerManagerDublin, false), Entry("Should be absent or set to loopback on CIS-compliant cluster", kubeControllerManagerCISCompliant, true), ) }) + Describe("Numeric flags", func() { + DescribeTable("Terminated pod garbage collector threshold", + func(params []string, expected bool) { + Expect(IsTerminatedPodGcThresholdValid(params)).To(Equal(expected)) + }, + Entry("Is absent on insecure cluster", []string{""}, false), + Entry("Is empty on insecure cluster", []string{"--terminated-pod-gc-threshold="}, false), + Entry("Should be explicitly set on CIS-compliant cluster", kubeControllerManagerCISCompliant, true), + Entry("Should be explicitly set on Dublin cluster", kubeControllerManagerDublin, true), + ) + }) + Describe("Argument list flags", func() { DescribeTable("RotateKubeletServerCertificate", func(params []string, expected bool) { @@ -97,7 +119,6 @@ var _ = Describe("Controllermanager", func() { }, Entry("Is not enabled on insecure cluster", []string{"--feature-gates=Foo=Bar,Baz=Quuz"}, false), Entry("Is explicitly disabled on insecure cluster", []string{"--feature-gates=Foo=Bar,RotateKubeletServerCertificate=false,Baz=Quuz"}, false), - Entry("Is not enabled on Casablanca cluster", kubeControllerManagerCasablanca, false), Entry("Is not enabled on Dublin cluster", kubeControllerManagerDublin, false), Entry("Should be enabled on CIS-compliant cluster", kubeControllerManagerCISCompliant, true), ) diff --git a/test/security/k8s/src/check/validators/master/master.go b/test/security/k8s/src/check/validators/master/master.go index 0f668f614..11c1b5052 100644 --- a/test/security/k8s/src/check/validators/master/master.go +++ b/test/security/k8s/src/check/validators/master/master.go @@ -70,7 +70,10 @@ func CheckScheduler(params []string) { func CheckControllerManager(params []string) { log.Println("==> Controller Manager:") log.Printf("IsProfilingDisabled: %t\n", controllermanager.IsProfilingDisabled(params)) + log.Printf("IsTerminatedPodGcThresholdValid: %t\n", controllermanager.IsTerminatedPodGcThresholdValid(params)) log.Printf("IsUseServiceAccountCredentialsEnabled: %t\n", controllermanager.IsUseServiceAccountCredentialsEnabled(params)) log.Printf("IsRotateKubeletServerCertificateIncluded: %t\n", controllermanager.IsRotateKubeletServerCertificateIncluded(params)) + log.Printf("IsServiceAccountPrivateKeyFileSet: %t\n", controllermanager.IsServiceAccountPrivateKeyFileSet(params)) + log.Printf("IsRootCertificateAuthoritySet: %t\n", controllermanager.IsRootCertificateAuthoritySet(params)) log.Printf("IsInsecureBindAddressAbsentOrLoopback: %t\n", controllermanager.IsInsecureBindAddressAbsentOrLoopback(params)) } diff --git a/test/security/k8s/src/check/validators/master/scheduler/scheduler_test.go b/test/security/k8s/src/check/validators/master/scheduler/scheduler_test.go index 4166a58d7..7fb13b820 100644 --- a/test/security/k8s/src/check/validators/master/scheduler/scheduler_test.go +++ b/test/security/k8s/src/check/validators/master/scheduler/scheduler_test.go @@ -16,14 +16,7 @@ var _ = Describe("Scheduler", func() { "--profiling=false", } - // kubeSchedulerCasablanca was obtained from virtual environment for testing - // (introduced in Change-Id: I57f9f3caac0e8b391e9ed480f6bebba98e006882). - kubeSchedulerCasablanca = []string{ - "--kubeconfig=/etc/kubernetes/ssl/kubeconfig", - "--address=0.0.0.0", - } - - // kubeSchedulerCasablanca was obtained from virtual environment for testing + // kubeSchedulerDublin was obtained from virtual environment for testing // (introduced in Change-Id: I54ada5fade3b984dedd1715f20579e3ce901faa3). kubeSchedulerDublin = []string{ "--kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml", @@ -41,7 +34,6 @@ var _ = Describe("Scheduler", func() { }, Entry("Is not set on insecure cluster", []string{}, false), Entry("Is explicitly enabled on insecure cluster", []string{"--profiling=true"}, false), - Entry("Is not set on Casablanca cluster", kubeSchedulerCasablanca, false), Entry("Should be set to false on CIS-compliant cluster", kubeSchedulerCISCompliant, true), Entry("Should be set to false on Dublin cluster", kubeSchedulerDublin, true), ) @@ -53,7 +45,6 @@ var _ = Describe("Scheduler", func() { Expect(IsInsecureBindAddressAbsentOrLoopback(params)).To(Equal(expected)) }, Entry("Is not absent on insecure cluster", []string{"--address=1.2.3.4"}, false), - Entry("Is not absent nor set to loopback on Casablanca cluster", kubeSchedulerCasablanca, false), Entry("Is not absent nor set to loopback on Dublin cluster", kubeSchedulerDublin, false), Entry("Should be absent or set to loopback on CIS-compliant cluster", kubeSchedulerCISCompliant, true), ) diff --git a/test/security/k8s/tools/casablanca/get_customization_scripts.sh b/test/security/k8s/tools/casablanca/get_customization_scripts.sh deleted file mode 100755 index 028f002fc..000000000 --- a/test/security/k8s/tools/casablanca/get_customization_scripts.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -wget \ - 'https://docs.onap.org/en/casablanca/_downloads/0b365a2342af5abd655f1724b962f5b5/openstack-rancher.sh' \ - 'https://docs.onap.org/en/casablanca/_downloads/b20b581d56982e9f15a72527a358d56b/openstack-k8s-node.sh' diff --git a/test/security/k8s/tools/casablanca/get_ranchercli.sh b/test/security/k8s/tools/casablanca/get_ranchercli.sh deleted file mode 100755 index 8ffbc5f58..000000000 --- a/test/security/k8s/tools/casablanca/get_ranchercli.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# -# @file test/security/k8s/tools/casablanca/get_ranchercli.sh -# @author Pawel Wieczorek <p.wieczorek2@samsung.com> -# @brief Utility for obtaining Rancher CLI tool -# - -# Dependencies: -# wget -# tar -# coreutils -# -# Privileges: -# Script expects to be run with administrative privileges for accessing /usr/local/bin -# -# Usage: -# # ./get_ranchercli.sh [VERSION [ARCH [SYSTEM]]] -# - -# Constants -DEFAULT_VERSION='v0.6.12' -DEFAULT_ARCH='amd64' -DEFAULT_SYSTEM='linux' - -# Variables -VERSION="${1:-$DEFAULT_VERSION}" -ARCH="${2:-$DEFAULT_ARCH}" -SYSTEM="${3:-$DEFAULT_SYSTEM}" - -ARCHIVE="rancher-${SYSTEM}-${ARCH}-${VERSION}.tar.gz" -DIRECTORY="rancher-${VERSION}" -URL="https://releases.rancher.com/cli/${VERSION}/${ARCHIVE}" - - -# Prerequistes -wget "$URL" -tar xf "$ARCHIVE" - -# Installation -mv "${DIRECTORY}/rancher" /usr/local/bin/ - -# Cleanup -rmdir "$DIRECTORY" -rm "$ARCHIVE" diff --git a/test/security/k8s/tools/casablanca/imported/openstack-k8s-node.sh b/test/security/k8s/tools/casablanca/imported/openstack-k8s-node.sh deleted file mode 100644 index b8462aa5e..000000000 --- a/test/security/k8s/tools/casablanca/imported/openstack-k8s-node.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -DOCKER_VERSION=17.03 -KUBECTL_VERSION=1.11.2 -HELM_VERSION=2.9.1 - -# setup root access - default login: oom/oom - comment out to restrict access too ssh key only -sed -i 's/PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config -sed -i 's/PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config -service sshd restart -echo -e "oom\noom" | passwd root - -apt-get update -curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh -mkdir -p /etc/systemd/system/docker.service.d/ -cat > /etc/systemd/system/docker.service.d/docker.conf << EOF -[Service] -ExecStart= -ExecStart=/usr/bin/dockerd -H fd:// --insecure-registry=nexus3.onap.org:10001 -EOF -systemctl daemon-reload -systemctl restart docker -apt-mark hold docker-ce - -IP_ADDY=`ip address |grep ens|grep inet|awk '{print $2}'| awk -F / '{print $1}'` -HOSTNAME=`hostname` - -echo "$IP_ADDY $HOSTNAME" >> /etc/hosts - -docker login -u docker -p docker nexus3.onap.org:10001 - -sudo apt-get install make -y - -sudo curl -LO https://storage.googleapis.com/kubernetes-release/release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl -sudo chmod +x ./kubectl -sudo mv ./kubectl /usr/local/bin/kubectl -sudo mkdir ~/.kube -wget http://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz -sudo tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz -sudo mv linux-amd64/helm /usr/local/bin/helm - -# install nfs -sudo apt-get install nfs-common -y - - -exit 0 diff --git a/test/security/k8s/tools/casablanca/imported/openstack-rancher.sh b/test/security/k8s/tools/casablanca/imported/openstack-rancher.sh deleted file mode 100644 index bcf542aed..000000000 --- a/test/security/k8s/tools/casablanca/imported/openstack-rancher.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -DOCKER_VERSION=17.03 -RANCHER_VERSION=1.6.22 -KUBECTL_VERSION=1.11.2 -HELM_VERSION=2.9.1 - -# setup root access - default login: oom/oom - comment out to restrict access too ssh key only -sed -i 's/PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config -sed -i 's/PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config -service sshd restart -echo -e "oom\noom" | passwd root - -apt-get update -curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh -mkdir -p /etc/systemd/system/docker.service.d/ -cat > /etc/systemd/system/docker.service.d/docker.conf << EOF -[Service] -ExecStart= -ExecStart=/usr/bin/dockerd -H fd:// --insecure-registry=nexus3.onap.org:10001 -EOF -systemctl daemon-reload -systemctl restart docker -apt-mark hold docker-ce - -IP_ADDY=`ip address |grep ens|grep inet|awk '{print $2}'| awk -F / '{print $1}'` -HOSTNAME=`hostname` - -echo "$IP_ADDY $HOSTNAME" >> /etc/hosts - -docker login -u docker -p docker nexus3.onap.org:10001 - -sudo apt-get install make -y - -sudo docker run -d --restart=unless-stopped -p 8080:8080 --name rancher_server rancher/server:v$RANCHER_VERSION -sudo curl -LO https://storage.googleapis.com/kubernetes-release/release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl -sudo chmod +x ./kubectl -sudo mv ./kubectl /usr/local/bin/kubectl -sudo mkdir ~/.kube -wget http://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz -sudo tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz -sudo mv linux-amd64/helm /usr/local/bin/helm - -# nfs server -sudo apt-get install nfs-kernel-server -y - -sudo mkdir -p /nfs_share -sudo chown nobody:nogroup /nfs_share/ - - -exit 0 diff --git a/test/security/k8s/vagrant/casablanca/Vagrantfile b/test/security/k8s/vagrant/casablanca/Vagrantfile deleted file mode 100644 index c7e6883ce..000000000 --- a/test/security/k8s/vagrant/casablanca/Vagrantfile +++ /dev/null @@ -1,48 +0,0 @@ -# -*- mode: ruby -*- -# -*- coding: utf-8 -*- - -host_ip = "192.168.121.1" - -rancher_port = 8080 - -vm_memory = 2 * 1024 -vm_cpus = 1 -vm_box = "generic/ubuntu1604" - -cluster = [ - { name: 'master', hostname: 'master', ip: '172.17.3.100' }, - { name: 'worker', hostname: 'worker', ip: '172.17.3.101' } -] - -Vagrant.configure('2') do |config| - cluster.each do |node| - config.vm.define node[:name] do |config| - config.vm.box = vm_box - config.vm.hostname = node[:hostname] - - config.vm.provider :virtualbox do |v| - v.name = node[:name] - v.memory = vm_memory - v.cpus = vm_cpus - end - - config.vm.provider :libvirt do |v| - v.memory = vm_memory - v.cpus = vm_cpus - end - - config.vm.network :private_network, ip: node[:ip] - config.vm.provision :shell, inline: "echo nameserver #{host_ip} | resolvconf -a eth0.inet" - - if node[:name] == 'master' - config.vm.network "forwarded_port", guest: rancher_port, host: rancher_port - config.vm.provision :shell, path: "../../tools/casablanca/imported/openstack-rancher.sh" - config.vm.provision :shell, path: "../../tools/casablanca/get_ranchercli.sh" - end - - if node[:name] == 'worker' - config.vm.provision :shell, path: "../../tools/casablanca/imported/openstack-k8s-node.sh" - end - end - end -end diff --git a/test/security/k8s/vagrant/dublin/cluster.yml b/test/security/k8s/vagrant/dublin/cluster.yml index df93a8863..e4eef11cd 100644 --- a/test/security/k8s/vagrant/dublin/cluster.yml +++ b/test/security/k8s/vagrant/dublin/cluster.yml @@ -32,7 +32,6 @@ network: plugin: canal authentication: strategy: x509 -ssh_key_path: *ssh_key_path ssh_agent_auth: false authorization: mode: rbac diff --git a/test/security/requirements.txt b/test/security/requirements.txt new file mode 100644 index 000000000..8683da168 --- /dev/null +++ b/test/security/requirements.txt @@ -0,0 +1,2 @@ +kubernetes +colorama diff --git a/test/security/setup.py b/test/security/setup.py new file mode 100644 index 000000000..7e71bda03 --- /dev/null +++ b/test/security/setup.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2020 Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +import setuptools + +setuptools.setup( + install_requires=[ + 'kubernetes', + 'colorama' + ] +) diff --git a/test/security/sslendpoints/.dockerignore b/test/security/sslendpoints/.dockerignore new file mode 100644 index 000000000..7bc4c9048 --- /dev/null +++ b/test/security/sslendpoints/.dockerignore @@ -0,0 +1,5 @@ +.git* +.dockerignore +Dockerfile +Makefile +bin/ diff --git a/test/security/sslendpoints/.gitignore b/test/security/sslendpoints/.gitignore new file mode 100644 index 000000000..035097d0f --- /dev/null +++ b/test/security/sslendpoints/.gitignore @@ -0,0 +1,2 @@ +# No binaries should be committed. +/bin/ diff --git a/test/security/sslendpoints/Dockerfile b/test/security/sslendpoints/Dockerfile new file mode 100644 index 000000000..415101e44 --- /dev/null +++ b/test/security/sslendpoints/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.13.8 AS build + +WORKDIR /opt/onap.local/sslendpoints +COPY . /opt/onap.local/sslendpoints +RUN go mod download +RUN CGO_ENABLED=0 go build -ldflags '-extldflags "-static"' + +FROM scratch +COPY --from=build /opt/onap.local/sslendpoints/sslendpoints /bin/sslendpoints +ENTRYPOINT ["/bin/sslendpoints"] +CMD ["--help"] diff --git a/test/security/sslendpoints/Makefile b/test/security/sslendpoints/Makefile new file mode 100644 index 000000000..e853dc4dd --- /dev/null +++ b/test/security/sslendpoints/Makefile @@ -0,0 +1,47 @@ +BUILD_DIR = bin +BUILD_DOCKER_IMAGE = sslendpoints-build-img +BUILD_DOCKER_CONTAINER = sslendpoints-build +BINARIES = sslendpoints +BINARIES := $(addprefix ${BUILD_DIR}/, ${BINARIES}) + +.PHONY: all +all: docker-build + +.PHONY: build +build: ${BUILD_DIR} + go build -o "${BINARIES}" + +.PHONY: clean +clean: clean-docker-build clean-build + +.PHONY: test +test: + go test ./... + +.PHONY: docker-build +docker-build: ${BINARIES} + docker rm "${BUILD_DOCKER_CONTAINER}" + +${BINARIES}: docker-container | ${BUILD_DIR} + docker cp "${BUILD_DOCKER_CONTAINER}:/$@" $@ + +.PHONY: docker-container +docker-container: docker-build-img + docker create --name "${BUILD_DOCKER_CONTAINER}" "${BUILD_DOCKER_IMAGE}" + +.PHONY: docker-build-img +docker-build-img: + docker build --tag "${BUILD_DOCKER_IMAGE}" . + +${BUILD_DIR}: + mkdir -p "${BUILD_DIR}" + +.PHONY: clean-docker-build +clean-docker-build: + -docker rm "${BUILD_DOCKER_CONTAINER}" + -docker rmi "${BUILD_DOCKER_IMAGE}" + +.PHONY: clean-build +clean-build: + -rm -f ${BINARIES} + -rmdir ${BUILD_DIR} diff --git a/test/security/sslendpoints/README.rst b/test/security/sslendpoints/README.rst new file mode 100644 index 000000000..257946d88 --- /dev/null +++ b/test/security/sslendpoints/README.rst @@ -0,0 +1,135 @@ +===================== + SSL endpoints check +===================== + +Utility for checking if all of the ports exposed outside of Kubernetes cluster +use SSL tunnels. + +Prerequisites +------------- + +Configuration +~~~~~~~~~~~~~ + +``-kubeconfig`` + Optional unless ``$HOME`` is not set. Defaults to ``$HOME/.kube/config``. + +``-xfail`` + Optional list of services with corresponding NodePorts which do not use SSL + tunnels. These ports are known as "expected failures" and will not be + checked. + +Dependencies +~~~~~~~~~~~~ + +- nmap_ + +.. _nmap: https://nmap.org/book/install.html + +Build (local) +~~~~~~~~~~~~~ + +- go_ (1.11+, tested on 1.13) + +.. _go: https://golang.org/doc/install + +Build (Docker) +~~~~~~~~~~~~~~ + +- Docker_ engine +- make (optional) + +.. _Docker: https://docs.docker.com/install + +Test +~~~~ + +- Ginkgo_ +- GolangCI-Lint_ (optional) + +.. _Ginkgo: https://onsi.github.io/ginkgo/#getting-ginkgo +.. _GolangCI-Lint: https://github.com/golangci/golangci-lint#install + +Building +-------- + +Command (local) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ mkdir bin + $ go build -o bin/sslendpoints + +Additional ``bin`` directory and specifying ``go build`` output are used to +declutter project and maintain compatibility with Docker-based process. Running +``go build`` without parameters will create ``sslendpoints`` binary in current +directory. + +Command (Docker) +~~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ make # or commands from corresponding "make" targets + + +Running +------- + +Command (local) +~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ bin/sslendpoints [-kubeconfig KUBECONFIG] [-xfail XFAIL] + +Command (Docker) +~~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ docker run --rm --volume $KUBECONFIG:/.kube/config \ + sslendpoints-build-img /bin/sslendpoints + + $ docker run --rm --volume $KUBECONFIG:/opt/config \ + sslendpoints-build-img /bin/sslendpoints -kubeconfig /opt/config + + $ docker run --rm \ + --volume $KUBECONFIG:/opt/config \ + --volume $XFAIL:/opt/xfail \ + sslendpoints-build-img /bin/sslendpoints \ + -kubeconfig /opt/config + -xfail /opt/xfail + +Output +~~~~~~ + +.. code-block:: shell + + $ ./sslendpoints -kubeconfig ~/.kube/config.onap + 2020/03/17 10:40:29 Host 192.168.2.10 + 2020/03/17 10:40:29 PORT SERVICE + 2020/03/17 10:40:29 30203 sdnc-dgbuilder + 2020/03/17 10:40:29 30204 sdc-be + 2020/03/17 10:40:29 30207 sdc-fe + 2020/03/17 10:40:29 30220 aai-sparky-be + 2020/03/17 10:40:29 30226 message-router + 2020/03/17 10:40:29 30233 aai + 2020/03/17 10:40:29 30256 sdc-wfd-fe + 2020/03/17 10:40:29 30257 sdc-wfd-be + 2020/03/17 10:40:29 30264 sdc-dcae-fe + 2020/03/17 10:40:29 30266 sdc-dcae-dt + 2020/03/17 10:40:29 30279 aai-babel + 2020/03/17 10:40:29 30406 so-vnfm-adapter + 2020/03/17 10:40:29 There are 12 non-SSL NodePorts in the cluster + + +Testing +------- + +.. code-block:: shell + + $ go test ./... # basic + $ ginkgo -r # pretty + $ golangci-lint run # linters diff --git a/test/security/sslendpoints/go.mod b/test/security/sslendpoints/go.mod new file mode 100644 index 000000000..6037ee0ae --- /dev/null +++ b/test/security/sslendpoints/go.mod @@ -0,0 +1,16 @@ +module onap.local/sslendpoints + +go 1.13 + +require ( + github.com/Ullaakut/nmap v2.0.0+incompatible + github.com/imdario/mergo v0.3.8 // indirect + github.com/onsi/ginkgo v1.10.1 + github.com/onsi/gomega v1.7.0 + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + k8s.io/api v0.17.3 + k8s.io/apimachinery v0.17.3 + k8s.io/client-go v0.0.0-20190819141724-e14f31a72a77 + k8s.io/utils v0.0.0-20200124190032-861946025e34 // indirect +) diff --git a/test/security/sslendpoints/go.sum b/test/security/sslendpoints/go.sum new file mode 100644 index 000000000..2ed062aa4 --- /dev/null +++ b/test/security/sslendpoints/go.sum @@ -0,0 +1,168 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Ullaakut/nmap v2.0.0+incompatible h1:tNXub052dsnG8+yrgpph9nhVixIBdpRRgzvmQoc8eBA= +github.com/Ullaakut/nmap v2.0.0+incompatible/go.mod h1:fkC066hwfcoKwlI7DS2ARTggSVtBTZYCjVH1TzuTMaQ= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= +github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/api v0.0.0-20190819141258-3544db3b9e44/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= +k8s.io/api v0.17.3 h1:XAm3PZp3wnEdzekNkcmj/9Y1zdmQYJ1I4GKSBBZ8aG0= +k8s.io/api v0.17.3/go.mod h1:YZ0OTkuw7ipbe305fMpIdf3GLXZKRigjtZaV5gzC2J0= +k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d/go.mod h1:3jediapYqJ2w1BFw7lAZPCx7scubsTfosqHkhXCWJKw= +k8s.io/apimachinery v0.17.3 h1:f+uZV6rm4/tHE7xXgLyToprg6xWairaClGVkm2t8omg= +k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= +k8s.io/client-go v0.0.0-20190819141724-e14f31a72a77 h1:w1BoabVnPpPqQCY3sHK4qVwa12Lk8ip1pKMR1C+qbdo= +k8s.io/client-go v0.0.0-20190819141724-e14f31a72a77/go.mod h1:DmkJD5UDP87MVqUQ5VJ6Tj9Oen8WzXPhk3la4qpyG4g= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= +k8s.io/utils v0.0.0-20200124190032-861946025e34 h1:HjlUD6M0K3P8nRXmr2B9o4F9dUy9TCj/aEpReeyi6+k= +k8s.io/utils v0.0.0-20200124190032-861946025e34/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/test/security/sslendpoints/main.go b/test/security/sslendpoints/main.go new file mode 100644 index 000000000..ddad51c68 --- /dev/null +++ b/test/security/sslendpoints/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "encoding/csv" + "flag" + "log" + "os" + "path/filepath" + "strconv" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/Ullaakut/nmap" + + "onap.local/sslendpoints/ports" +) + +const ( + ipv4AddrType = "ipv4" + + xfailComma = ' ' + xfailComment = '#' + xfailFields = 2 +) + +var ( + kubeconfig *string + namespace *string + xfailName *string +) + +func main() { + if home := os.Getenv("HOME"); home != "" { + kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") + } else { + kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file") + } + namespace = flag.String("namespace", "", "(optional) name of specific namespace to scan") + xfailName = flag.String("xfail", "", "(optional) absolute path to the expected failures file") + flag.Parse() + + var listOptions metav1.ListOptions + if *namespace != "" { + listOptions = metav1.ListOptions{FieldSelector: "metadata.namespace=" + *namespace} + } + + var xfails map[uint16]string + if *xfailName != "" { + xfailFile, err := os.Open(*xfailName) + if err != nil { + log.Printf("Unable to open expected failures file: %v", err) + log.Println("All non-SSL NodePorts will be reported") + } + defer xfailFile.Close() + + r := csv.NewReader(xfailFile) + r.Comma = xfailComma + r.Comment = xfailComment + r.FieldsPerRecord = xfailFields + + xfailData, err := r.ReadAll() + if err != nil { + log.Printf("Unable to read expected failures file: %v", err) + log.Println("All non-SSL NodePorts will be reported") + } + + var ok bool + xfails, ok = ports.ConvertNodePorts(xfailData) + if !ok { + log.Println("No usable data in expected failures file") + log.Println("All non-SSL NodePorts will be reported") + } + } + + // use the current context in kubeconfig + config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + if err != nil { + log.Panicf("Unable to build cluster config: %v", err) + } + + // create the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Panicf("Unable to build client: %v", err) + } + + // get list of nodes to extract addresses for running scan + nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + log.Panicf("Unable to get list of nodes: %v", err) + } + + // filter out addresses for running scan + addresses, ok := ports.FilterIPAddresses(nodes) + if !ok { + log.Println("There are no IP addresses to run scan") + os.Exit(0) + } + + // get list of services to extract nodeport information + services, err := clientset.CoreV1().Services("").List(listOptions) + if err != nil { + log.Panicf("Unable to get list of services: %v", err) + } + + // filter out nodeports with corresponding services from service list + nodeports, ok := ports.FilterNodePorts(services) + if !ok { + log.Println("There are no NodePorts in the cluster") + os.Exit(0) + } + + // filter out expected failures here before running the scan + ports.FilterXFailNodePorts(xfails, nodeports) + + // extract ports for running the scan + var ports []string + for port := range nodeports { + ports = append(ports, strconv.Itoa(int(port))) + } + + // run nmap on the first address found for given cluster [1] filtering out SSL-tunelled ports + // [1] https://kubernetes.io/docs/concepts/services-networking/service/#nodeport + // "Each node proxies that port (the same port number on every Node) into your Service." + scanner, err := nmap.NewScanner( + nmap.WithTargets(addresses[0]), + nmap.WithPorts(ports...), + nmap.WithServiceInfo(), + nmap.WithTimingTemplate(nmap.TimingAggressive), + nmap.WithFilterPort(func(p nmap.Port) bool { + if p.Service.Tunnel == "ssl" { + return false + } + if strings.HasPrefix(p.State.State, "closed") { + return false + } + if strings.HasPrefix(p.State.State, "filtered") { + return false + } + return true + }), + ) + if err != nil { + log.Panicf("Unable to create nmap scanner: %v", err) + } + + result, _, err := scanner.Run() + if err != nil { + log.Panicf("Scan failed: %v", err) + } + + // scan was run on a single host + if len(result.Hosts) < 1 { + log.Panicln("No host information in scan results") + } + + // host address in the results might be ipv4 or mac + for _, address := range result.Hosts[0].Addresses { + if address.AddrType == ipv4AddrType { + log.Printf("Host %s\n", address) + } + } + log.Printf("PORT\tSERVICE") + for _, port := range result.Hosts[0].Ports { + log.Printf("%d\t%s\n", port.ID, nodeports[port.ID]) + } + + // report non-SSL services and their number + log.Printf("There are %d non-SSL NodePorts in the cluster\n", len(result.Hosts[0].Ports)) + os.Exit(len(result.Hosts[0].Ports)) +} diff --git a/test/security/sslendpoints/ports/ports.go b/test/security/sslendpoints/ports/ports.go new file mode 100644 index 000000000..dae7dbec7 --- /dev/null +++ b/test/security/sslendpoints/ports/ports.go @@ -0,0 +1,66 @@ +package ports + +import ( + "log" + "strconv" + + v1 "k8s.io/api/core/v1" +) + +// ConvertNodePorts converts CSV data to NodePorts map. +func ConvertNodePorts(data [][]string) (map[uint16]string, bool) { + result := make(map[uint16]string) + for _, record := range data { + port, err := strconv.Atoi(record[1]) + if err != nil { + log.Printf("Unable to parse port field: %v", err) + continue + } + result[uint16(port)] = record[0] + } + return result, len(result) > 0 +} + +// FilterXFailNodePorts removes NodePorts expected to fail from map. +func FilterXFailNodePorts(xfails, nodeports map[uint16]string) { + for port, xfailService := range xfails { + service, ok := nodeports[port] + if !ok { + continue + } + if service != xfailService { + continue + } + delete(nodeports, port) + } +} + +// FilterNodePorts extracts NodePorts from ServiceList. +func FilterNodePorts(services *v1.ServiceList) (map[uint16]string, bool) { + nodeports := make(map[uint16]string) + for _, service := range services.Items { + for _, port := range service.Spec.Ports { + if port.NodePort != 0 { + nodeports[uint16(port.NodePort)] = service.ObjectMeta.Name + } + } + } + return nodeports, len(nodeports) > 0 +} + +// FilterIPAddresses extracts IP addresses from NodeList. +// External IP addresses take precedence over internal ones. +func FilterIPAddresses(nodes *v1.NodeList) ([]string, bool) { + addresses := make([]string, 0) + for _, node := range nodes.Items { + for _, address := range node.Status.Addresses { + switch address.Type { + case "InternalIP": + addresses = append(addresses, address.Address) + case "ExternalIP": + addresses = append([]string{address.Address}, addresses...) + } + } + } + return addresses, len(addresses) > 0 +} diff --git a/test/security/sslendpoints/ports/ports_suite_test.go b/test/security/sslendpoints/ports/ports_suite_test.go new file mode 100644 index 000000000..8a6431e5e --- /dev/null +++ b/test/security/sslendpoints/ports/ports_suite_test.go @@ -0,0 +1,13 @@ +package ports_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestNodeports(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Nodeports Suite") +} diff --git a/test/security/sslendpoints/ports/ports_test.go b/test/security/sslendpoints/ports/ports_test.go new file mode 100644 index 000000000..2f4f042fa --- /dev/null +++ b/test/security/sslendpoints/ports/ports_test.go @@ -0,0 +1,423 @@ +package ports_test + +import ( + "strconv" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "onap.local/sslendpoints/ports" +) + +var _ = Describe("Ports", func() { + const ( + notNodePort = 0 + nodePortO = 30200 + nodePortN = 30201 + nodePortA = 30202 + nodePortP = 30203 + serviceR = "serviceR" + serviceL = "serviceL" + serviceZ = "serviceZ" + + notParsablePort1 = "3p1c" + notParsablePort2 = "0n4p" + notParsablePort3 = "5GxD" + + externalIpControl = "1.2.3.4" + internalIpControl = "192.168.121.100" + internalIpWorker = "192.168.121.200" + hostnameControl = "onap-control-1" + hostnameWorker = "onap-worker-1" + ) + + var ( + csvSomeUnparsable [][]string + csvAllUnparsable [][]string + + servicesEmpty *v1.ServiceList + servicesSingleWithNodePort *v1.ServiceList + servicesSingleWithMultipleNodePorts *v1.ServiceList + servicesManyWithoutNodePorts *v1.ServiceList + servicesManyWithNodePort *v1.ServiceList + servicesManyWithMultipleNodePorts *v1.ServiceList + servicesManyMixedNodePorts *v1.ServiceList + + nodesEmpty *v1.NodeList + nodesSingleWithIP *v1.NodeList + nodesSingleWithBothIPs *v1.NodeList + nodesManyWithHostnames *v1.NodeList + nodesManyWithMixedIPs *v1.NodeList + ) + + BeforeEach(func() { + csvSomeUnparsable = [][]string{ + {serviceR, strconv.Itoa(nodePortO)}, + {serviceL, strconv.Itoa(nodePortN)}, + {serviceZ, notParsablePort1}, + } + csvAllUnparsable = [][]string{ + {serviceR, notParsablePort1}, + {serviceL, notParsablePort2}, + {serviceZ, notParsablePort3}, + } + + servicesEmpty = &v1.ServiceList{} + servicesSingleWithNodePort = &v1.ServiceList{ + Items: []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{Name: serviceR}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortO}, + }, + }, + }, + }, + } + servicesSingleWithMultipleNodePorts = &v1.ServiceList{ + Items: []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{Name: serviceR}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortO}, + {NodePort: nodePortN}, + }, + }, + }, + }, + } + servicesManyWithoutNodePorts = &v1.ServiceList{ + Items: []v1.Service{ + { + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: notNodePort}, + }, + }, + }, + { + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: notNodePort}, + }, + }, + }, + }, + } + servicesManyWithNodePort = &v1.ServiceList{ + Items: []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{Name: serviceR}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortO}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: serviceL}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortN}, + }, + }, + }, + }, + } + servicesManyWithMultipleNodePorts = &v1.ServiceList{ + Items: []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{Name: serviceR}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortO}, + {NodePort: nodePortN}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: serviceL}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortA}, + {NodePort: nodePortP}, + }, + }, + }, + }, + } + servicesManyMixedNodePorts = &v1.ServiceList{ + Items: []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{Name: serviceR}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: notNodePort}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: serviceL}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortO}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: serviceZ}, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {NodePort: nodePortN}, + {NodePort: nodePortA}, + }, + }, + }, + }, + } + + nodesEmpty = &v1.NodeList{} + nodesSingleWithIP = &v1.NodeList{ + Items: []v1.Node{ + { + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: "InternalIP", Address: internalIpControl}, + {Type: "Hostname", Address: hostnameControl}, + }, + }, + }, + }, + } + nodesSingleWithBothIPs = &v1.NodeList{ + Items: []v1.Node{ + { + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: "ExternalIP", Address: externalIpControl}, + {Type: "InternalIP", Address: internalIpControl}, + {Type: "Hostname", Address: hostnameControl}, + }, + }, + }, + }, + } + nodesManyWithHostnames = &v1.NodeList{ + Items: []v1.Node{ + { + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: "Hostname", Address: hostnameControl}, + }, + }, + }, + { + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: "Hostname", Address: hostnameWorker}, + }, + }, + }, + }, + } + nodesManyWithMixedIPs = &v1.NodeList{ + Items: []v1.Node{ + { + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: "ExternalIP", Address: externalIpControl}, + {Type: "InternalIP", Address: internalIpControl}, + {Type: "Hostname", Address: hostnameControl}, + }, + }, + }, + { + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: "InternalIP", Address: internalIpWorker}, + {Type: "Hostname", Address: hostnameWorker}, + }, + }, + }, + }, + } + }) + + Describe("CSV data to NodePorts conversion", func() { + Context("With no data", func() { + It("should return an empty map", func() { + cnp, ok := ConvertNodePorts([][]string{}) + Expect(ok).To(BeFalse()) + Expect(cnp).To(BeEmpty()) + }) + }) + Context("With some ports unparsable", func() { + It("should return only parsable records", func() { + expected := map[uint16]string{nodePortO: serviceR, nodePortN: serviceL} + cnp, ok := ConvertNodePorts(csvSomeUnparsable) + Expect(ok).To(BeTrue()) + Expect(cnp).To(Equal(expected)) + }) + }) + Context("With all ports unparsable", func() { + It("should return an empty map", func() { + cnp, ok := ConvertNodePorts(csvAllUnparsable) + Expect(ok).To(BeFalse()) + Expect(cnp).To(BeEmpty()) + }) + }) + }) + + Describe("NodePorts expected to fail filtering", func() { + Context("With no data", func() { + It("should leave nodeports unchanged", func() { + nodeports := map[uint16]string{nodePortO: serviceR} + expected := make(map[uint16]string) + for k, v := range nodeports { + expected[k] = v + } + FilterXFailNodePorts(map[uint16]string{}, nodeports) + Expect(nodeports).To(Equal(expected)) + }) + }) + Context("With port absent in NodePorts", func() { + It("should leave nodeports unchanged", func() { + xfail := map[uint16]string{nodePortP: serviceZ} + nodeports := map[uint16]string{nodePortO: serviceR} + expected := make(map[uint16]string) + for k, v := range nodeports { + expected[k] = v + } + FilterXFailNodePorts(xfail, nodeports) + Expect(nodeports).To(Equal(expected)) + }) + }) + Context("With other service than in NodePorts", func() { + It("should leave nodeports unchanged", func() { + xfail := map[uint16]string{nodePortO: serviceZ} + nodeports := map[uint16]string{nodePortO: serviceR} + expected := make(map[uint16]string) + for k, v := range nodeports { + expected[k] = v + } + FilterXFailNodePorts(xfail, nodeports) + Expect(nodeports).To(Equal(expected)) + }) + }) + Context("With all NodePorts expected to fail", func() { + It("should leave no nodeports", func() { + xfail := map[uint16]string{nodePortO: serviceR, nodePortN: serviceL} + nodeports := map[uint16]string{nodePortO: serviceR, nodePortN: serviceL} + FilterXFailNodePorts(xfail, nodeports) + Expect(nodeports).To(BeEmpty()) + }) + }) + }) + + Describe("NodePorts extraction", func() { + Context("With empty service list", func() { + It("should report no NodePorts", func() { + nodeports, ok := FilterNodePorts(servicesEmpty) + Expect(ok).To(BeFalse()) + Expect(nodeports).To(BeEmpty()) + }) + }) + Context("With service using single NodePort", func() { + It("should report single NodePort", func() { + expected := map[uint16]string{nodePortO: serviceR} + nodeports, ok := FilterNodePorts(servicesSingleWithNodePort) + Expect(ok).To(BeTrue()) + Expect(nodeports).To(Equal(expected)) + }) + }) + Context("With service using multiple NodePorts", func() { + It("should report all NodePorts", func() { + expected := map[uint16]string{nodePortO: serviceR, nodePortN: serviceR} + nodeports, ok := FilterNodePorts(servicesSingleWithMultipleNodePorts) + Expect(ok).To(BeTrue()) + Expect(nodeports).To(Equal(expected)) + }) + }) + Context("With many services using no NodePorts", func() { + It("should report no NodePorts", func() { + nodeports, ok := FilterNodePorts(servicesManyWithoutNodePorts) + Expect(ok).To(BeFalse()) + Expect(nodeports).To(BeEmpty()) + }) + }) + Context("With services using single NodePort", func() { + It("should report all NodePorts", func() { + expected := map[uint16]string{nodePortO: serviceR, nodePortN: serviceL} + nodeports, ok := FilterNodePorts(servicesManyWithNodePort) + Expect(ok).To(BeTrue()) + Expect(nodeports).To(Equal(expected)) + }) + }) + Context("With services using multiple NodePorts", func() { + It("should report all NodePorts", func() { + expected := map[uint16]string{ + nodePortO: serviceR, nodePortN: serviceR, + nodePortA: serviceL, nodePortP: serviceL, + } + nodeports, ok := FilterNodePorts(servicesManyWithMultipleNodePorts) + Expect(ok).To(BeTrue()) + Expect(nodeports).To(Equal(expected)) + }) + }) + Context("With mixed services", func() { + It("should report all NodePorts", func() { + expected := map[uint16]string{ + nodePortO: serviceL, nodePortN: serviceZ, nodePortA: serviceZ, + } + nodeports, ok := FilterNodePorts(servicesManyMixedNodePorts) + Expect(ok).To(BeTrue()) + Expect(nodeports).To(Equal(expected)) + }) + }) + }) + + Describe("IP addresses extraction", func() { + Context("With empty node list", func() { + It("should report no IP addresses", func() { + addresses, ok := FilterIPAddresses(nodesEmpty) + Expect(ok).To(BeFalse()) + Expect(addresses).To(BeEmpty()) + }) + }) + Context("With nodes using only hostnames", func() { + It("should report no IP addresses", func() { + addresses, ok := FilterIPAddresses(nodesManyWithHostnames) + Expect(ok).To(BeFalse()) + Expect(addresses).To(BeEmpty()) + }) + }) + Context("With node using only internal IP", func() { + It("should report internal IP", func() { + expected := []string{internalIpControl} + addresses, ok := FilterIPAddresses(nodesSingleWithIP) + Expect(ok).To(BeTrue()) + Expect(addresses).To(Equal(expected)) + }) + }) + Context("With node in the cloud", func() { + It("should report all IPs in correct order", func() { + expected := []string{externalIpControl, internalIpControl} + addresses, ok := FilterIPAddresses(nodesSingleWithBothIPs) + Expect(ok).To(BeTrue()) + Expect(addresses).To(Equal(expected)) + }) + }) + Context("With nodes in the mixed cloud", func() { + It("should report external IP as the first one", func() { + addresses, ok := FilterIPAddresses(nodesManyWithMixedIPs) + Expect(ok).To(BeTrue()) + Expect(addresses[0]).To(Equal(externalIpControl)) + }) + }) + }) +}) diff --git a/test/security/tox.ini b/test/security/tox.ini new file mode 100644 index 000000000..7ebf8e4c8 --- /dev/null +++ b/test/security/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = security +skipsdist = True + +[testenv] +deps = -r{toxinidir}/requirements.txt + +[testenv:security] +basepython = python3.8 |