From 29dbb3106d28d6e53f0263eb34020cedd1fbd390 Mon Sep 17 00:00:00 2001 From: mrichomme Date: Tue, 16 Jun 2020 18:32:13 +0200 Subject: Initiate check certificate validity test Issue-ID: INT-1570 Signed-off-by: mrichomme Change-Id: I9794ec17a254ac21e87e3a251b6cad849a763742 Signed-off-by: mrichomme --- .../check_certificates_validity.py | 296 +++++++++++++++++++++ .../check_certificates/nodeports_xfail.txt | 2 + .../check_certificates/templates/base.html.j2 | 231 ++++++++++++++++ .../templates/cert-nodeports.html.j2 | 129 +++++++++ test/security/check_certificates/requirements.txt | 3 + test/security/check_certificates/setup.cfg | 6 + test/security/check_certificates/setup.py | 4 + .../check_certificates/test-requirements.txt | 6 + test/security/check_certificates/tox.ini | 9 + 9 files changed, 686 insertions(+) create mode 100644 test/security/check_certificates/check_certificates/check_certificates_validity.py create mode 100644 test/security/check_certificates/check_certificates/nodeports_xfail.txt create mode 100644 test/security/check_certificates/check_certificates/templates/base.html.j2 create mode 100644 test/security/check_certificates/check_certificates/templates/cert-nodeports.html.j2 create mode 100644 test/security/check_certificates/requirements.txt create mode 100644 test/security/check_certificates/setup.cfg create mode 100644 test/security/check_certificates/setup.py create mode 100644 test/security/check_certificates/test-requirements.txt create mode 100644 test/security/check_certificates/tox.ini diff --git a/test/security/check_certificates/check_certificates/check_certificates_validity.py b/test/security/check_certificates/check_certificates/check_certificates_validity.py new file mode 100644 index 000000000..a6fd9cd1b --- /dev/null +++ b/test/security/check_certificates/check_certificates/check_certificates_validity.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2020 Orange, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# Check all the kubernetes pods, evaluate the certificate and build a +# certificate dashboard. +# +# Dependencies: +# See requirements.txt +# The dashboard is based on bulma framework +# +# Environment: +# This script should be run on the local machine which has network access to +# the onap K8S cluster. +# It requires k8s cluster config file on local machine +# It requires also the ONAP IP provided through an env variable ONAP_IP +# ONAP_NAMESPACE env variable is also considered +# if not set we set it to onap +# Example usage: +# python check_certificates_validity.py +# the summary html page will be generated where the script is launched +""" +Check ONAP certificates +""" +import argparse +import logging +import os +import ssl +import sys +import OpenSSL +from datetime import datetime +from kubernetes import client, config +from jinja2 import Environment, FileSystemLoader, select_autoescape + +# Logger +LOG_LEVEL = 'INFO' +logging.basicConfig() +LOGGER = logging.getLogger("Gating-Index") +LOGGER.setLevel(LOG_LEVEL) +CERT_MODES = ['nodeport', 'ingress', 'internal'] +EXP_CRITERIA_MIN = 30 +EXP_CRITERIA_MAX = 389 +EXPECTED_CERT_STRING = "C=US;O=ONAP;OU=OSAAF;CN=intermediateCA_9" +RESULT_PATH = "." + + +# Get arguments +parser = argparse.ArgumentParser() +parser.add_argument( + '-m', + '--mode', + choices=CERT_MODES, + help='Mode (nodeport, ingress, internal). If not set all modes are tried', + default='nodeport') +parser.add_argument( + '-i', + '--ip', + help='ONAP IP needed (for nodeport mode)', + default=os.environ.get('ONAP_IP')) +parser.add_argument( + '-n', + '--namespace', + help='ONAP namespace', + default='onap') +parser.add_argument( + '-d', + '--dir', + help='Result directory', + default=RESULT_PATH) + +args = parser.parse_args() + +# Get the ONAP namespace +onap_namespace = args.namespace +LOGGER.info("Verification of the %s certificates started", onap_namespace) + +# Nodeport specific section +# Retrieve the kubernetes IP for mode nodeport +if args.mode == "nodeport": + if args.ip is None: + LOGGER.error( + "In nodeport mode, the IP of the ONAP cluster is required." + + "The value can be set using -i option " + + "or retrieved from the ONAP_IP env variable") + exit(parser.print_usage()) + try: + nodeports_xfail_list = [] + with open('nodeports_xfail.txt', 'r') as f: + first_line = f.readline() + for line in f: + nodeports_xfail_list.append( + line.split(" ", 1)[0].strip('\n')) + LOGGER.info("nodeports xfail list: %s", + nodeports_xfail_list) + except KeyError: + LOGGER.error("Please set the environment variable ONAP_IP") + sys.exit(1) + except FileNotFoundError: + LOGGER.warning("Nodeport xfail list not found") + +# Kubernetes section +# retrieve the candidate ports first +k8s_config = config.load_kube_config() + +core = client.CoreV1Api() +api_instance = client.ExtensionsV1beta1Api( + client.ApiClient(k8s_config)) +k8s_services = core.list_namespaced_service(onap_namespace).items +k8s_ingress = api_instance.list_namespaced_ingress(onap_namespace).items + + +def get_certifificate_info(host, port): + LOGGER.debug("Host: %s", host) + LOGGER.debug("Port: %s", port) + cert = ssl.get_server_certificate( + (host, port)) + LOGGER.debug("get certificate") + x509 = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, cert) + + LOGGER.debug("get certificate") + exp_date = datetime.strptime( + x509.get_notAfter().decode('ascii'), '%Y%m%d%H%M%SZ') + LOGGER.debug("Expiration date retrieved %s", exp_date) + issuer = x509.get_issuer().get_components() + + issuer_info = '' + # format issuer nicely + for issuer_info_key, issuer_info_val in issuer: + issuer_info += (issuer_info_key.decode('utf-8') + "=" + + issuer_info_val.decode('utf-8') + ";") + cert_validity = False + if issuer_info[:-1] == EXPECTED_CERT_STRING: + cert_validity = True + + return {'expiration_date': exp_date, + 'issuer': issuer_info[:-1], + 'validity': cert_validity} + + +def test_services(k8s_services, mode): + success_criteria = True # success criteria per scan + # looks for the certificates + node_ports_list = [] + node_ports_ssl_error_list = [] + node_ports_connection_error_list = [] + node_ports_type_error_list = [] + node_ports_reset_error_list = [] + + # for node ports and internal we consider the services + # for the ingress we consider the ingress + for service in k8s_services: + try: + for port in service.spec.ports: + # For nodeport mode, we consider + # - the IP of the cluster + # - spec.port.node_port + # + # For internal mode, we consider + # - spec.selector.app + # - spec.port.port + test_name = service.metadata.name + test_port = None + error_waiver = False # waiver per port + if mode == 'nodeport': + test_url = args.ip + test_port = port.node_port + + # Retrieve the nodeport xfail list + # to consider SECCOM waiver if needed + if test_port in nodeports_xfail_list: + error_waiver = True + else: # internal mode + test_url = service.spec.selector.app + test_port = port.port + + if test_port is not None: + LOGGER.info( + "Look for certificate %s (%s:%s)", + test_name, + test_url, + test_port) + cert_info = get_certifificate_info(test_url, test_port) + exp_date = cert_info['expiration_date'] + LOGGER.info("Expiration date retrieved %s", exp_date) + # calculate the remaining time + delta_time = (exp_date - datetime.now()).days + + # Test criteria + if error_waiver: + LOGGER.info("Port found in the xfail list," + + "do not consider it for success criteria") + else: + if (delta_time < EXP_CRITERIA_MIN or + delta_time > EXP_CRITERIA_MAX): + success_criteria = False + if cert_info['validity'] is False: + success_criteria = False + # add certificate to the list + node_ports_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'expiration_date': str(exp_date), + 'remaining_days': delta_time, + 'cluster_ip': service.spec.cluster_ip, + 'issuer': cert_info['issuer'], + 'validity': cert_info['validity']}) + else: + LOGGER.debug("Port value retrieved as None") + except ssl.SSLError as e: + LOGGER.exception("Bad certificate for port %s" % port) + node_ports_ssl_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + except ConnectionRefusedError as e: + LOGGER.exception("ConnectionrefusedError for port %s" % port) + node_ports_connection_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + except TypeError as e: + LOGGER.exception("Type Error for port %s" % port) + node_ports_type_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + except ConnectionResetError as e: + LOGGER.exception("ConnectionResetError for port %s" % port) + node_ports_reset_error_list.append( + {'pod_name': test_name, + 'pod_port': test_port, + 'error_details': str(e)}) + + # Create html summary + jinja_env = Environment( + autoescape=select_autoescape(['html']), + loader=FileSystemLoader('./templates')) + if args.mode == 'nodeport': + jinja_env.get_template('cert-nodeports.html.j2').stream( + node_ports_list=node_ports_list, + node_ports_ssl_error_list=node_ports_ssl_error_list, + node_ports_connection_error_list=node_ports_connection_error_list, + node_ports_type_error_list=node_ports_type_error_list, + node_ports_reset_error_list=node_ports_reset_error_list).dump( + '{}/certificates.html'.format(args.dir)) + return success_criteria + + +def test_ingress(k8s_ingress, mode): + LOGGER.debug('Test %s mode', mode) + for ingress in k8s_ingress: + LOGGER.debug(ingress) + return True + + +# *************************************************************************** +# *************************************************************************** +# start of the test +# *************************************************************************** +# *************************************************************************** +test_status = True +if args.mode == "ingress": + test_routine = test_ingress + test_param = k8s_ingress +else: + test_routine = test_services + test_param = k8s_services + +LOGGER.info(">>>> Test certificates: mode = %s", args.mode) +if test_routine(test_param, args.mode): + LOGGER.warning(">>>> Test PASS") +else: + LOGGER.warning(">>>> Test FAIL") + test_status = False + +if test_status: + LOGGER.info(">>>> Test Check certificates PASS") +else: + LOGGER.error(">>>> Test Check certificates FAIL") + sys.exit(1) diff --git a/test/security/check_certificates/check_certificates/nodeports_xfail.txt b/test/security/check_certificates/check_certificates/nodeports_xfail.txt new file mode 100644 index 000000000..5c0801014 --- /dev/null +++ b/test/security/check_certificates/check_certificates/nodeports_xfail.txt @@ -0,0 +1,2 @@ +# Expected failure list for certificates associated to nodeports +666 # foo example nodeport diff --git a/test/security/check_certificates/check_certificates/templates/base.html.j2 b/test/security/check_certificates/check_certificates/templates/base.html.j2 new file mode 100644 index 000000000..cbb4e4428 --- /dev/null +++ b/test/security/check_certificates/check_certificates/templates/base.html.j2 @@ -0,0 +1,231 @@ +{% macro color(failing, total) %} +{% if failing == 0 %} +is-success +{% else %} +{% if (failing / total) <= 0.1 %} +is-warning +{% else %} +is-danger +{% endif %} +{% endif %} +{% endmacro %} + +{% macro percentage(failing, total) %} +{{ ((total - failing) / total) | round }} +{% endmacro %} + +{% macro statistic(resource_name, failing, total) %} +{% set success = total - failing %} +
+
+

{{ resource_name | capitalize }}

+

{{ success }}/{{ total }}

+ {{ percentage(failing, total) }} +
+
+{% endmacro %} + +{% macro pods_table(pods) %} +
+ + + + + + + + + + + + {% for pod in pods %} + + + {% if pod.init_done %} + + {% else %} + + {% endif %} + + + {% if pod.init_done %} + + {% else %} + + {% endif %} + + {% endfor %} + +
NameReadyStatusReasonRestarts
{{ pod.k8s.metadata.name }}{{ pod.running_containers }}/{{ (pod.containers | length) }}Init:{{ pod.runned_init_containers }}/{{ (pod.init_containers | length) }}{{ pod.k8s.status.phase }}{{ pod.k8s.status.reason }}{{ pod.restart_count }}{{ pod.init_restart_count }}
+
+{% endmacro %} + +{% macro key_value_description_list(title, dict) %} +
{{ title | capitalize }}:
+
+ {% if dict %} + {% for key, value in dict.items() %} + {% if loop.first %} +
+ {% endif %} +
{{ key }}:
+
{{ value }}
+ {% if loop.last %} +
+ {% endif %} + {% endfor %} + {% endif %} +
+{% endmacro %} + +{% macro description(k8s) %} +
+

Description

+
+
+ {% if k8s.spec.type %} +
Type:
+
{{ k8s.spec.type }}
+ {% if (k8s.spec.type | lower) == "clusterip" %} +
Headless:
+
{% if (k8s.spec.cluster_ip | lower) == "none" %}Yes{% else %}No{% endif %}
+ {% endif %} + {% endif %} + {{ key_value_description_list('Labels', k8s.metadata.labels) | indent(width=6) }} + {{ key_value_description_list('Annotations', k8s.metadata.annotations) | indent(width=6) }} + {% if k8s.spec.selector %} + {% if k8s.spec.selector.match_labels %} + {{ key_value_description_list('Selector', k8s.spec.selector.match_labels) | indent(width=6) }} + {% else %} + {{ key_value_description_list('Selector', k8s.spec.selector) | indent(width=6) }} + {% endif %} + {% endif %} + {% if k8s.phase %} +
Status:
+
{{ k8s.phase }}
+ {% endif %} + {% if k8s.metadata.owner_references %} +
Controlled By:
+
{{ k8s.metadata.owner_references[0].kind }}/{{ k8s.metadata.owner_references[0].name }}
+ {% endif %} +
+
+
+{% endmacro %} + +{% macro pods_container(pods, parent, has_title=True) %} +
+ {% if has_title %} +

Pods

+ {% endif %} + {% if (pods | length) > 0 %} + {{ pods_table(pods) | indent(width=2) }} + {% else %} +
{{ parent }} has no pods!
+ {% endif %} +
+{% endmacro %} + +{% macro two_level_breadcrumb(title, name) %} +
+
+ +
+
+{% endmacro %} + +{% macro pod_parent_summary(title, name, failed_pods, pods) %} +{{ summary(title, name, [{'title': 'Pod', 'failing': failed_pods, 'total': (pods | length)}]) }} +{% endmacro %} + +{% macro number_ok(number, none_value, total=None) %} +{% if number %} +{% if total and number < total %} +{{ number }} +{% else %} +{{ number }} +{% endif %} +{% else %} +{{ none_value }} +{% endif %} +{% endmacro %} + +{% macro summary(title, name, statistics) %} +
+
+
+

+ {{ title | capitalize }} {{ name }} Summary +

+ +
+
+
+{% endmacro %} + + + + + + + Tests results - {% block title %}{% endblock %} + + + {% block more_head %}{% endblock %} + + + + + {% block content %}{% endblock %} + + + + diff --git a/test/security/check_certificates/check_certificates/templates/cert-nodeports.html.j2 b/test/security/check_certificates/check_certificates/templates/cert-nodeports.html.j2 new file mode 100644 index 000000000..df37c3da9 --- /dev/null +++ b/test/security/check_certificates/check_certificates/templates/cert-nodeports.html.j2 @@ -0,0 +1,129 @@ +{% extends "base.html.j2" %} +{% block title %}ONAP Certificates expiration page{% endblock %} + +{% block content %} +

ONAP Certificates

+
+
+

Node ports

+ + + + + + + + + + + + + + {% for cert in node_ports_list %} + 389 %} class="has-background-warning-light" {%elif cert.remaining_days == 364 and cert.validity %} class="has-background-success-light" {% endif %}> + + + + + + + + {% endfor %} + +
ComponentPortExpected Expiration DateRemaining DaysRoot CARoot CA Validity
{{ cert.pod_name }}{{ cert.pod_port }}{{ cert.expiration_date }}{{ cert.remaining_days }}{{ cert.issuer }}{% if cert.validity %} + + + + {% else %} + + + + {% endif %}
+ + {% if node_ports_ssl_error_list|length > 0 %} +

Node ports SSL errors

+ + + + + + + + + + {% for cert in node_ports_ssl_error_list %} + + + + + {% endfor %} + +
ComponentPortError Details
{{ cert.pod_name }}{{ cert.pod_port }}{{ cert.error_details }}
+{% endif %} + +{% if node_ports_connection_error_list|length > 0 %} +

Node ports Connection errors

+ + + + + + + + + + {% for cert in node_ports_connection_error_list %} + + + + + {% endfor %} + +
ComponentPortError Details
{{ cert.pod_name }}{{ cert.pod_port }}{{ cert.error_details }}
+{% endif %} + +{% if node_ports_list_type_error_list|length > 0 %} +

Node ports Type Error

+ + + + + + + + + + {% for cert in node_ports_list_type_error_list %} + + + + + {% endfor %} + +
ComponentPortError Details
{{ cert.pod_name }}{{ cert.pod_port }}{{ cert.error_details }}
+{% endif %} + +{% if node_ports_reset_error_list|length > 0 %} +

Node ports Connections Error

+ + + + + + + + + + {% for cert in node_ports_reset_error_list %} + + + + + {% endfor %} + +
ComponentPortError Details
{{ cert.pod_name }}{{ cert.pod_port }}{{ cert.error_details }}
+{% endif %} + +{% endblock %} +
+
diff --git a/test/security/check_certificates/requirements.txt b/test/security/check_certificates/requirements.txt new file mode 100644 index 000000000..15d50c44c --- /dev/null +++ b/test/security/check_certificates/requirements.txt @@ -0,0 +1,3 @@ +pyopenssl +kubernetes +jinja2 diff --git a/test/security/check_certificates/setup.cfg b/test/security/check_certificates/setup.cfg new file mode 100644 index 000000000..a678abced --- /dev/null +++ b/test/security/check_certificates/setup.cfg @@ -0,0 +1,6 @@ +[metadata] +name = check_certificates +version = 0.1 + +[files] +packages = check_certificates diff --git a/test/security/check_certificates/setup.py b/test/security/check_certificates/setup.py new file mode 100644 index 000000000..9a370e270 --- /dev/null +++ b/test/security/check_certificates/setup.py @@ -0,0 +1,4 @@ +import setuptools +setuptools.setup( + setup_requires=['pbr', 'setuptools'], + pbr=True) diff --git a/test/security/check_certificates/test-requirements.txt b/test/security/check_certificates/test-requirements.txt new file mode 100644 index 000000000..a0679b703 --- /dev/null +++ b/test/security/check_certificates/test-requirements.txt @@ -0,0 +1,6 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +coverage!=4.4,>=4.0 # Apache-2.0 +flake8 # MIT +pylint # GPLv2 diff --git a/test/security/check_certificates/tox.ini b/test/security/check_certificates/tox.ini new file mode 100644 index 000000000..2172bbc96 --- /dev/null +++ b/test/security/check_certificates/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py3 + +[testenv] +deps = + -r{toxinidir}/requirements.txt + +[testenv:py3] +commands = python {toxinidir}/setup.py test -- cgit 1.2.3-korg