From 5de622a8247c4cf4fc2bd4e5f8a947e60a8c4bfb Mon Sep 17 00:00:00 2001 From: mrichomme Date: Mon, 20 Apr 2020 14:50:27 +0200 Subject: Resync integration/xtesting repo Issue-ID: INT-1366 Signed-off-by: mrichomme Change-Id: I3af9c4697f0e67d3ce5b6d2fceeb978aeb20a0ff --- healthcheck/docker/Dockerfile | 10 +- infra-healthcheck/README.md | 32 +++-- infra-healthcheck/docker/Dockerfile | 7 +- infra-healthcheck/docker/logging.debug.ini | 70 ++++++++++ infra-healthcheck/docker/logging.ini | 70 ++++++++++ infra-healthcheck/docker/testcases.yaml | 12 ++ infra-healthcheck/infra_healthcheck/k8stest.py | 21 +-- infra-healthcheck/requirements.txt | 3 + infra-healthcheck/setup.cfg | 3 +- infra-healthcheck/upper-constraints.txt | 1 + security/README.md | 76 +++++++++++ security/docker/Dockerfile | 58 ++++++++- security/docker/testcases.yaml | 64 ++++++++- security/onap_security/__init__.py | 0 security/onap_security/security_tests.py | 173 +++++++++++++++++++++++++ security/onap_security/test_security_test.py | 40 ++++++ security/requirements.txt | 12 ++ security/scripts/check_cis_kubernetes.sh | 25 ++++ security/scripts/check_security_root.sh | 76 +++++++++++ security/scripts/check_unlimitted_pods.sh | 28 ++++ security/scripts/root_pods_xfail.txt | 33 +++++ security/setup.cfg | 16 +++ security/setup.py | 29 +++++ smoke-usecases-robot/docker/Dockerfile | 12 +- 24 files changed, 826 insertions(+), 45 deletions(-) create mode 100644 infra-healthcheck/docker/logging.debug.ini create mode 100644 infra-healthcheck/docker/logging.ini create mode 100644 infra-healthcheck/upper-constraints.txt create mode 100644 security/onap_security/__init__.py create mode 100644 security/onap_security/security_tests.py create mode 100644 security/onap_security/test_security_test.py create mode 100644 security/scripts/check_cis_kubernetes.sh create mode 100644 security/scripts/check_security_root.sh create mode 100644 security/scripts/check_unlimitted_pods.sh create mode 100644 security/scripts/root_pods_xfail.txt create mode 100644 security/setup.cfg create mode 100644 security/setup.py diff --git a/healthcheck/docker/Dockerfile b/healthcheck/docker/Dockerfile index 6496a70..bcaf378 100644 --- a/healthcheck/docker/Dockerfile +++ b/healthcheck/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM opnfv/xtesting +FROM opnfv/xtesting:jerma ARG OPENSTACK_TAG=master ARG OPNFV_TAG=master @@ -9,7 +9,7 @@ ENV PYTHONPATH $PYTHONPATH:/src/testing-utils/robotframework-onap/eteutils ENV TAG all COPY requirements.txt requirements.txt -RUN apk --no-cache add --update openssl && \ +RUN apk --no-cache add --update openssl chromium chromium-chromedriver && \ apk --no-cache add --virtual .build-deps --update \ python3-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ @@ -28,12 +28,12 @@ RUN apk --no-cache add --update openssl && \ mkdir -p /var/opt/ONAP/demo/heat && cp -Rf /src/demo/heat/vFW /var/opt/ONAP/demo/heat/ && \ mkdir -p /demo/service_mapping && cp -Rf /src/demo/service_mapping /demo/ && \ mkdir -p /var/opt/ONAP/demo/preload_data && cp -Rf /src/demo/preload_data /var/opt/ONAP/demo/ && \ - ln -s /usr/lib/python3.8/site-packages/vcpeutils /usr/lib/python3.8/site-packages/SoUtils && \ - ln -s /usr/lib/python3.8/site-packages/heatbridge /usr/lib/python3.8/site-packages/HeatBridge && \ + ln -s /usr/lib/python3.7/site-packages/vcpeutils /usr/lib/python3.7/site-packages/SoUtils && \ + ln -s /usr/lib/python3.7/site-packages/heatbridge /usr/lib/python3.7/site-packages/HeatBridge && \ rm -r requirements.txt /var/opt/ONAP/.git /src/demo && \ cd / && ln -s /var/opt/ONAP/robot/ /robot && \ apk del .build-deps -COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml +COPY docker/testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml COPY scripts/cmd.sh / CMD ["/cmd.sh"] diff --git a/infra-healthcheck/README.md b/infra-healthcheck/README.md index ba873af..231ff88 100644 --- a/infra-healthcheck/README.md +++ b/infra-healthcheck/README.md @@ -12,6 +12,8 @@ It includes 2 tests: are up&running * onap-helm: list the helm charts. The success criteria is all the helm charts are completed. +* nodeport_ingress: check that we have a 1:1 corresdpondance between nodeports + and ingress (run only when the env variable DEPLOY_SCENARIO includes ingress) ## Usage @@ -21,7 +23,7 @@ Mandatory: * The kubernetes configuration: usually hosted on the.kube/config of your jumphost. It corresponds the kubernetes credentials and are needed to perform - the different operations. This file shall be copied in /config/.kube/config in + the different operations. This file shall be copied in /root/.kube/config in the docker. Optional: @@ -35,9 +37,9 @@ Optional: You can run this docker by typing: ``` -docker run -v :/config/.kube/config -v +docker run -v :/root/.kube/config -v :/var/lib/xtesting/results -registry.gitlab.com/orange-opensource/lfn/onap/integration/xtesting:latest +registry.gitlab.com/orange-opensource/lfn/onap/integration/xtesting/infra-healthcheck:latest ``` Options: @@ -46,9 +48,9 @@ Options: specify the -r option in the command line. Please note that in this case, you must precise some env variables. -environement variables: +environment variables: -* Mandatory: +* Mandatory (if you want to report the results in the database): * TEST_DB_URL: the url of the target Database with the env variable . * NODE_NAME: the name of your test environement. It must be declared in the test database (e.g. windriver-SB00) @@ -58,22 +60,26 @@ environement variables: * BUILD_TAG: a unique tag of your CI system. It can be usefull to get all the tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]* to find the version (e.g. daily-elalto-123456789). + * DEPLOY_SCENARIO: your scenario deployment. ingress test run only if the + scenario includes 'ingress' The command becomes: ``` -docker run -v :/config/.kube/config -v +docker run -v :/root/.kube/config -v :/var/lib/xtesting/results registry.gitlab.com/orange-opensour -ce/lfn/onap/integration/xtesting:latest /bin/bash -c "run_tests -r -t all +ce/lfn/onap/integration/xtesting/infra-healthcheck:latest:latest /bin/bash -c " +run_tests -r -t all ``` ### Output ``` -+------------+-------------+-------------------+----------+--------+ -| TEST CASE | PROJECT | TIER | DURATION | RESULT | -+------------+-------------+-------------------+----------+--------+ -| onap-k8s | integration | infra-healthcheck | 00:06 | PASS | -| onap-helm | integration | infra-healthcheck | 00:01 | PASS | -+------------+-------------+-------------------+----------+--------+ ++------------------+-------------+-------------------+----------+--------+ +| TEST CASE | PROJECT | TIER | DURATION | RESULT | ++------------------+-------------+-------------------+----------+--------+ +| onap-k8s | integration | infra-healthcheck | 00:06 | PASS | +| onap-helm | integration | infra-healthcheck | 00:01 | PASS | +| nodeport_ingress | security | security | 00:01 | FAIL | ++------------------+-------------+-------------------+----------+--------+ ``` diff --git a/infra-healthcheck/docker/Dockerfile b/infra-healthcheck/docker/Dockerfile index 849268c..07b417b 100644 --- a/infra-healthcheck/docker/Dockerfile +++ b/infra-healthcheck/docker/Dockerfile @@ -3,6 +3,7 @@ FROM opnfv/xtesting ARG KUBERNETES_VERSION="v1.15.2" ARG HELM_VERSION="v2.14.1" ARG ONAP_TESTS_TAG=master +ARG ONAP_TAG=master # Install kubectl # Note: Latest version may be found on: @@ -10,8 +11,8 @@ ARG ONAP_TESTS_TAG=master ADD https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl /usr/local/bin/kubectl -COPY scripts/check_onap_k8s.sh /check_onap_k8s.sh COPY scripts/check_onap_helm.sh /check_onap_helm.sh +COPY upper-constraints.txt . RUN set -x && \ apk --no-cache add --update curl ca-certificates && \ @@ -20,12 +21,16 @@ RUN set -x && \ chmod +x /usr/local/bin/kubectl && \ adduser kubectl -Du 2342 -h /config && \ wget https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm && \ + wget -O /check_for_ingress_and_nodeports.py https://git.onap.org/integration/plain/test/security/check_for_ingress_and_nodeports.py?h=$ONAP_TAG &&\ chmod +x /usr/local/bin/helm && \ chmod +x /check_onap_*.sh && \ pip3 install --upgrade pip && \ + pip3 install --no-cache-dir -r upper-constraints.txt && \ pip3 install --no-cache-dir \ git+https://gitlab.com/Orange-OpenSource/lfn/onap/integration/xtesting.git@$ONAP_TESTS_TAG#subdirectory=infra-healthcheck && \ apk del .build-deps COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml +COPY docker/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/logging.ini +COPY docker/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/logging.debug.ini CMD ["run_tests", "-t", "all"] diff --git a/infra-healthcheck/docker/logging.debug.ini b/infra-healthcheck/docker/logging.debug.ini new file mode 100644 index 0000000..8b2644f --- /dev/null +++ b/infra-healthcheck/docker/logging.debug.ini @@ -0,0 +1,70 @@ +[loggers] +keys=root,xtesting,ci,core,warnings,kubernetes_status,infra_healthcheck + +[handlers] +keys=console,wconsole,file,dfile + +[formatters] +keys=standard + +[logger_root] +level=NOTSET +handlers=dfile + +[logger_xtesting] +level=NOTSET +handlers=file +qualname=xtesting + +[logger_ci] +level=NOTSET +handlers=console +qualname=xtesting.ci + +[logger_core] +level=NOTSET +handlers=console +qualname=xtesting.core + +[logger_warnings] +level=NOTSET +handlers=file,console +qualname=py.warnings + +[logger_kubernetes_status] +level=NOTSET +handlers=wconsole,file +qualname=kubernetes_status + +[logger_infra_healthcheck] +level=NOTSET +handlers=wconsole,file +qualname=infra_healthcheck + +[handler_console] +class=StreamHandler +level=INFO +formatter=standard +args=(sys.stdout,) + +[handler_wconsole] +class=StreamHandler +level=WARN +formatter=standard +args=(sys.stdout,) + +[handler_file] +class=FileHandler +level=DEBUG +formatter=standard +args=("/var/lib/xtesting/results/xtesting.log",) + +[handler_dfile] +class=FileHandler +level=DEBUG +formatter=standard +args=("/var/lib/xtesting/results/xtesting.debug.log",) + +[formatter_standard] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= diff --git a/infra-healthcheck/docker/logging.ini b/infra-healthcheck/docker/logging.ini new file mode 100644 index 0000000..bdf651a --- /dev/null +++ b/infra-healthcheck/docker/logging.ini @@ -0,0 +1,70 @@ +[loggers] +keys=root,xtesting,ci,core,warnings,kubernetes_status,infra_healthcheck + +[handlers] +keys=console,wconsole,file,null + +[formatters] +keys=standard + +[logger_root] +level=NOTSET +handlers=null + +[logger_xtesting] +level=NOTSET +handlers=file +qualname=xtesting + +[logger_ci] +level=NOTSET +handlers=console +qualname=xtesting.ci + +[logger_core] +level=NOTSET +handlers=console +qualname=xtesting.core + +[logger_warnings] +level=NOTSET +handlers=file,console +qualname=py.warnings + +[logger_kubernetes_status] +level=NOTSET +handlers=wconsole,file +qualname=kubernetes_status + +[logger_infra_healthcheck] +level=NOTSET +handlers=wconsole,file +qualname=infra_healthcheck + +[handler_null] +class=NullHandler +level=NOTSET +formatter=standard +args=() + +[handler_console] +class=StreamHandler +level=INFO +formatter=standard +args=(sys.stdout,) + +[handler_wconsole] +class=StreamHandler +level=WARN +formatter=standard +args=(sys.stdout,) + +[handler_file] +class=FileHandler +level=DEBUG +formatter=standard +args=("/var/lib/xtesting/results/xtesting.log",) + +[formatter_standard] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= diff --git a/infra-healthcheck/docker/testcases.yaml b/infra-healthcheck/docker/testcases.yaml index fb30c85..346daf5 100644 --- a/infra-healthcheck/docker/testcases.yaml +++ b/infra-healthcheck/docker/testcases.yaml @@ -27,3 +27,15 @@ tiers: DEPLOYED status run: name: 'onap_helm' + - + case_name: nodeport_ingress + project_name: security + criteria: 100 + blocking: false + description: >- + Check that there is no NodePort without corresponding + Ingress port + dependencies: + - DEPLOY_SCENARIO: 'ingress' + run: + name: 'nodeport_ingress' diff --git a/infra-healthcheck/infra_healthcheck/k8stest.py b/infra-healthcheck/infra_healthcheck/k8stest.py index 8bb7dde..f8d618f 100644 --- a/infra-healthcheck/infra_healthcheck/k8stest.py +++ b/infra-healthcheck/infra_healthcheck/k8stest.py @@ -101,16 +101,6 @@ class K8sTesting(testcase.TestCase): return res -class OnapK8sTest(K8sTesting): - """Kubernetes smoke test suite""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs.get("case_name", 'onap-k8s') - super(OnapK8sTest, self).__init__(**kwargs) - self.cmd = ['/check_onap_k8s.sh'] - self.criteria_string = "Nb Failed Pods" - - class OnapHelmTest(K8sTesting): """Kubernetes conformance test suite""" def __init__(self, **kwargs): @@ -119,3 +109,14 @@ class OnapHelmTest(K8sTesting): super(OnapHelmTest, self).__init__(**kwargs) self.cmd = ['/check_onap_helm.sh'] self.criteria_string = "Nb Failed Helm Charts" + + +class OnapSecurityNodePortsIngress(K8sTesting): + """Check that there is no NodePort without corresponding Ingress port.""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'nodeport_ingress') + super(OnapSecurityNodePortsIngress, self).__init__(**kwargs) + self.cmd = ['python3', '/check_for_ingress_and_nodeports.py', + '--conf', '/root/.kube/config'] + self.error_string = "NodePort without corresponding Ingress found" diff --git a/infra-healthcheck/requirements.txt b/infra-healthcheck/requirements.txt index aed40a6..91babf8 100644 --- a/infra-healthcheck/requirements.txt +++ b/infra-healthcheck/requirements.txt @@ -9,3 +9,6 @@ pylint>=2.1 # GPLv2 yamllint bashate # Apache-2.0 xtesting +kubernetes # Apache-2.0 +colorama # BSD +kubernetes_status diff --git a/infra-healthcheck/setup.cfg b/infra-healthcheck/setup.cfg index 2cccb82..a3ec3db 100644 --- a/infra-healthcheck/setup.cfg +++ b/infra-healthcheck/setup.cfg @@ -7,5 +7,6 @@ packages = infra_healthcheck [entry_points] xtesting.testcase = - onap_k8s = infra_healthcheck.k8stest:OnapK8sTest + onap_k8s = kubernetes_status.status:Status onap_helm = infra_healthcheck.k8stest:OnapHelmTest + nodeport_ingress = infra_healthcheck.k8stest:OnapSecurityNodePortsIngress diff --git a/infra-healthcheck/upper-constraints.txt b/infra-healthcheck/upper-constraints.txt new file mode 100644 index 0000000..d37d29d --- /dev/null +++ b/infra-healthcheck/upper-constraints.txt @@ -0,0 +1 @@ +git+https://gitlab.com/Orange-OpenSource/lfn/tools/kubernetes-status.git#egg=kubernetes_status diff --git a/security/README.md b/security/README.md index 3defbf8..b350078 100644 --- a/security/README.md +++ b/security/README.md @@ -2,10 +2,86 @@ ## Goal +This security docker includes the test suites dealing with security aspects +of an ONAP deployment. + +It includes 6 tests: + +* root_pods: check that pods are nor using root user or started as root +* unlimitted_pods: check that limits are set for pods +* cis_kubernetes: perform the k8s cis test suite (upstream src aquasecurity) +* http_public_endpoints: check that there is no public http endpoints exposed in + ONAP cluster +* nonssl_endpoints: check that all public HTTP endpoints exposed in ONAP + cluster use SSL tunnels +* jdpw_ports: check that there are no internal java ports +* kube_hunter: security suite to search k8s vulnerabilities (upstream src + aquasecurity) + ## Usage ### Configuration +Mandatory: + +* The kubernetes configuration: usually hosted on the.kube/config of your + jumphost. It corresponds the kubernetes credentials and are needed to perform + the different operations. This file shall be copied in /root/.kube/config in + the docker. + +Optional: + +* The local result directory path: to store the results in your local + environement. It shall corresponds to the internal result docker path + /var/lib/xtesting/results + ### Command +You can run this docker by typing: + +``` +docker run -v :/root/.kube/config -v +:/var/lib/xtesting/results +registry.gitlab.com/orange-opensource/lfn/onap/integration/xtesting/security:latest +``` + +Options: + +* -r: by default the reporting to the Database is not enabled. You need to + specify the -r option in the command line. Please note that in this case, you + must precise some env variables. + +environment variables: + +* Mandatory (if you want to report the results in the database): + * TEST_DB_URL: the url of the target Database with the env variable . + * NODE_NAME: the name of your test environement. It must be declared in the + test database (e.g. windriver-SB00) +* Optionnal + * INSTALLER_TYPE: precise how your ONAP has been installed (e.g. kubespray-oom, + rke-oom) + * BUILD_TAG: a unique tag of your CI system. It can be usefull to get all the + tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]* to find the + version (e.g. daily-elalto-123456789). + +The command becomes: + +``` +docker run -v :/root/.kube/config -v +:/var/lib/xtesting/results registry.gitlab.com/orange-opensour +ce/lfn/onap/integration/xtesting/security:latest /bin/bash -c "run_tests -r -t all +``` + ### Output + +``` ++-----------------------+------------+------------+------------+-----------+ +| TEST CASE | PROJECT | TIER | DURATION | RESULT | ++-----------------------+------------+------------+------------+-----------+ +| root_pods | security | security | 03:48 | FAIL | +| unlimitted_pods | security | security | 00:37 | FAIL | +| cis_kubernetes | security | security | 00:01 | FAIL | +| http_public_endpoints | security | security | 00:01 | FAIL | +| jdpw_ports | security | security | 05:39 | FAIL | ++-----------------------+------------+------------+------------+-----------+ +``` diff --git a/security/docker/Dockerfile b/security/docker/Dockerfile index 667900f..e38f565 100644 --- a/security/docker/Dockerfile +++ b/security/docker/Dockerfile @@ -1,12 +1,58 @@ -FROM opnfv/xtesting +FROM golang:1.13 AS build_aqua +WORKDIR /go/src/github.com/aquasecurity/ +RUN git clone https://github.com/aquasecurity/kube-bench.git --depth 1 +WORKDIR /go/src/github.com/aquasecurity/kube-bench +RUN GO111MODULE=on CGO_ENABLED=0 go install -a -ldflags "-w" +FROM golang:1.13 AS build_onap +WORKDIR /opt/onap +RUN git clone https://git.onap.org/integration --depth 1 +WORKDIR /opt/onap/integration/test/security/sslendpoints +RUN CGO_ENABLED=0 go install -a -ldflags '-w -s -extldflags "-static"' + +FROM opnfv/xtesting AS run + +ARG KUBERNETES_VERSION="v1.15.2" +ARG HELM_VERSION="v2.14.1" ARG ONAP_TAG=master -ARG PIP_TAG=19.3.1 +ARG ONAP_TESTS_TAG=master + +# Install kubectl +# Note: Latest version may be found on: +# https://aur.archlinux.org/packages/kubectl-bin/ + +ADD https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl /usr/local/bin/kubectl + +COPY scripts/check_security_root.sh /check_security_root.sh +COPY scripts/root_pods_xfail.txt /root_pods_xfail.txt +COPY scripts/check_unlimitted_pods.sh /check_unlimitted_pods.sh +COPY scripts/check_cis_kubernetes.sh /check_cis_kubernetes.sh +COPY --from=build_aqua /go/bin/kube-bench /usr/local/bin/kube-bench +COPY --from=build_aqua /go/src/github.com/aquasecurity/kube-bench/cfg/ /cfg/ +COPY --from=build_onap /go/bin/sslendpoints /usr/local/bin/sslendpoints -COPY requirements.txt requirements.txt -RUN apk --no-cache add --virtual .build-deps --update \ - openssl-dev libjpeg-turbo-dev && \ +RUN set -x && \ + apk --no-cache add --update curl ca-certificates openssl procps util-linux \ + nmap nmap-scripts && \ + apk --no-cache add --virtual .build-deps --update \ + python3-dev linux-headers gcc musl-dev && \ + chmod +x /usr/local/bin/kubectl && \ + git clone --depth 1 https://github.com/aquasecurity/kube-hunter.git /kube-hunter && \ + adduser kubectl -Du 2342 -h /config && \ + wget https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm && \ + wget -O /check_for_nonssl_endpoints.sh https://git.onap.org/integration/plain/test/security/check_for_nonssl_endpoints.sh?h=$ONAP_TAG &&\ + wget -O /check_for_jdwp.sh https://git.onap.org/integration/plain/test/security/check_for_jdwp.sh?h=$ONAP_TAG &&\ + wget -O /jdwp_xfail.txt https://git.onap.org/integration/plain/test/security/jdwp_xfail.txt?h=$ONAP_TAG &&\ + wget -O /nonssl_xfail.txt https://git.onap.org/integration/plain/test/security/nonssl_xfail.txt?h=$ONAP_TAG &&\ + chmod +x /usr/local/bin/helm && \ + chmod +x /usr/local/bin/kube-bench && \ + chmod +x /usr/local/bin/sslendpoints && \ + chmod +x /check_*.sh && \ + pip3 install --upgrade pip && \ + pip3 install --no-cache-dir \ + git+https://gitlab.com/Orange-OpenSource/lfn/onap/integration/xtesting.git@$ONAP_TESTS_TAG#subdirectory=security && \ + cd /kube-hunter && pip3 install -r /kube-hunter/requirements.txt && \ apk del .build-deps COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml -CMD ["run_test -t all -r"] +CMD ["run_tests", "-t", "all"] diff --git a/security/docker/testcases.yaml b/security/docker/testcases.yaml index ed281f2..6b9d482 100644 --- a/security/docker/testcases.yaml +++ b/security/docker/testcases.yaml @@ -8,11 +8,67 @@ tiers: Set of basic Functional security tests. testcases: - - case_name: osji - project_name: integration + case_name: root_pods + project_name: security criteria: 100 blocking: false description: >- - run osji scan. + test if pods are run in root. run: - name: 'onap_osji' + name: 'root_pods' + - + case_name: unlimitted_pods + project_name: security + criteria: 100 + blocking: false + description: >- + test if pods are run without limit. + run: + name: 'unlimitted_pods' + - + case_name: cis_kubernetes + project_name: security + criteria: 100 + blocking: false + description: >- + test if kubernetes install is CIS compliant. + run: + name: 'cis_kubernetes' + - + case_name: http_public_endpoints + project_name: security + criteria: 100 + blocking: false + description: >- + Check all ports exposed outside of kubernetes cluster + looking for plain http endpoint. + run: + name: 'http_public_endpoints' + - + case_name: nonssl_endpoints + project_name: security + criteria: 100 + blocking: false + description: >- + Check that all ports exposed outside of kubernetes cluster + use SSL tunnels. + run: + name: 'nonssl_endpoints' + - + case_name: jdpw_ports + project_name: security + criteria: 100 + blocking: false + description: >- + Check that no jdwp ports are exposed + run: + name: 'jdpw_ports' + - + case_name: kube_hunter + project_name: security + criteria: 100 + blocking: false + description: >- + Check k8s CVE. + run: + name: 'kube_hunter' diff --git a/security/onap_security/__init__.py b/security/onap_security/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/security/onap_security/security_tests.py b/security/onap_security/security_tests.py new file mode 100644 index 0000000..4136f66 --- /dev/null +++ b/security/onap_security/security_tests.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +# +# Copyright (c) 2018 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# + +""" +Define the parent for Kubernetes testing. +""" + +from __future__ import division + +import logging +import subprocess +import time + +from xtesting.core import testcase +from kubernetes import client, config + + +class SecurityTesting(testcase.TestCase): + """Security test runner""" + + __logger = logging.getLogger(__name__) + + def __init__(self, **kwargs): + super(SecurityTesting, self).__init__(**kwargs) + self.cmd = [] + self.result = 0 + self.details = {} + self.start_time = 0 + self.stop_time = 0 + self.error_string = "" + + def run_security(self): # pylint: disable=too-many-branches + """Run the test suites""" + cmd_line = self.cmd + self.__logger.info("Starting k8s test: '%s'.", cmd_line) + + process = subprocess.Popen(cmd_line, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + process.wait() + output = process.stdout.read().decode("utf-8") + if ('Error loading client' in output or + 'Unexpected error' in output): + raise Exception(output) + + # create a log file + file_name = "/var/lib/xtesting/results/" + self.case_name + ".log" + log_file = open(file_name, "w") + log_file.write(output) + log_file.close() + + # we consider the command return code for success criteria + if process.returncode is None: + success = False + details = self.error_string + if (self.case_name == 'kube_hunter' and + "No vulnerabilities were found" in output): + success = True + elif process.returncode != 0: + success = False + details = self.error_string + else: + success = True + details = "Test PASS" + + self.details = details + self.__logger.info("details: %s", details) + + if success: + self.result = 100 + else: + self.result = 0 + + def run(self, **kwargs): + """Generic Run.""" + + self.start_time = time.time() + try: + self.run_security() + res = self.EX_OK + except Exception: # pylint: disable=broad-except + self.__logger.exception("Error with running Security tests:") + res = self.EX_RUN_ERROR + + self.stop_time = time.time() + return res + + +class OnapSecurityDockerRootTest(SecurityTesting): + """Test that the dockers launched as root.""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'root_pods') + super(OnapSecurityDockerRootTest, self).__init__(**kwargs) + self.cmd = ['/check_security_root.sh', 'onap', '-l', '/root_pods_xfail.txt'] + self.error_string = "Pods launched with root users" + + +class OnapSecurityUnlimittedPodTest(SecurityTesting): + """Check that no pod is launch without limits.""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'unlimitted_pods') + super(OnapSecurityUnlimittedPodTest, self).__init__(**kwargs) + self.cmd = ['/check_unlimitted_pods.sh'] + self.error_string = "Pods lauched without limits" + + +class OnapSecurityCisKubernetes(SecurityTesting): + """Check that kubernetes install is CIS compliant""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'cis_kubernetes') + super(OnapSecurityCisKubernetes, self).__init__(**kwargs) + self.cmd = ['/check_cis_kubernetes.sh'] + self.error_string = "Kubernetes Deployment is not CIS compatible" + + +class OnapSecurityHttpPorts(SecurityTesting): + """Check all ports exposed outside of kubernetes cluster looking for plain + http endpoint.""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'http_public_endpoints') + super(OnapSecurityHttpPorts, self).__init__(**kwargs) + self.cmd = ['/check_for_nonssl_endpoints.sh', 'onap', '-l', '/nonssl_xfail.txt'] + self.error_string = "Public http endpoints still found" + + +class OnapSecurityNonSSLPorts(SecurityTesting): + """Check that all ports exposed outside of kubernetes cluster use SSL + tunnels.""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'nonssl_endpoints') + super(OnapSecurityNonSSLPorts, self).__init__(**kwargs) + self.cmd = ['/usr/local/bin/sslendpoints', '-xfail', '/nonssl_xfail.txt'] + self.error_string = "Public non-SSL endpoints still found" + + +class OnapSecurityJdwpPorts(SecurityTesting): + """Check that no jdwp ports are exposed.""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'jdpw_ports') + super(OnapSecurityJdwpPorts, self).__init__(**kwargs) + self.cmd = ['/check_for_jdwp.sh', 'onap', '-l', '/jdwp_xfail.txt'] + self.error_string = "JDWP ports found" + + +class OnapSecurityKubeHunter(SecurityTesting): + """Check k8s vulnerabilities.""" + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs.get("case_name", 'kube_hunter') + super(OnapSecurityKubeHunter, self).__init__(**kwargs) + config.load_kube_config(config_file='/root/.kube/config') + client_kubernetes = client.CoreV1Api() + node_list = client_kubernetes.list_node() + kube_hunter_cmd = ['/kube-hunter/kube-hunter.py', '--remote'] + for i in node_list.items: + addresses = i.status.addresses + for j in addresses: + if "External" in str(j): + kube_hunter_cmd.append(j.address) + self.cmd = kube_hunter_cmd + self.error_string = "Vulnerabilties detected." diff --git a/security/onap_security/test_security_test.py b/security/onap_security/test_security_test.py new file mode 100644 index 0000000..7906d8a --- /dev/null +++ b/security/onap_security/test_security_test.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# +# Copyright (c) 2018 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# + +"""Define the classes required to fully cover k8s.""" + +import logging +import os +import unittest + + +from security_tests import SecurityTesting + + +class SecurityTests(unittest.TestCase): + + # pylint: disable=missing-docstring + + def setUp(self): + os.environ["DEPLOY_SCENARIO"] = "k8-test" + os.environ["KUBE_MASTER_IP"] = "127.0.0.1" + os.environ["KUBE_MASTER_URL"] = "https://127.0.0.1:6443" + os.environ["KUBERNETES_PROVIDER"] = "local" + + self.security_stesting = SecurityTesting.SecurityTesting() + + def test_run_kubetest_cmd_none(self): + with self.assertRaises(TypeError): + self.security_stesting.run_security() + + +if __name__ == "__main__": + logging.disable(logging.CRITICAL) + unittest.main(verbosity=2) diff --git a/security/requirements.txt b/security/requirements.txt index b2c729c..6e8b944 100644 --- a/security/requirements.txt +++ b/security/requirements.txt @@ -1 +1,13 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +coverage!=4.4 # Apache-2.0 +mock # BSD +nose # LGPL +flake8>=2.5.4 # MIT +pylint>=2.1 # GPLv2 yamllint +bashate # Apache-2.0 +xtesting +Kubernetes +colorama diff --git a/security/scripts/check_cis_kubernetes.sh b/security/scripts/check_cis_kubernetes.sh new file mode 100644 index 0000000..33ffdf5 --- /dev/null +++ b/security/scripts/check_cis_kubernetes.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +echo "------------------------------------------------------------------------" +echo "-------------------- ONAP Security tests ----------------------------" +echo "----------------- Test if K8S is CIS compliant ----------------------" +echo "------------------------------------------------------------------------" + +code=0 + +CIS_VERSION=${CIS_VERSION:-1.4} +echo "Running CIS test case version ${CIS_VERSION}" +kube-bench master --benchmark cis-${CIS_VERSION} > cis_full_test.txt +cat cis_full_test.txt | grep "\[FAIL]" > cisK8s.txt + +if [ -s cisK8s.txt ] +then + code=1 + nb_errors=`cat cisK8s.txt | wc -l` + echo "Test FAIL: $nb_errors assertions not passed" + cat cis_full_test.txt +else + echo "Test PASS: Kubernetes Deployment is CIS compatible" +fi + +exit $code diff --git a/security/scripts/check_security_root.sh b/security/scripts/check_security_root.sh new file mode 100644 index 0000000..ca388fd --- /dev/null +++ b/security/scripts/check_security_root.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +usage() { + cat < [-l ] + -l: rooted pod xfail file +EOF + exit ${1:-0} +} + +if [ "$#" -lt 1 ]; then + usage + exit 1 +fi + +K8S_NAMESPACE=$1 +FILTERED_PODS_LIST=$(mktemp rooted_pods_XXXXXX) +WL_RAW_FILE_PATH=$(mktemp raw_filtered_pods_XXXXXX) + +manage_white_list() { + # init filtered port list file + if [ ! -f $WL_FILE_PATH ];then + echo "File not found" + usage + fi + grep -o '^[^#]*' $WL_FILE_PATH > $WL_RAW_FILE_PATH +} + +### getopts +while : +do + case $2 in + -h|--help|help) usage;; + -l) WL_FILE_PATH=$3;manage_white_list;shift;; + -*) usage 1 ;; + *) break ;; + esac +done + +echo "------------------------------------------------------------------------" +echo "-------------------- ONAP Security tests ----------------------------" +echo "-------------------- Test root user in pods -------------------------" +echo "------------------------------------------------------------------------" + +code=0 + +# get the pod list +for pod in `kubectl get pod -n $K8S_NAMESPACE| grep "Running" | grep -v functest | grep -v integration | awk '{print $1}'` ;do + list=`kubectl top pod $pod --containers -n onap |grep -v "POD"|awk '{print $1":"$2}'`; + for po in $list; do + contname=`echo $po|cut -d':' -f2`;uid=`kubectl exec $pod --container $contname -n $K8S_NAMESPACE id|sed -r "s/^uid=(.*) gid.*$/\1/"`;echo "POD: $pod container: $contname uid: $uid"; + done; +done | grep root > $FILTERED_PODS_LIST + +while IFS= read -r line; do + # for each line we test if it is in the white list with a regular expression + while IFS= read -r wl_line; do + wl_name=$(echo $wl_line | awk {'print $1'}) + if grep -e $K8S_NAMESPACE-$wl_name <<< "$line" > /dev/null ;then + # Found in white list, exclude it + sed -i "/$line/d" $FILTERED_PODS_LIST + fi + done < $WL_RAW_FILE_PATH +done < $FILTERED_PODS_LIST + +if [ -s $FILTERED_PODS_LIST ] +then + code=1 + nb_errors=`cat $FILTERED_PODS_LIST | wc -l` + echo "Test FAIL: $nb_errors pod(s) launched as root found" + cat $FILTERED_PODS_LIST +else + echo "Test PASS: No pod launched as root found" +fi + +exit $code diff --git a/security/scripts/check_unlimitted_pods.sh b/security/scripts/check_unlimitted_pods.sh new file mode 100644 index 0000000..fdef6f3 --- /dev/null +++ b/security/scripts/check_unlimitted_pods.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +echo "------------------------------------------------------------------------" +echo "-------------------- ONAP Security tests ----------------------------" +echo "-------------------- Test pods without limit ------------------------" +echo "------------------------------------------------------------------------" + +code=0 + +# get the pod list +for pod in `kubectl get pod -n onap|grep -v "NAME"|grep "Running\|Completed" |grep -v functest |grep -v integration | awk '{print $1}'`;do + kubectl describe pod $pod -n onap|grep "Limits"; + if [ $? == 1 ] ; then + echo $pod ; + fi; +done | grep -v Limits > NoLimitContainer.txt + +if [ -s NoLimitContainer.txt ] +then + code=1 + nb_errors=`cat NoLimitContainer.txt | wc -l` + echo "Test FAIL: $nb_errors pod(s) launched without limit" + cat NoLimitContainer.txt +else + echo "Test PASS: No pod launched without limit" +fi + +exit $code diff --git a/security/scripts/root_pods_xfail.txt b/security/scripts/root_pods_xfail.txt new file mode 100644 index 0000000..c282cf8 --- /dev/null +++ b/security/scripts/root_pods_xfail.txt @@ -0,0 +1,33 @@ +# Expected failure list for rooted ports +# Upstream pods are excluded in Frankfurt +# We consider only the pods we built +aaf-cass # cassandra +aaf-sms-vault # upstream vault and consul docker used by aaf AAF-1102 +aai # aai pods not launched as root even root user still in dockers AAI-2822 +awx # ansible +cassandra # common cassandra +consul # nobody remembers who is responsible for consul +dcae-redis # redis container +dcae-mongo # mongo container +dcae-cloudify-manager # DCAEGEN2-2121 +mariadb # common mariadb +msb-consul # another consul +multicloud-fcaps # rabbit-mq upstream pod MULTICLOUD-1017 +multicloud-k8s-etcd +multicloud-k8s-mongo +music-cassandra # music has itw own cassandra +nbi-mongo # a mongo db +netbox # netbox +pomba-elasticsearch # elasticsearch +portal-cassandra # portal cassandra +portal-db # portal mariadb +portal-zookeeper # portal zookeeper +zookeeper # common zookeper + +# other waivers +robot # testing +sniro-emulator # testing +oof-cmso-service # testing +vnfsdk # testing VNFSDK-565 +pomba # nobody taking cares of pomba for several releases +dcaemod # dcae experimental pods for Frankfurt diff --git a/security/setup.cfg b/security/setup.cfg new file mode 100644 index 0000000..61fe7fa --- /dev/null +++ b/security/setup.cfg @@ -0,0 +1,16 @@ +[metadata] +name = onap_security +version = 1 + +[files] +packages = onap_security + +[entry_points] +xtesting.testcase = + root_pods = onap_security.security_tests:OnapSecurityDockerRootTest + unlimitted_pods = onap_security.security_tests:OnapSecurityUnlimittedPodTest + cis_kubernetes = onap_security.security_tests:OnapSecurityCisKubernetes + http_public_endpoints = onap_security.security_tests:OnapSecurityHttpPorts + nonssl_endpoints = onap_security.security_tests:OnapSecurityNonSSLPorts + jdpw_ports = onap_security.security_tests:OnapSecurityJdwpPorts + kube_hunter = onap_security.security_tests:OnapSecurityKubeHunter diff --git a/security/setup.py b/security/setup.py new file mode 100644 index 0000000..566d844 --- /dev/null +++ b/security/setup.py @@ -0,0 +1,29 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr>=2.0.0'], + pbr=True) diff --git a/smoke-usecases-robot/docker/Dockerfile b/smoke-usecases-robot/docker/Dockerfile index 38616aa..8dbe29a 100644 --- a/smoke-usecases-robot/docker/Dockerfile +++ b/smoke-usecases-robot/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM opnfv/xtesting +FROM opnfv/xtesting:jerma MAINTAINER Morgan Richomme @@ -10,7 +10,8 @@ ARG PIP_TAG=19.3.1 ENV TAG all COPY requirements.txt requirements.txt -RUN apk --no-cache add --virtual .build-deps --update \ +RUN apk --no-cache add --update openssl chromium chromium-chromedriver && \ + apk --no-cache add --virtual .build-deps --update \ python3-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ pip3 install --upgrade pip && \ @@ -26,14 +27,15 @@ RUN apk --no-cache add --virtual .build-deps --update \ -chttps://git.opnfv.org/functest/plain/upper-constraints.txt?h=$OPNFV_TAG \ -rrequirements.txt && \ mkdir -p /var/opt/ONAP/demo/heat && cp -Rf /src/demo/heat/vFW /var/opt/ONAP/demo/heat/ && \ + mkdir -p /var/opt/ONAP/demo/tosca && cp -Rf /src/demo/tosca/pNF /var/opt/ONAP/demo/tosca/ && \ mkdir -p /demo/service_mapping && cp -Rf /src/demo/service_mapping /demo/ && \ mkdir -p /var/opt/ONAP/demo/preload_data && cp -Rf /src/demo/preload_data /var/opt/ONAP/demo/ && \ - ln -s /usr/lib/python3.8/site-packages/vcpeutils /usr/lib/python3.8/site-packages/SoUtils && \ - ln -s /usr/lib/python3.8/site-packages/heatbridge /usr/lib/python3.8/site-packages/HeatBridge && \ + ln -s /usr/lib/python3.7/site-packages/vcpeutils /usr/lib/python3.7/site-packages/SoUtils && \ + ln -s /usr/lib/python3.7/site-packages/heatbridge /usr/lib/python3.7/site-packages/HeatBridge && \ rm -r requirements.txt /var/opt/ONAP/.git /src/demo && \ cd / && ln -s /var/opt/ONAP/robot/ /robot && \ apk del .build-deps -COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml +COPY docker/testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml COPY scripts/cmd.sh / CMD ["/cmd.sh"] -- cgit 1.2.3-korg