From 5de622a8247c4cf4fc2bd4e5f8a947e60a8c4bfb Mon Sep 17 00:00:00 2001
From: mrichomme <morgan.richomme@orange.com>
Date: Mon, 20 Apr 2020 14:50:27 +0200
Subject: Resync integration/xtesting repo
Issue-ID: INT-1366
Signed-off-by: mrichomme <morgan.richomme@orange.com>
Change-Id: I3af9c4697f0e67d3ce5b6d2fceeb978aeb20a0ff
---
healthcheck/docker/Dockerfile | 10 +-
infra-healthcheck/README.md | 32 +++--
infra-healthcheck/docker/Dockerfile | 7 +-
infra-healthcheck/docker/logging.debug.ini | 70 ++++++++++
infra-healthcheck/docker/logging.ini | 70 ++++++++++
infra-healthcheck/docker/testcases.yaml | 12 ++
infra-healthcheck/infra_healthcheck/k8stest.py | 21 +--
infra-healthcheck/requirements.txt | 3 +
infra-healthcheck/setup.cfg | 3 +-
infra-healthcheck/upper-constraints.txt | 1 +
security/README.md | 76 +++++++++++
security/docker/Dockerfile | 58 ++++++++-
security/docker/testcases.yaml | 64 ++++++++-
security/onap_security/__init__.py | 0
security/onap_security/security_tests.py | 173 +++++++++++++++++++++++++
security/onap_security/test_security_test.py | 40 ++++++
security/requirements.txt | 12 ++
security/scripts/check_cis_kubernetes.sh | 25 ++++
security/scripts/check_security_root.sh | 76 +++++++++++
security/scripts/check_unlimitted_pods.sh | 28 ++++
security/scripts/root_pods_xfail.txt | 33 +++++
security/setup.cfg | 16 +++
security/setup.py | 29 +++++
smoke-usecases-robot/docker/Dockerfile | 12 +-
24 files changed, 826 insertions(+), 45 deletions(-)
create mode 100644 infra-healthcheck/docker/logging.debug.ini
create mode 100644 infra-healthcheck/docker/logging.ini
create mode 100644 infra-healthcheck/upper-constraints.txt
create mode 100644 security/onap_security/__init__.py
create mode 100644 security/onap_security/security_tests.py
create mode 100644 security/onap_security/test_security_test.py
create mode 100644 security/scripts/check_cis_kubernetes.sh
create mode 100644 security/scripts/check_security_root.sh
create mode 100644 security/scripts/check_unlimitted_pods.sh
create mode 100644 security/scripts/root_pods_xfail.txt
create mode 100644 security/setup.cfg
create mode 100644 security/setup.py
diff --git a/healthcheck/docker/Dockerfile b/healthcheck/docker/Dockerfile
index 6496a70..bcaf378 100644
--- a/healthcheck/docker/Dockerfile
+++ b/healthcheck/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM opnfv/xtesting
+FROM opnfv/xtesting:jerma
ARG OPENSTACK_TAG=master
ARG OPNFV_TAG=master
@@ -9,7 +9,7 @@ ENV PYTHONPATH $PYTHONPATH:/src/testing-utils/robotframework-onap/eteutils
ENV TAG all
COPY requirements.txt requirements.txt
-RUN apk --no-cache add --update openssl && \
+RUN apk --no-cache add --update openssl chromium chromium-chromedriver && \
apk --no-cache add --virtual .build-deps --update \
python3-dev build-base linux-headers libffi-dev \
openssl-dev libjpeg-turbo-dev && \
@@ -28,12 +28,12 @@ RUN apk --no-cache add --update openssl && \
mkdir -p /var/opt/ONAP/demo/heat && cp -Rf /src/demo/heat/vFW /var/opt/ONAP/demo/heat/ && \
mkdir -p /demo/service_mapping && cp -Rf /src/demo/service_mapping /demo/ && \
mkdir -p /var/opt/ONAP/demo/preload_data && cp -Rf /src/demo/preload_data /var/opt/ONAP/demo/ && \
- ln -s /usr/lib/python3.8/site-packages/vcpeutils /usr/lib/python3.8/site-packages/SoUtils && \
- ln -s /usr/lib/python3.8/site-packages/heatbridge /usr/lib/python3.8/site-packages/HeatBridge && \
+ ln -s /usr/lib/python3.7/site-packages/vcpeutils /usr/lib/python3.7/site-packages/SoUtils && \
+ ln -s /usr/lib/python3.7/site-packages/heatbridge /usr/lib/python3.7/site-packages/HeatBridge && \
rm -r requirements.txt /var/opt/ONAP/.git /src/demo && \
cd / && ln -s /var/opt/ONAP/robot/ /robot && \
apk del .build-deps
-COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
+COPY docker/testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
COPY scripts/cmd.sh /
CMD ["/cmd.sh"]
diff --git a/infra-healthcheck/README.md b/infra-healthcheck/README.md
index ba873af..231ff88 100644
--- a/infra-healthcheck/README.md
+++ b/infra-healthcheck/README.md
@@ -12,6 +12,8 @@ It includes 2 tests:
are up&running
* onap-helm: list the helm charts. The success criteria is all the helm charts
are completed.
+* nodeport_ingress: check that we have a 1:1 corresdpondance between nodeports
+ and ingress (run only when the env variable DEPLOY_SCENARIO includes ingress)
## Usage
@@ -21,7 +23,7 @@ Mandatory:
* The kubernetes configuration: usually hosted on the.kube/config of your
jumphost. It corresponds the kubernetes credentials and are needed to perform
- the different operations. This file shall be copied in /config/.kube/config in
+ the different operations. This file shall be copied in /root/.kube/config in
the docker.
Optional:
@@ -35,9 +37,9 @@ Optional:
You can run this docker by typing:
```
-docker run -v <the kube config>:/config/.kube/config -v
+docker run -v <the kube config>:/root/.kube/config -v
<result directory>:/var/lib/xtesting/results
-registry.gitlab.com/orange-opensource/lfn/onap/integration/xtesting:latest
+registry.gitlab.com/orange-opensource/lfn/onap/integration/xtesting/infra-healthcheck:latest
```
Options:
@@ -46,9 +48,9 @@ Options:
specify the -r option in the command line. Please note that in this case, you
must precise some env variables.
-environement variables:
+environment variables:
-* Mandatory:
+* Mandatory (if you want to report the results in the database):
* TEST_DB_URL: the url of the target Database with the env variable .
* NODE_NAME: the name of your test environement. It must be declared in the
test database (e.g. windriver-SB00)
@@ -58,22 +60,26 @@ environement variables:
* BUILD_TAG: a unique tag of your CI system. It can be usefull to get all the
tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]* to find the
version (e.g. daily-elalto-123456789).
+ * DEPLOY_SCENARIO: your scenario deployment. ingress test run only if the
+ scenario includes 'ingress'
The command becomes:
```
-docker run -v <the kube config>:/config/.kube/config -v
+docker run -v <the kube config>:/root/.kube/config -v
<result directory>:/var/lib/xtesting/results registry.gitlab.com/orange-opensour
-ce/lfn/onap/integration/xtesting:latest /bin/bash -c "run_tests -r -t all
+ce/lfn/onap/integration/xtesting/infra-healthcheck:latest:latest /bin/bash -c "
+run_tests -r -t all
```
### Output
```
-+------------+-------------+-------------------+----------+--------+
-| TEST CASE | PROJECT | TIER | DURATION | RESULT |
-+------------+-------------+-------------------+----------+--------+
-| onap-k8s | integration | infra-healthcheck | 00:06 | PASS |
-| onap-helm | integration | infra-healthcheck | 00:01 | PASS |
-+------------+-------------+-------------------+----------+--------+
++------------------+-------------+-------------------+----------+--------+
+| TEST CASE | PROJECT | TIER | DURATION | RESULT |
++------------------+-------------+-------------------+----------+--------+
+| onap-k8s | integration | infra-healthcheck | 00:06 | PASS |
+| onap-helm | integration | infra-healthcheck | 00:01 | PASS |
+| nodeport_ingress | security | security | 00:01 | FAIL |
++------------------+-------------+-------------------+----------+--------+
```
diff --git a/infra-healthcheck/docker/Dockerfile b/infra-healthcheck/docker/Dockerfile
index 849268c..07b417b 100644
--- a/infra-healthcheck/docker/Dockerfile
+++ b/infra-healthcheck/docker/Dockerfile
@@ -3,6 +3,7 @@ FROM opnfv/xtesting
ARG KUBERNETES_VERSION="v1.15.2"
ARG HELM_VERSION="v2.14.1"
ARG ONAP_TESTS_TAG=master
+ARG ONAP_TAG=master
# Install kubectl
# Note: Latest version may be found on:
@@ -10,8 +11,8 @@ ARG ONAP_TESTS_TAG=master
ADD https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl /usr/local/bin/kubectl
-COPY scripts/check_onap_k8s.sh /check_onap_k8s.sh
COPY scripts/check_onap_helm.sh /check_onap_helm.sh
+COPY upper-constraints.txt .
RUN set -x && \
apk --no-cache add --update curl ca-certificates && \
@@ -20,12 +21,16 @@ RUN set -x && \
chmod +x /usr/local/bin/kubectl && \
adduser kubectl -Du 2342 -h /config && \
wget https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm && \
+ wget -O /check_for_ingress_and_nodeports.py https://git.onap.org/integration/plain/test/security/check_for_ingress_and_nodeports.py?h=$ONAP_TAG &&\
chmod +x /usr/local/bin/helm && \
chmod +x /check_onap_*.sh && \
pip3 install --upgrade pip && \
+ pip3 install --no-cache-dir -r upper-constraints.txt && \
pip3 install --no-cache-dir \
git+https://gitlab.com/Orange-OpenSource/lfn/onap/integration/xtesting.git@$ONAP_TESTS_TAG#subdirectory=infra-healthcheck && \
apk del .build-deps
COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
+COPY docker/logging.ini /usr/lib/python3.8/site-packages/xtesting/ci/logging.ini
+COPY docker/logging.debug.ini /usr/lib/python3.8/site-packages/xtesting/ci/logging.debug.ini
CMD ["run_tests", "-t", "all"]
diff --git a/infra-healthcheck/docker/logging.debug.ini b/infra-healthcheck/docker/logging.debug.ini
new file mode 100644
index 0000000..8b2644f
--- /dev/null
+++ b/infra-healthcheck/docker/logging.debug.ini
@@ -0,0 +1,70 @@
+[loggers]
+keys=root,xtesting,ci,core,warnings,kubernetes_status,infra_healthcheck
+
+[handlers]
+keys=console,wconsole,file,dfile
+
+[formatters]
+keys=standard
+
+[logger_root]
+level=NOTSET
+handlers=dfile
+
+[logger_xtesting]
+level=NOTSET
+handlers=file
+qualname=xtesting
+
+[logger_ci]
+level=NOTSET
+handlers=console
+qualname=xtesting.ci
+
+[logger_core]
+level=NOTSET
+handlers=console
+qualname=xtesting.core
+
+[logger_warnings]
+level=NOTSET
+handlers=file,console
+qualname=py.warnings
+
+[logger_kubernetes_status]
+level=NOTSET
+handlers=wconsole,file
+qualname=kubernetes_status
+
+[logger_infra_healthcheck]
+level=NOTSET
+handlers=wconsole,file
+qualname=infra_healthcheck
+
+[handler_console]
+class=StreamHandler
+level=INFO
+formatter=standard
+args=(sys.stdout,)
+
+[handler_wconsole]
+class=StreamHandler
+level=WARN
+formatter=standard
+args=(sys.stdout,)
+
+[handler_file]
+class=FileHandler
+level=DEBUG
+formatter=standard
+args=("/var/lib/xtesting/results/xtesting.log",)
+
+[handler_dfile]
+class=FileHandler
+level=DEBUG
+formatter=standard
+args=("/var/lib/xtesting/results/xtesting.debug.log",)
+
+[formatter_standard]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=
diff --git a/infra-healthcheck/docker/logging.ini b/infra-healthcheck/docker/logging.ini
new file mode 100644
index 0000000..bdf651a
--- /dev/null
+++ b/infra-healthcheck/docker/logging.ini
@@ -0,0 +1,70 @@
+[loggers]
+keys=root,xtesting,ci,core,warnings,kubernetes_status,infra_healthcheck
+
+[handlers]
+keys=console,wconsole,file,null
+
+[formatters]
+keys=standard
+
+[logger_root]
+level=NOTSET
+handlers=null
+
+[logger_xtesting]
+level=NOTSET
+handlers=file
+qualname=xtesting
+
+[logger_ci]
+level=NOTSET
+handlers=console
+qualname=xtesting.ci
+
+[logger_core]
+level=NOTSET
+handlers=console
+qualname=xtesting.core
+
+[logger_warnings]
+level=NOTSET
+handlers=file,console
+qualname=py.warnings
+
+[logger_kubernetes_status]
+level=NOTSET
+handlers=wconsole,file
+qualname=kubernetes_status
+
+[logger_infra_healthcheck]
+level=NOTSET
+handlers=wconsole,file
+qualname=infra_healthcheck
+
+[handler_null]
+class=NullHandler
+level=NOTSET
+formatter=standard
+args=()
+
+[handler_console]
+class=StreamHandler
+level=INFO
+formatter=standard
+args=(sys.stdout,)
+
+[handler_wconsole]
+class=StreamHandler
+level=WARN
+formatter=standard
+args=(sys.stdout,)
+
+[handler_file]
+class=FileHandler
+level=DEBUG
+formatter=standard
+args=("/var/lib/xtesting/results/xtesting.log",)
+
+[formatter_standard]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=
diff --git a/infra-healthcheck/docker/testcases.yaml b/infra-healthcheck/docker/testcases.yaml
index fb30c85..346daf5 100644
--- a/infra-healthcheck/docker/testcases.yaml
+++ b/infra-healthcheck/docker/testcases.yaml
@@ -27,3 +27,15 @@ tiers:
DEPLOYED status
run:
name: 'onap_helm'
+ -
+ case_name: nodeport_ingress
+ project_name: security
+ criteria: 100
+ blocking: false
+ description: >-
+ Check that there is no NodePort without corresponding
+ Ingress port
+ dependencies:
+ - DEPLOY_SCENARIO: 'ingress'
+ run:
+ name: 'nodeport_ingress'
diff --git a/infra-healthcheck/infra_healthcheck/k8stest.py b/infra-healthcheck/infra_healthcheck/k8stest.py
index 8bb7dde..f8d618f 100644
--- a/infra-healthcheck/infra_healthcheck/k8stest.py
+++ b/infra-healthcheck/infra_healthcheck/k8stest.py
@@ -101,16 +101,6 @@ class K8sTesting(testcase.TestCase):
return res
-class OnapK8sTest(K8sTesting):
- """Kubernetes smoke test suite"""
- def __init__(self, **kwargs):
- if "case_name" not in kwargs:
- kwargs.get("case_name", 'onap-k8s')
- super(OnapK8sTest, self).__init__(**kwargs)
- self.cmd = ['/check_onap_k8s.sh']
- self.criteria_string = "Nb Failed Pods"
-
-
class OnapHelmTest(K8sTesting):
"""Kubernetes conformance test suite"""
def __init__(self, **kwargs):
@@ -119,3 +109,14 @@ class OnapHelmTest(K8sTesting):
super(OnapHelmTest, self).__init__(**kwargs)
self.cmd = ['/check_onap_helm.sh']
self.criteria_string = "Nb Failed Helm Charts"
+
+
+class OnapSecurityNodePortsIngress(K8sTesting):
+ """Check that there is no NodePort without corresponding Ingress port."""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'nodeport_ingress')
+ super(OnapSecurityNodePortsIngress, self).__init__(**kwargs)
+ self.cmd = ['python3', '/check_for_ingress_and_nodeports.py',
+ '--conf', '/root/.kube/config']
+ self.error_string = "NodePort without corresponding Ingress found"
diff --git a/infra-healthcheck/requirements.txt b/infra-healthcheck/requirements.txt
index aed40a6..91babf8 100644
--- a/infra-healthcheck/requirements.txt
+++ b/infra-healthcheck/requirements.txt
@@ -9,3 +9,6 @@ pylint>=2.1 # GPLv2
yamllint
bashate # Apache-2.0
xtesting
+kubernetes # Apache-2.0
+colorama # BSD
+kubernetes_status
diff --git a/infra-healthcheck/setup.cfg b/infra-healthcheck/setup.cfg
index 2cccb82..a3ec3db 100644
--- a/infra-healthcheck/setup.cfg
+++ b/infra-healthcheck/setup.cfg
@@ -7,5 +7,6 @@ packages = infra_healthcheck
[entry_points]
xtesting.testcase =
- onap_k8s = infra_healthcheck.k8stest:OnapK8sTest
+ onap_k8s = kubernetes_status.status:Status
onap_helm = infra_healthcheck.k8stest:OnapHelmTest
+ nodeport_ingress = infra_healthcheck.k8stest:OnapSecurityNodePortsIngress
diff --git a/infra-healthcheck/upper-constraints.txt b/infra-healthcheck/upper-constraints.txt
new file mode 100644
index 0000000..d37d29d
--- /dev/null
+++ b/infra-healthcheck/upper-constraints.txt
@@ -0,0 +1 @@
+git+https://gitlab.com/Orange-OpenSource/lfn/tools/kubernetes-status.git#egg=kubernetes_status
diff --git a/security/README.md b/security/README.md
index 3defbf8..b350078 100644
--- a/security/README.md
+++ b/security/README.md
@@ -2,10 +2,86 @@
## Goal
+This security docker includes the test suites dealing with security aspects
+of an ONAP deployment.
+
+It includes 6 tests:
+
+* root_pods: check that pods are nor using root user or started as root
+* unlimitted_pods: check that limits are set for pods
+* cis_kubernetes: perform the k8s cis test suite (upstream src aquasecurity)
+* http_public_endpoints: check that there is no public http endpoints exposed in
+ ONAP cluster
+* nonssl_endpoints: check that all public HTTP endpoints exposed in ONAP
+ cluster use SSL tunnels
+* jdpw_ports: check that there are no internal java ports
+* kube_hunter: security suite to search k8s vulnerabilities (upstream src
+ aquasecurity)
+
## Usage
### Configuration
+Mandatory:
+
+* The kubernetes configuration: usually hosted on the.kube/config of your
+ jumphost. It corresponds the kubernetes credentials and are needed to perform
+ the different operations. This file shall be copied in /root/.kube/config in
+ the docker.
+
+Optional:
+
+* The local result directory path: to store the results in your local
+ environement. It shall corresponds to the internal result docker path
+ /var/lib/xtesting/results
+
### Command
+You can run this docker by typing:
+
+```
+docker run -v <the kube config>:/root/.kube/config -v
+<result directory>:/var/lib/xtesting/results
+registry.gitlab.com/orange-opensource/lfn/onap/integration/xtesting/security:latest
+```
+
+Options:
+
+* -r: by default the reporting to the Database is not enabled. You need to
+ specify the -r option in the command line. Please note that in this case, you
+ must precise some env variables.
+
+environment variables:
+
+* Mandatory (if you want to report the results in the database):
+ * TEST_DB_URL: the url of the target Database with the env variable .
+ * NODE_NAME: the name of your test environement. It must be declared in the
+ test database (e.g. windriver-SB00)
+* Optionnal
+ * INSTALLER_TYPE: precise how your ONAP has been installed (e.g. kubespray-oom,
+ rke-oom)
+ * BUILD_TAG: a unique tag of your CI system. It can be usefull to get all the
+ tests of one CI run. It uses the regex (dai|week)ly-(.+?)-[0-9]* to find the
+ version (e.g. daily-elalto-123456789).
+
+The command becomes:
+
+```
+docker run -v <the kube config>:/root/.kube/config -v
+<result directory>:/var/lib/xtesting/results registry.gitlab.com/orange-opensour
+ce/lfn/onap/integration/xtesting/security:latest /bin/bash -c "run_tests -r -t all
+```
+
### Output
+
+```
++-----------------------+------------+------------+------------+-----------+
+| TEST CASE | PROJECT | TIER | DURATION | RESULT |
++-----------------------+------------+------------+------------+-----------+
+| root_pods | security | security | 03:48 | FAIL |
+| unlimitted_pods | security | security | 00:37 | FAIL |
+| cis_kubernetes | security | security | 00:01 | FAIL |
+| http_public_endpoints | security | security | 00:01 | FAIL |
+| jdpw_ports | security | security | 05:39 | FAIL |
++-----------------------+------------+------------+------------+-----------+
+```
diff --git a/security/docker/Dockerfile b/security/docker/Dockerfile
index 667900f..e38f565 100644
--- a/security/docker/Dockerfile
+++ b/security/docker/Dockerfile
@@ -1,12 +1,58 @@
-FROM opnfv/xtesting
+FROM golang:1.13 AS build_aqua
+WORKDIR /go/src/github.com/aquasecurity/
+RUN git clone https://github.com/aquasecurity/kube-bench.git --depth 1
+WORKDIR /go/src/github.com/aquasecurity/kube-bench
+RUN GO111MODULE=on CGO_ENABLED=0 go install -a -ldflags "-w"
+FROM golang:1.13 AS build_onap
+WORKDIR /opt/onap
+RUN git clone https://git.onap.org/integration --depth 1
+WORKDIR /opt/onap/integration/test/security/sslendpoints
+RUN CGO_ENABLED=0 go install -a -ldflags '-w -s -extldflags "-static"'
+
+FROM opnfv/xtesting AS run
+
+ARG KUBERNETES_VERSION="v1.15.2"
+ARG HELM_VERSION="v2.14.1"
ARG ONAP_TAG=master
-ARG PIP_TAG=19.3.1
+ARG ONAP_TESTS_TAG=master
+
+# Install kubectl
+# Note: Latest version may be found on:
+# https://aur.archlinux.org/packages/kubectl-bin/
+
+ADD https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl /usr/local/bin/kubectl
+
+COPY scripts/check_security_root.sh /check_security_root.sh
+COPY scripts/root_pods_xfail.txt /root_pods_xfail.txt
+COPY scripts/check_unlimitted_pods.sh /check_unlimitted_pods.sh
+COPY scripts/check_cis_kubernetes.sh /check_cis_kubernetes.sh
+COPY --from=build_aqua /go/bin/kube-bench /usr/local/bin/kube-bench
+COPY --from=build_aqua /go/src/github.com/aquasecurity/kube-bench/cfg/ /cfg/
+COPY --from=build_onap /go/bin/sslendpoints /usr/local/bin/sslendpoints
-COPY requirements.txt requirements.txt
-RUN apk --no-cache add --virtual .build-deps --update \
- openssl-dev libjpeg-turbo-dev && \
+RUN set -x && \
+ apk --no-cache add --update curl ca-certificates openssl procps util-linux \
+ nmap nmap-scripts && \
+ apk --no-cache add --virtual .build-deps --update \
+ python3-dev linux-headers gcc musl-dev && \
+ chmod +x /usr/local/bin/kubectl && \
+ git clone --depth 1 https://github.com/aquasecurity/kube-hunter.git /kube-hunter && \
+ adduser kubectl -Du 2342 -h /config && \
+ wget https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz -O - | tar -xzO linux-amd64/helm > /usr/local/bin/helm && \
+ wget -O /check_for_nonssl_endpoints.sh https://git.onap.org/integration/plain/test/security/check_for_nonssl_endpoints.sh?h=$ONAP_TAG &&\
+ wget -O /check_for_jdwp.sh https://git.onap.org/integration/plain/test/security/check_for_jdwp.sh?h=$ONAP_TAG &&\
+ wget -O /jdwp_xfail.txt https://git.onap.org/integration/plain/test/security/jdwp_xfail.txt?h=$ONAP_TAG &&\
+ wget -O /nonssl_xfail.txt https://git.onap.org/integration/plain/test/security/nonssl_xfail.txt?h=$ONAP_TAG &&\
+ chmod +x /usr/local/bin/helm && \
+ chmod +x /usr/local/bin/kube-bench && \
+ chmod +x /usr/local/bin/sslendpoints && \
+ chmod +x /check_*.sh && \
+ pip3 install --upgrade pip && \
+ pip3 install --no-cache-dir \
+ git+https://gitlab.com/Orange-OpenSource/lfn/onap/integration/xtesting.git@$ONAP_TESTS_TAG#subdirectory=security && \
+ cd /kube-hunter && pip3 install -r /kube-hunter/requirements.txt && \
apk del .build-deps
COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
-CMD ["run_test -t all -r"]
+CMD ["run_tests", "-t", "all"]
diff --git a/security/docker/testcases.yaml b/security/docker/testcases.yaml
index ed281f2..6b9d482 100644
--- a/security/docker/testcases.yaml
+++ b/security/docker/testcases.yaml
@@ -8,11 +8,67 @@ tiers:
Set of basic Functional security tests.
testcases:
-
- case_name: osji
- project_name: integration
+ case_name: root_pods
+ project_name: security
criteria: 100
blocking: false
description: >-
- run osji scan.
+ test if pods are run in root.
run:
- name: 'onap_osji'
+ name: 'root_pods'
+ -
+ case_name: unlimitted_pods
+ project_name: security
+ criteria: 100
+ blocking: false
+ description: >-
+ test if pods are run without limit.
+ run:
+ name: 'unlimitted_pods'
+ -
+ case_name: cis_kubernetes
+ project_name: security
+ criteria: 100
+ blocking: false
+ description: >-
+ test if kubernetes install is CIS compliant.
+ run:
+ name: 'cis_kubernetes'
+ -
+ case_name: http_public_endpoints
+ project_name: security
+ criteria: 100
+ blocking: false
+ description: >-
+ Check all ports exposed outside of kubernetes cluster
+ looking for plain http endpoint.
+ run:
+ name: 'http_public_endpoints'
+ -
+ case_name: nonssl_endpoints
+ project_name: security
+ criteria: 100
+ blocking: false
+ description: >-
+ Check that all ports exposed outside of kubernetes cluster
+ use SSL tunnels.
+ run:
+ name: 'nonssl_endpoints'
+ -
+ case_name: jdpw_ports
+ project_name: security
+ criteria: 100
+ blocking: false
+ description: >-
+ Check that no jdwp ports are exposed
+ run:
+ name: 'jdpw_ports'
+ -
+ case_name: kube_hunter
+ project_name: security
+ criteria: 100
+ blocking: false
+ description: >-
+ Check k8s CVE.
+ run:
+ name: 'kube_hunter'
diff --git a/security/onap_security/__init__.py b/security/onap_security/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/security/onap_security/security_tests.py b/security/onap_security/security_tests.py
new file mode 100644
index 0000000..4136f66
--- /dev/null
+++ b/security/onap_security/security_tests.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2018 All rights reserved
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+"""
+Define the parent for Kubernetes testing.
+"""
+
+from __future__ import division
+
+import logging
+import subprocess
+import time
+
+from xtesting.core import testcase
+from kubernetes import client, config
+
+
+class SecurityTesting(testcase.TestCase):
+ """Security test runner"""
+
+ __logger = logging.getLogger(__name__)
+
+ def __init__(self, **kwargs):
+ super(SecurityTesting, self).__init__(**kwargs)
+ self.cmd = []
+ self.result = 0
+ self.details = {}
+ self.start_time = 0
+ self.stop_time = 0
+ self.error_string = ""
+
+ def run_security(self): # pylint: disable=too-many-branches
+ """Run the test suites"""
+ cmd_line = self.cmd
+ self.__logger.info("Starting k8s test: '%s'.", cmd_line)
+
+ process = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ process.wait()
+ output = process.stdout.read().decode("utf-8")
+ if ('Error loading client' in output or
+ 'Unexpected error' in output):
+ raise Exception(output)
+
+ # create a log file
+ file_name = "/var/lib/xtesting/results/" + self.case_name + ".log"
+ log_file = open(file_name, "w")
+ log_file.write(output)
+ log_file.close()
+
+ # we consider the command return code for success criteria
+ if process.returncode is None:
+ success = False
+ details = self.error_string
+ if (self.case_name == 'kube_hunter' and
+ "No vulnerabilities were found" in output):
+ success = True
+ elif process.returncode != 0:
+ success = False
+ details = self.error_string
+ else:
+ success = True
+ details = "Test PASS"
+
+ self.details = details
+ self.__logger.info("details: %s", details)
+
+ if success:
+ self.result = 100
+ else:
+ self.result = 0
+
+ def run(self, **kwargs):
+ """Generic Run."""
+
+ self.start_time = time.time()
+ try:
+ self.run_security()
+ res = self.EX_OK
+ except Exception: # pylint: disable=broad-except
+ self.__logger.exception("Error with running Security tests:")
+ res = self.EX_RUN_ERROR
+
+ self.stop_time = time.time()
+ return res
+
+
+class OnapSecurityDockerRootTest(SecurityTesting):
+ """Test that the dockers launched as root."""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'root_pods')
+ super(OnapSecurityDockerRootTest, self).__init__(**kwargs)
+ self.cmd = ['/check_security_root.sh', 'onap', '-l', '/root_pods_xfail.txt']
+ self.error_string = "Pods launched with root users"
+
+
+class OnapSecurityUnlimittedPodTest(SecurityTesting):
+ """Check that no pod is launch without limits."""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'unlimitted_pods')
+ super(OnapSecurityUnlimittedPodTest, self).__init__(**kwargs)
+ self.cmd = ['/check_unlimitted_pods.sh']
+ self.error_string = "Pods lauched without limits"
+
+
+class OnapSecurityCisKubernetes(SecurityTesting):
+ """Check that kubernetes install is CIS compliant"""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'cis_kubernetes')
+ super(OnapSecurityCisKubernetes, self).__init__(**kwargs)
+ self.cmd = ['/check_cis_kubernetes.sh']
+ self.error_string = "Kubernetes Deployment is not CIS compatible"
+
+
+class OnapSecurityHttpPorts(SecurityTesting):
+ """Check all ports exposed outside of kubernetes cluster looking for plain
+ http endpoint."""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'http_public_endpoints')
+ super(OnapSecurityHttpPorts, self).__init__(**kwargs)
+ self.cmd = ['/check_for_nonssl_endpoints.sh', 'onap', '-l', '/nonssl_xfail.txt']
+ self.error_string = "Public http endpoints still found"
+
+
+class OnapSecurityNonSSLPorts(SecurityTesting):
+ """Check that all ports exposed outside of kubernetes cluster use SSL
+ tunnels."""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'nonssl_endpoints')
+ super(OnapSecurityNonSSLPorts, self).__init__(**kwargs)
+ self.cmd = ['/usr/local/bin/sslendpoints', '-xfail', '/nonssl_xfail.txt']
+ self.error_string = "Public non-SSL endpoints still found"
+
+
+class OnapSecurityJdwpPorts(SecurityTesting):
+ """Check that no jdwp ports are exposed."""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'jdpw_ports')
+ super(OnapSecurityJdwpPorts, self).__init__(**kwargs)
+ self.cmd = ['/check_for_jdwp.sh', 'onap', '-l', '/jdwp_xfail.txt']
+ self.error_string = "JDWP ports found"
+
+
+class OnapSecurityKubeHunter(SecurityTesting):
+ """Check k8s vulnerabilities."""
+ def __init__(self, **kwargs):
+ if "case_name" not in kwargs:
+ kwargs.get("case_name", 'kube_hunter')
+ super(OnapSecurityKubeHunter, self).__init__(**kwargs)
+ config.load_kube_config(config_file='/root/.kube/config')
+ client_kubernetes = client.CoreV1Api()
+ node_list = client_kubernetes.list_node()
+ kube_hunter_cmd = ['/kube-hunter/kube-hunter.py', '--remote']
+ for i in node_list.items:
+ addresses = i.status.addresses
+ for j in addresses:
+ if "External" in str(j):
+ kube_hunter_cmd.append(j.address)
+ self.cmd = kube_hunter_cmd
+ self.error_string = "Vulnerabilties detected."
diff --git a/security/onap_security/test_security_test.py b/security/onap_security/test_security_test.py
new file mode 100644
index 0000000..7906d8a
--- /dev/null
+++ b/security/onap_security/test_security_test.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2018 All rights reserved
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+"""Define the classes required to fully cover k8s."""
+
+import logging
+import os
+import unittest
+
+
+from security_tests import SecurityTesting
+
+
+class SecurityTests(unittest.TestCase):
+
+ # pylint: disable=missing-docstring
+
+ def setUp(self):
+ os.environ["DEPLOY_SCENARIO"] = "k8-test"
+ os.environ["KUBE_MASTER_IP"] = "127.0.0.1"
+ os.environ["KUBE_MASTER_URL"] = "https://127.0.0.1:6443"
+ os.environ["KUBERNETES_PROVIDER"] = "local"
+
+ self.security_stesting = SecurityTesting.SecurityTesting()
+
+ def test_run_kubetest_cmd_none(self):
+ with self.assertRaises(TypeError):
+ self.security_stesting.run_security()
+
+
+if __name__ == "__main__":
+ logging.disable(logging.CRITICAL)
+ unittest.main(verbosity=2)
diff --git a/security/requirements.txt b/security/requirements.txt
index b2c729c..6e8b944 100644
--- a/security/requirements.txt
+++ b/security/requirements.txt
@@ -1 +1,13 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+coverage!=4.4 # Apache-2.0
+mock # BSD
+nose # LGPL
+flake8>=2.5.4 # MIT
+pylint>=2.1 # GPLv2
yamllint
+bashate # Apache-2.0
+xtesting
+Kubernetes
+colorama
diff --git a/security/scripts/check_cis_kubernetes.sh b/security/scripts/check_cis_kubernetes.sh
new file mode 100644
index 0000000..33ffdf5
--- /dev/null
+++ b/security/scripts/check_cis_kubernetes.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+echo "------------------------------------------------------------------------"
+echo "-------------------- ONAP Security tests ----------------------------"
+echo "----------------- Test if K8S is CIS compliant ----------------------"
+echo "------------------------------------------------------------------------"
+
+code=0
+
+CIS_VERSION=${CIS_VERSION:-1.4}
+echo "Running CIS test case version ${CIS_VERSION}"
+kube-bench master --benchmark cis-${CIS_VERSION} > cis_full_test.txt
+cat cis_full_test.txt | grep "\[FAIL]" > cisK8s.txt
+
+if [ -s cisK8s.txt ]
+then
+ code=1
+ nb_errors=`cat cisK8s.txt | wc -l`
+ echo "Test FAIL: $nb_errors assertions not passed"
+ cat cis_full_test.txt
+else
+ echo "Test PASS: Kubernetes Deployment is CIS compatible"
+fi
+
+exit $code
diff --git a/security/scripts/check_security_root.sh b/security/scripts/check_security_root.sh
new file mode 100644
index 0000000..ca388fd
--- /dev/null
+++ b/security/scripts/check_security_root.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+usage() {
+ cat <<EOF
+Usage: $(basename $0) <k8s-namespace> [-l <white list file>]
+ -l: rooted pod xfail file
+EOF
+ exit ${1:-0}
+}
+
+if [ "$#" -lt 1 ]; then
+ usage
+ exit 1
+fi
+
+K8S_NAMESPACE=$1
+FILTERED_PODS_LIST=$(mktemp rooted_pods_XXXXXX)
+WL_RAW_FILE_PATH=$(mktemp raw_filtered_pods_XXXXXX)
+
+manage_white_list() {
+ # init filtered port list file
+ if [ ! -f $WL_FILE_PATH ];then
+ echo "File not found"
+ usage
+ fi
+ grep -o '^[^#]*' $WL_FILE_PATH > $WL_RAW_FILE_PATH
+}
+
+### getopts
+while :
+do
+ case $2 in
+ -h|--help|help) usage;;
+ -l) WL_FILE_PATH=$3;manage_white_list;shift;;
+ -*) usage 1 ;;
+ *) break ;;
+ esac
+done
+
+echo "------------------------------------------------------------------------"
+echo "-------------------- ONAP Security tests ----------------------------"
+echo "-------------------- Test root user in pods -------------------------"
+echo "------------------------------------------------------------------------"
+
+code=0
+
+# get the pod list
+for pod in `kubectl get pod -n $K8S_NAMESPACE| grep "Running" | grep -v functest | grep -v integration | awk '{print $1}'` ;do
+ list=`kubectl top pod $pod --containers -n onap |grep -v "POD"|awk '{print $1":"$2}'`;
+ for po in $list; do
+ contname=`echo $po|cut -d':' -f2`;uid=`kubectl exec $pod --container $contname -n $K8S_NAMESPACE id|sed -r "s/^uid=(.*) gid.*$/\1/"`;echo "POD: $pod container: $contname uid: $uid";
+ done;
+done | grep root > $FILTERED_PODS_LIST
+
+while IFS= read -r line; do
+ # for each line we test if it is in the white list with a regular expression
+ while IFS= read -r wl_line; do
+ wl_name=$(echo $wl_line | awk {'print $1'})
+ if grep -e $K8S_NAMESPACE-$wl_name <<< "$line" > /dev/null ;then
+ # Found in white list, exclude it
+ sed -i "/$line/d" $FILTERED_PODS_LIST
+ fi
+ done < $WL_RAW_FILE_PATH
+done < $FILTERED_PODS_LIST
+
+if [ -s $FILTERED_PODS_LIST ]
+then
+ code=1
+ nb_errors=`cat $FILTERED_PODS_LIST | wc -l`
+ echo "Test FAIL: $nb_errors pod(s) launched as root found"
+ cat $FILTERED_PODS_LIST
+else
+ echo "Test PASS: No pod launched as root found"
+fi
+
+exit $code
diff --git a/security/scripts/check_unlimitted_pods.sh b/security/scripts/check_unlimitted_pods.sh
new file mode 100644
index 0000000..fdef6f3
--- /dev/null
+++ b/security/scripts/check_unlimitted_pods.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+echo "------------------------------------------------------------------------"
+echo "-------------------- ONAP Security tests ----------------------------"
+echo "-------------------- Test pods without limit ------------------------"
+echo "------------------------------------------------------------------------"
+
+code=0
+
+# get the pod list
+for pod in `kubectl get pod -n onap|grep -v "NAME"|grep "Running\|Completed" |grep -v functest |grep -v integration | awk '{print $1}'`;do
+ kubectl describe pod $pod -n onap|grep "Limits";
+ if [ $? == 1 ] ; then
+ echo $pod ;
+ fi;
+done | grep -v Limits > NoLimitContainer.txt
+
+if [ -s NoLimitContainer.txt ]
+then
+ code=1
+ nb_errors=`cat NoLimitContainer.txt | wc -l`
+ echo "Test FAIL: $nb_errors pod(s) launched without limit"
+ cat NoLimitContainer.txt
+else
+ echo "Test PASS: No pod launched without limit"
+fi
+
+exit $code
diff --git a/security/scripts/root_pods_xfail.txt b/security/scripts/root_pods_xfail.txt
new file mode 100644
index 0000000..c282cf8
--- /dev/null
+++ b/security/scripts/root_pods_xfail.txt
@@ -0,0 +1,33 @@
+# Expected failure list for rooted ports
+# Upstream pods are excluded in Frankfurt
+# We consider only the pods we built
+aaf-cass # cassandra
+aaf-sms-vault # upstream vault and consul docker used by aaf AAF-1102
+aai # aai pods not launched as root even root user still in dockers AAI-2822
+awx # ansible
+cassandra # common cassandra
+consul # nobody remembers who is responsible for consul
+dcae-redis # redis container
+dcae-mongo # mongo container
+dcae-cloudify-manager # DCAEGEN2-2121
+mariadb # common mariadb
+msb-consul # another consul
+multicloud-fcaps # rabbit-mq upstream pod MULTICLOUD-1017
+multicloud-k8s-etcd
+multicloud-k8s-mongo
+music-cassandra # music has itw own cassandra
+nbi-mongo # a mongo db
+netbox # netbox
+pomba-elasticsearch # elasticsearch
+portal-cassandra # portal cassandra
+portal-db # portal mariadb
+portal-zookeeper # portal zookeeper
+zookeeper # common zookeper
+
+# other waivers
+robot # testing
+sniro-emulator # testing
+oof-cmso-service # testing
+vnfsdk # testing VNFSDK-565
+pomba # nobody taking cares of pomba for several releases
+dcaemod # dcae experimental pods for Frankfurt
diff --git a/security/setup.cfg b/security/setup.cfg
new file mode 100644
index 0000000..61fe7fa
--- /dev/null
+++ b/security/setup.cfg
@@ -0,0 +1,16 @@
+[metadata]
+name = onap_security
+version = 1
+
+[files]
+packages = onap_security
+
+[entry_points]
+xtesting.testcase =
+ root_pods = onap_security.security_tests:OnapSecurityDockerRootTest
+ unlimitted_pods = onap_security.security_tests:OnapSecurityUnlimittedPodTest
+ cis_kubernetes = onap_security.security_tests:OnapSecurityCisKubernetes
+ http_public_endpoints = onap_security.security_tests:OnapSecurityHttpPorts
+ nonssl_endpoints = onap_security.security_tests:OnapSecurityNonSSLPorts
+ jdpw_ports = onap_security.security_tests:OnapSecurityJdwpPorts
+ kube_hunter = onap_security.security_tests:OnapSecurityKubeHunter
diff --git a/security/setup.py b/security/setup.py
new file mode 100644
index 0000000..566d844
--- /dev/null
+++ b/security/setup.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+ import multiprocessing # noqa
+except ImportError:
+ pass
+
+setuptools.setup(
+ setup_requires=['pbr>=2.0.0'],
+ pbr=True)
diff --git a/smoke-usecases-robot/docker/Dockerfile b/smoke-usecases-robot/docker/Dockerfile
index 38616aa..8dbe29a 100644
--- a/smoke-usecases-robot/docker/Dockerfile
+++ b/smoke-usecases-robot/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM opnfv/xtesting
+FROM opnfv/xtesting:jerma
MAINTAINER Morgan Richomme <morgan.richomme@orange.com>
@@ -10,7 +10,8 @@ ARG PIP_TAG=19.3.1
ENV TAG all
COPY requirements.txt requirements.txt
-RUN apk --no-cache add --virtual .build-deps --update \
+RUN apk --no-cache add --update openssl chromium chromium-chromedriver && \
+ apk --no-cache add --virtual .build-deps --update \
python3-dev build-base linux-headers libffi-dev \
openssl-dev libjpeg-turbo-dev && \
pip3 install --upgrade pip && \
@@ -26,14 +27,15 @@ RUN apk --no-cache add --virtual .build-deps --update \
-chttps://git.opnfv.org/functest/plain/upper-constraints.txt?h=$OPNFV_TAG \
-rrequirements.txt && \
mkdir -p /var/opt/ONAP/demo/heat && cp -Rf /src/demo/heat/vFW /var/opt/ONAP/demo/heat/ && \
+ mkdir -p /var/opt/ONAP/demo/tosca && cp -Rf /src/demo/tosca/pNF /var/opt/ONAP/demo/tosca/ && \
mkdir -p /demo/service_mapping && cp -Rf /src/demo/service_mapping /demo/ && \
mkdir -p /var/opt/ONAP/demo/preload_data && cp -Rf /src/demo/preload_data /var/opt/ONAP/demo/ && \
- ln -s /usr/lib/python3.8/site-packages/vcpeutils /usr/lib/python3.8/site-packages/SoUtils && \
- ln -s /usr/lib/python3.8/site-packages/heatbridge /usr/lib/python3.8/site-packages/HeatBridge && \
+ ln -s /usr/lib/python3.7/site-packages/vcpeutils /usr/lib/python3.7/site-packages/SoUtils && \
+ ln -s /usr/lib/python3.7/site-packages/heatbridge /usr/lib/python3.7/site-packages/HeatBridge && \
rm -r requirements.txt /var/opt/ONAP/.git /src/demo && \
cd / && ln -s /var/opt/ONAP/robot/ /robot && \
apk del .build-deps
-COPY docker/testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml
+COPY docker/testcases.yaml /usr/lib/python3.7/site-packages/xtesting/ci/testcases.yaml
COPY scripts/cmd.sh /
CMD ["/cmd.sh"]
--
cgit