diff options
26 files changed, 407 insertions, 197 deletions
diff --git a/jenkins-config/clouds/openstack/cattle/centos7-docker-2c-8g.cfg b/jenkins-config/clouds/openstack/cattle/centos7-docker-2c-8g.cfg new file mode 100644 index 000000000..6a7f3ddfe --- /dev/null +++ b/jenkins-config/clouds/openstack/cattle/centos7-docker-2c-8g.cfg @@ -0,0 +1,3 @@ +IMAGE_NAME=ZZCI - CentOS 7 - docker - x86_64 - 20220919-201912.261 +LABELS=centos7-docker-2c-8g +HARDWARE_ID=v3-standard-2 diff --git a/jenkins-config/clouds/openstack/cattle/centos8-docker-2c-8g.cfg b/jenkins-config/clouds/openstack/cattle/centos8-docker-2c-8g.cfg new file mode 100644 index 000000000..9e8361fdf --- /dev/null +++ b/jenkins-config/clouds/openstack/cattle/centos8-docker-2c-8g.cfg @@ -0,0 +1,3 @@ +IMAGE_NAME=ZZCI - CentOS Stream 8 - docker - x86_64 - 20230215-011800.297 +LABELS=centos8-docker-2c-8g +HARDWARE_ID=v3-standard-2 diff --git a/jenkins-config/clouds/openstack/cattle/centos8-docker-8c-8g.cfg b/jenkins-config/clouds/openstack/cattle/centos8-docker-8c-8g.cfg index 6df78aa6c..bebde4b23 100644 --- a/jenkins-config/clouds/openstack/cattle/centos8-docker-8c-8g.cfg +++ b/jenkins-config/clouds/openstack/cattle/centos8-docker-8c-8g.cfg @@ -1,3 +1,3 @@ -IMAGE_NAME=ZZCI - CentOS Stream 8 - docker - x86_64 - 20230131-200840.321 +IMAGE_NAME=ZZCI - CentOS Stream 8 - docker - x86_64 - 20230215-011800.297 LABELS=centos8-docker-8c-8g HARDWARE_ID=v3-standard-8 diff --git a/jjb/ccsdk/apps.yaml b/jjb/ccsdk/apps.yaml index b16836987..2b277ba4f 100644 --- a/jjb/ccsdk/apps.yaml +++ b/jjb/ccsdk/apps.yaml @@ -15,11 +15,11 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - gerrit-maven-docker-stage: container-public-registry: 'nexus3.onap.org:10001' @@ -28,17 +28,16 @@ maven-versions-plugin: true cron: '@daily' - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'ccsdk/apps' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version: 'mvn38' mvn-settings: 'ccsdk-apps-settings' # due to a strange macro / variable translation problem this needs to be @@ -48,7 +47,7 @@ deployAtEnd=true files: '**' archive-artifacts: '' - build-node: centos7-docker-8c-8g + build-node: centos8-docker-8c-8g mvn-snapshot-id: 'ecomp-snapshots' nexus-snapshot-repo: 'snapshots' @@ -69,7 +68,7 @@ mvn-version: 'mvn38' mvn-settings: 'ccsdk-apps-settings' mvn-goals: 'clean install' - mvn-opts: '-Xmx1024m -XX:MaxPermSize=256m' + mvn-opts: '-Xmx1024m' - project: name: ccsdk-apps-clm diff --git a/jjb/ccsdk/ccsdk-csit.yaml b/jjb/ccsdk/ccsdk-csit.yaml index bfebf74e4..11c537411 100644 --- a/jjb/ccsdk/ccsdk-csit.yaml +++ b/jjb/ccsdk/ccsdk-csit.yaml @@ -15,8 +15,6 @@ stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' - 'kohn': branch: 'kohn' mvn-settings: 'ccsdk-distribution-settings' @@ -37,10 +35,10 @@ project: 'ccsdk/oran' stream: - 'master': + java-version: openjdk17 branch: 'master' - - 'jakarta': - branch: 'jakarta' - 'kohn': + java-version: openjdk11 branch: 'kohn' mvn-settings: 'ccsdk-oran-settings' mvn-version: 'mvn38' diff --git a/jjb/ccsdk/ccsdk-oran.yaml b/jjb/ccsdk/ccsdk-oran.yaml index 6b499393e..ec57b889b 100644 --- a/jjb/ccsdk/ccsdk-oran.yaml +++ b/jjb/ccsdk/ccsdk-oran.yaml @@ -9,12 +9,12 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos8-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' mvn-opts: '-Ddocker.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - gerrit-maven-docker-stage: build-node: centos8-docker-8c-8g container-public-registry: 'nexus3.onap.org:10001' @@ -23,16 +23,15 @@ maven-versions-plugin: true cron: '@daily' - '{project-name}-gerrit-release-jobs': - build-node: centos8-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'ccsdk/oran' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version: 'mvn38' mvn-settings: 'ccsdk-oran-settings' files: '**' @@ -87,6 +86,6 @@ mvn-version: 'mvn38' mvn-settings: 'ccsdk-oran-settings' mvn-goals: 'clean install' - mvn-opts: '-Xmx1024m -XX:MaxPermSize=256m' + mvn-opts: '-Xmx1024m' mvn-opts: '-Ddocker.skip=true' mvn-params: '-f a1-policy-management' diff --git a/jjb/ccsdk/cds.yaml b/jjb/ccsdk/cds.yaml index ae8e83393..65bef8e99 100644 --- a/jjb/ccsdk/cds.yaml +++ b/jjb/ccsdk/cds.yaml @@ -13,7 +13,7 @@ sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - gerrit-maven-docker-stage: container-public-registry: 'nexus3.onap.org:10001' container-staging-registry: 'nexus3.onap.org:10003' @@ -27,11 +27,10 @@ stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version: 'mvn38' mvn-settings: 'ccsdk-cds-settings' # due to a strange macro / variable translation problem this needs to be @@ -101,7 +100,7 @@ mvn-version: 'mvn38' mvn-settings: 'ccsdk-cds-settings' mvn-goals: 'clean install' - mvn-opts: '-Xmx1024m -XX:MaxPermSize=256m' + mvn-opts: '-Xmx1024m' - project: name: ccsdk-cds-clm diff --git a/jjb/ccsdk/distribution.yaml b/jjb/ccsdk/distribution.yaml index b48f01b0a..5f11f9263 100644 --- a/jjb/ccsdk/distribution.yaml +++ b/jjb/ccsdk/distribution.yaml @@ -9,38 +9,37 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' maven-versions-plugin: true cron: '' - sbom-generator: false + sbom-generator: true - gerrit-maven-docker-stage: - build-node: centos7-docker-8c-8g + build-node: centos8-docker-8c-8g container-public-registry: 'nexus3.onap.org:10001' container-staging-registry: 'nexus3.onap.org:10003' mvn-params: '-Dmaven.test.skip=true -Ddocker.skip.run=true -Pdocker' maven-versions-plugin: true cron: '@daily' - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'ccsdk/distribution' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk11 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk17 mvn-version: 'mvn38' mvn-settings: 'ccsdk-distribution-settings' files: '**' archive-artifacts: '' docker-pom: 'pom.xml' mvn-profile: docker - build-node: centos7-docker-8c-8g + build-node: centos8-docker-8c-8g mvn-snapshot-id: 'ecomp-snapshots' nexus-snapshot-repo: 'snapshots' @@ -60,7 +59,7 @@ # archive-artifacts: '' # docker-pom: 'pom.xml' # mvn-profile: docker -# build-node: centos7-docker-arm64-4c-2g +# build-node: centos8-docker-arm64-4c-2g # - project: name: ccsdk-distribution-clm @@ -75,7 +74,7 @@ java-version: openjdk17 mvn-version: 'mvn38' mvn-settings: 'ccsdk-distribution-settings' - build-node: centos7-docker-8c-8g + build-node: centos8-docker-8c-8g - project: @@ -83,6 +82,6 @@ project-name: ccsdk-distribution jobs: - gerrit-info-yaml-verify - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g project: ccsdk/distribution branch: master diff --git a/jjb/ccsdk/features.yaml b/jjb/ccsdk/features.yaml index b581395ac..862aea035 100644 --- a/jjb/ccsdk/features.yaml +++ b/jjb/ccsdk/features.yaml @@ -9,23 +9,24 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'ccsdk/features' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 + mvn-opts: '-Xmx3072m' - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 + mvn-opts: '-Xmx3072m -XX:MaxPermSize=1024m' mvn-version: 'mvn38' mvn-settings: 'ccsdk-features-settings' # due to a strange macro / variable translation problem this needs to be @@ -35,8 +36,7 @@ deployAtEnd=true files: '**' archive-artifacts: '' - mvn-opts: '-Xmx3072m -XX:MaxPermSize=1024m' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g mvn-snapshot-id: 'ecomp-snapshots' nexus-snapshot-repo: 'snapshots' @@ -49,7 +49,7 @@ sonarcloud-project-key: '{sonarcloud_project_organization}_{project-name}' sonar-mvn-goal: '{sonar_mvn_goal}' cron: '@daily' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g project: 'ccsdk/features' project-name: 'ccsdk-features' branch: 'master' @@ -57,7 +57,7 @@ mvn-version: 'mvn38' mvn-settings: 'ccsdk-features-settings' mvn-goals: 'clean install' - mvn-opts: '-Xmx3072m -XX:MaxPermSize=1024m' + mvn-opts: '-Xmx3072m' - project: name: ccsdk-features-clm @@ -72,14 +72,14 @@ java-version: openjdk17 mvn-version: 'mvn38' mvn-settings: 'ccsdk-features-settings' - mvn-opts: '-Xmx3072m -XX:MaxPermSize=1024m' - build-node: centos7-builder-4c-4g + mvn-opts: '-Xmx3072m' + build-node: centos8-builder-4c-4g - project: name: ccsdk-features-info project-name: ccsdk-features jobs: - gerrit-info-yaml-verify - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g project: ccsdk/features branch: master diff --git a/jjb/ccsdk/odl-legacy.yaml b/jjb/ccsdk/odl-legacy.yaml index e3546b150..5bab8f2bb 100644 --- a/jjb/ccsdk/odl-legacy.yaml +++ b/jjb/ccsdk/odl-legacy.yaml @@ -15,13 +15,13 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos8-docker-8c-8g project: 'ccsdk/odl-legacy' stream: @@ -37,7 +37,7 @@ deployAtEnd=true files: '**' archive-artifacts: '' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g mvn-snapshot-id: 'ecomp-snapshots' nexus-snapshot-repo: 'snapshots' @@ -50,7 +50,7 @@ sonarcloud-project-key: '{sonarcloud_project_organization}_{project-name}' sonar-mvn-goal: '{sonar_mvn_goal}' cron: '@daily' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g project: 'ccsdk/odl-legacy' project-name: 'ccsdk-odl-legacy' branch: 'master' @@ -58,7 +58,7 @@ mvn-version: 'mvn38' mvn-settings: 'ccsdk-odl-legacy-settings' mvn-goals: 'clean install' - mvn-opts: '-Xmx1024m -XX:MaxPermSize=256m' + mvn-opts: '-Xmx1024m' - project: name: ccsdk-odl-legacy-clm @@ -73,14 +73,14 @@ java-version: openjdk17 mvn-version: 'mvn38' mvn-settings: 'ccsdk-odl-legacy-settings' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g - project: name: ccsdk-odl-legacy-info project: ccsdk/odl-legacy project-name: ccsdk-odl-legacy - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g branch: master jobs: - gerrit-info-yaml-verify diff --git a/jjb/ccsdk/parent.yaml b/jjb/ccsdk/parent.yaml index e514e2e68..0134d6c8b 100644 --- a/jjb/ccsdk/parent.yaml +++ b/jjb/ccsdk/parent.yaml @@ -9,13 +9,13 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g mvn-snapshot-id: 'ecomp-snapshots' nexus-snapshot-repo: 'snapshots' @@ -23,6 +23,7 @@ stream: - 'master': branch: 'master' + java-version: openjdk17 jobs: - gerrit-maven-verify: mvn-params: '-Drevision=3.0.0-SNAPSHOT' @@ -32,11 +33,9 @@ mvn-params: '-Dmaven.test.skip=true -Drevision=3.0.0' - '{project-name}-gerrit-release-jobs': mvn-params: '-Drevision=3.0.0' - - 'jakarta': - branch: 'jakarta' - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version: 'mvn38' mvn-settings: 'ccsdk-parent-settings' # due to a strange macro / variable translation problem this needs to be @@ -60,13 +59,13 @@ java-version: openjdk17 mvn-version: 'mvn38' mvn-settings: 'ccsdk-parent-settings' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g - project: name: ccsdk-parent-info project-name: ccsdk-parent jobs: - gerrit-info-yaml-verify - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g project: ccsdk/parent branch: master diff --git a/jjb/ccsdk/sli.yaml b/jjb/ccsdk/sli.yaml index 1da1fdaf5..ff6eca062 100644 --- a/jjb/ccsdk/sli.yaml +++ b/jjb/ccsdk/sli.yaml @@ -9,23 +9,22 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'ccsdk/sli' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version: 'mvn38' mvn-settings: 'ccsdk-sli-settings' # due to a strange macro / variable translation problem this needs to be @@ -35,7 +34,7 @@ deployAtEnd=true files: '**' archive-artifacts: '' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g mvn-snapshot-id: 'ecomp-snapshots' nexus-snapshot-repo: 'snapshots' @@ -48,7 +47,7 @@ sonarcloud-project-key: '{sonarcloud_project_organization}_{project-name}' sonar-mvn-goal: '{sonar_mvn_goal}' cron: '@daily' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g project: 'ccsdk/sli' project-name: 'ccsdk-sli' branch: 'master' @@ -56,7 +55,7 @@ mvn-version: 'mvn38' mvn-settings: 'ccsdk-sli-settings' mvn-goals: 'clean install' - mvn-opts: '-Xmx1024m -XX:MaxPermSize=256m' + mvn-opts: '-Xmx1024m' - project: name: ccsdk-sli-clm @@ -71,13 +70,13 @@ java-version: openjdk17 mvn-version: 'mvn38' mvn-settings: 'ccsdk-sli-settings' - build-node: centos7-builder-4c-4g + build-node: centos8-builder-4c-4g - project: name: ccsdk-sli-info project-name: ccsdk-sli jobs: - gerrit-info-yaml-verify - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g project: ccsdk/sli branch: master diff --git a/jjb/doc/rules.yaml b/jjb/doc/rules.yaml new file mode 100644 index 000000000..253ca9de2 --- /dev/null +++ b/jjb/doc/rules.yaml @@ -0,0 +1,60 @@ +- scm: + name: doc-scm + scm: + - git: + url: 'https://gerrit.onap.org/gerrit/{project}' + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*' + branches: + - '{ref}' + +- trigger: + name: doc-rules-patchset-created + triggers: + - gerrit: + trigger-on: + - patchset-created-event + - comment-added-contains-event: + comment-contains-value: recheck + - comment-added-contains-event: + comment-contains-value: reverify + server-name: Primary + projects: + - project-compare-type: 'REG_EXP' + project-pattern: '*' + branches: + - branch-compare-type: 'ANT' + branch-pattern: master + - branch-compare-type: 'ANT' + branch-pattern: kohn + +- parameter: + name: doc-rules-node + parameters: + - label: + name: node + default: '{node}' + +- builder: + name: doc-rules + builders: + - shell: | + find -name tox.ini |xargs grep sphinx-build | grep -v \\-W > /dev/null && exit 1 || exit 0 + +- job-template: + name: doc-rules + triggers: + - doc-rules-patchset-created + scm: + - doc-scm: + ref: $GERRIT_REFSPEC + project: $GERRIT_PROJECT + parameters: + - doc-rules-node: + node: centos7-builder-2c-1g + builders: + - doc-rules + +- project: + name: 'doc-rules' + jobs: + - 'doc-rules' diff --git a/jjb/integration/include-raw-integration-install-robotframework-py3.sh b/jjb/integration/include-raw-integration-install-robotframework-py3.sh index 6d0b4968e..efeb0f78b 100644 --- a/jjb/integration/include-raw-integration-install-robotframework-py3.sh +++ b/jjb/integration/include-raw-integration-install-robotframework-py3.sh @@ -10,24 +10,70 @@ ############################################################################## # vim: sw=4 ts=4 sts=4 et ft=sh : -set -eu pipefail +set -euxo pipefail -# shellcheck disable=SC1090 -. ~/lf-env.sh +echo "---> install-robotframework-py3.sh" -# Create a virtual environment for robot tests and make sure setuptools & wheel -# are up-to-date in addition to pip -lf-activate-venv --python python3 --venv-file "${WORKSPACE}/.robot3_venv" \ - setuptools \ - wheel +### Common variables + +REQUIRED_PYTHON="3.7.0" + +### Common functions + +# Allows for the comparison of two Python version strings +ver_cmp() +{ + local IFS=. + # shellcheck disable=SC2206 + local V1=($1) V2=($2) I + for ((I=0 ; I<${#V1[*]} || I<${#V2[*]} ; I++)) ; do + [[ ${V1[$I]:-0} -lt ${V2[$I]:-0} ]] && echo -1 && return + [[ ${V1[$I]:-0} -gt ${V2[$I]:-0} ]] && echo 1 && return + done + echo 0 +} +# Checks if first version/string is greater than or equal to the second +ver_ge() +{ + [[ ! $(ver_cmp "$1" "$2") -eq -1 ]] +} -# Save the virtual environment in ROBOT_VENV -ROBOT3_VENV="$(cat "${WORKSPACE}/.robot3_venv")" -echo ROBOT3_VENV="${ROBOT3_VENV}" >> "${WORKSPACE}/env.properties" +### Main script entry point -set -exu +# Check for required Python versions and activate/warn appropriately +# Use PYENV for selecting the latest python version, if available +if [[ -d "/opt/pyenv" ]]; then + echo "Setup pyenv:" + export PYENV_ROOT="/opt/pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + pyenv versions + if command -v pyenv 1>/dev/null 2>&1; then + eval "$(pyenv init - --no-rehash)" + # Choose the latest numeric Python version from installed list + version=$(pyenv versions --bare | sed '/^[^0-9]/d' |\ + sort -V | tail -n 1) + pyenv local "${version}" + fi +fi -echo "Installing Python Requirements" +# Store the active/current Python3 version +PYTHON_VERSION=$(python3 --version | awk '{print $2}') + +# Check that the required minimum version has been met +if ! (ver_ge "${PYTHON_VERSION}" "${REQUIRED_PYTHON}"); then + echo "Warning: possible Python version problem" + echo "Python ${PYTHON_VERSION} does not meet requirement: ${REQUIRED_PYTHON}" +fi + +if (python3 -m robot.run --version > /dev/null 2>&1); then + echo "Working robot framework found; no installation necessary" + echo "Installed under Python version: ${PYTHON_VERSION}" + exit 0 +fi + + +# Create a requirements file; keep it around for potential later use +# Versions and dependencies below have been carefully tested for Python3 cat << 'EOF' > "requirements.txt" paramiko six @@ -79,6 +125,49 @@ PyVirtualDisplay odltools EOF -python3 -m pip install -r requirements.txt + +if [[ -f ~/lf-env.sh ]]; then + echo "Installing robot-framework using LF common tooling" + # shellcheck disable=SC1090 + source ~/lf-env.sh + + # Create a virtual environment for robot tests and make sure setuptools & wheel + # are up-to-date in addition to pip + lf-activate-venv --python python3 --venv-file "${WORKSPACE}/.robot3_venv" \ + setuptools \ + pip \ + wheel + + # Install the robot framework and other dependencies + python3 -m pip install -r requirements.txt + + # Save the virtual environment in ROBOT3_VENV + ROBOT3_VENV="$(cat "${WORKSPACE}/.robot3_venv")" + +else + echo "Installing robot-framework in a virtual environment" + if [[ -z "${WORKSPACE}" ]]; then + # Use a temporary folder location + WORKSPACE="/tmp" + ROBOT3_VENV=$(mktemp -d --suffix=-robot3_venv) + else + ROBOT3_VENV="${WORKSPACE}/.robot3_venv" + fi + + # The --system-site-packages parameter allows us to pick up system level + # installed packages. This allows us to bake matplotlib which takes very long + # to install into the image. + python3 -m venv --system-site-packages "${ROBOT3_VENV}" + source "${ROBOT3_VENV}/bin/activate" + + echo "Installing robot-framework using basic methods" + python3 -m pip install -r requirements.txt +fi + +# Store the virtual environment location +echo "ROBOT3_VENV=${ROBOT3_VENV}" >> "${WORKSPACE}/env.properties" + +# Display versioning/debugging output +python3 --version python3 -m pip freeze python3 -m robot.run --version || : diff --git a/jjb/integration/prepare-csit.sh b/jjb/integration/prepare-csit.sh index 1052bff86..0db140ff7 100644 --- a/jjb/integration/prepare-csit.sh +++ b/jjb/integration/prepare-csit.sh @@ -24,18 +24,48 @@ set -exo pipefail ROBOT_INSTALLER='include-raw-integration-install-robotframework-py3.sh' -if !(which git > /dev/null 2>&1); then - echo "GIT binary not found current PATH" - echo $PATH; exit 1 +# Allows testing for root permissions +REQ_USER=$(id -un) + +if ! (which git > /dev/null 2>&1); then + echo "GIT binary not found in current PATH" + # Add missing package to prevent script/job failures + if (grep Ubuntu /etc/os-release > /dev/null 2>&1) || \ + (grep Debian /etc/os-release > /dev/null 2>&1); then + echo "Installing package dependency for Ubuntu/Debian" + if [[ "${REQ_USER}" == 'root' ]]; then + apt-get update + apt-get install -y git + else + sudo apt-get update + sudo apt-get install -y git + fi + elif (grep RedHat /etc/os-release > /dev/null 2>&1) || \ + (grep CentOS /etc/os-release > /dev/null 2>&1); then + echo "Installing package dependency for CentOS/RedHat" + if [[ "${REQ_USER}" == 'root' ]]; then + yum install -y git + else + sudo yum install -y git + fi + else + echo "Warning: unmatched OS/distribution" + echo "Missing software will not be installed" + fi fi -if [ -z "$WORKSPACE" ]; then - # shellcheck disable=SC2155 - export WORKSPACE=`git rev-parse --show-toplevel` +if [[ -z "${WORKSPACE}" ]]; then + if (git rev-parse --show-toplevel > /dev/null 2>&1); then + WORKSPACE=$(git rev-parse --show-toplevel) + export WORKSPACE + else + WORKSPACE=$(pwd) + export WORKSPACE + fi fi # shellcheck disable=SC2034 -TESTPLANDIR=${WORKSPACE}/${TESTPLAN} +TESTPLANDIR="${WORKSPACE}/${TESTPLAN}" # Python version should match that used to setup # robot-framework in other jobs/stages @@ -56,22 +86,26 @@ fi # Assume that if ROBOT3_VENV is set, virtualenv # with system site packages can be activated -if [ -f ${WORKSPACE}/env.properties ]; then - source ${WORKSPACE}/env.properties +if [[ -f "${WORKSPACE}/env.properties" ]]; then + source "${WORKSPACE}/env.properties" +elif [[ -f /tmp/env.properties ]]; then + source /tmp/env.properties fi -if [ -f ${ROBOT3_VENV}/bin/activate ]; then - source ${ROBOT3_VENV}/bin/activate + +if [[ -f "${ROBOT3_VENV}/bin/activate" ]]; then + source "${ROBOT3_VENV}/bin/activate" else # Robot framework was not found # clone ci-management repository and use install script git clone "https://gerrit.onap.org/r/ci-management" \ /tmp/ci-management - source /tmp/ci-management/jjb/integration/${ROBOT_INSTALLER} + # shellcheck disable=SC1090 + source "/tmp/ci-management/jjb/integration/${ROBOT_INSTALLER}" fi # install eteutils -mkdir -p ${ROBOT3_VENV}/src/onap -rm -rf ${ROBOT3_VENV}/src/onap/testsuite +mkdir -p "${ROBOT3_VENV}/src/onap" +rm -rf "${ROBOT3_VENV}/src/onap/testsuite" # Source from the Nexus repository python3 -m pip install --upgrade \ --extra-index-url="https://nexus3.onap.org/repository/PyPi.staging/simple" \ diff --git a/jjb/integration/run-csit.sh b/jjb/integration/run-csit.sh index bd35ac438..877cebbba 100644 --- a/jjb/integration/run-csit.sh +++ b/jjb/integration/run-csit.sh @@ -15,13 +15,35 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# $1 project/functionality -# $2 robot options +# $1 project/functionality {TESTPLAN} +# $2 robot options {TESTOPTIONS} echo "---> run-csit.sh" WORKDIR=$(mktemp -d --suffix=-robot-workdir) +# Exit if no arguments are provided and required variables not set +if [[ $# -eq 0 ]] && [[ -z "${TESTPLAN}" ]] && [[ -z "${TESTOPTIONS}" ]]; then + echo + echo "Usage: $0 plans/<project>/<functionality> [<robot-options>]" + echo + echo " <project>, <functionality>, <robot-options>: " + echo " The same values as for the JJB job template:" + echo ' {project}-csit-{functionality}' + echo + exit 1 + +elif [[ $# -ne 2 ]] && [[ -z "${TESTPLAN}" ]] && [[ -z "${TESTOPTIONS}" ]]; then + echo + echo "Script called without arguments, but the following variables" + echo " must be set: {TESTPLAN} {TESTOPTIONS}" + echo + exit 1 + +elif [[ $# -eq 2 ]]; then + export TESTPLAN=$1; export TESTOPTIONS=$2 +fi + # Python version should match that used to setup # robot-framework in other jobs/stages # Use pyenv for selecting the python version @@ -43,12 +65,52 @@ fi # functions # +# load the saved set options +function load_set { + _setopts="$-" + + # bash shellopts + for i in $(echo "$SHELLOPTS" | tr ':' ' ') ; do + set +o "${i}" + done + for i in $(echo "$RUN_CSIT_SHELLOPTS" | tr ':' ' ') ; do + set -o "${i}" + done + + # other options + for i in $(echo "$_setopts" | sed 's/./& /g') ; do + set +"${i}" + done + set -"${RUN_CSIT_SAVE_SET}" +} + +# set options for quick bailout when error +function harden_set { + set -xeo pipefail + set +u # enabled it would probably fail too many often +} + +# relax set options so the sourced file will not fail +# the responsibility is shifted to the sourced file... +function relax_set { + set +e + set +o pipefail +} + +# save current set options +function save_set { + RUN_CSIT_SAVE_SET="$-" + RUN_CSIT_SHELLOPTS="$SHELLOPTS" +} + # wrapper for sourcing a file function source_safely { - [ -z "$1" ] && return 1 - relax_set - . "$1" - load_set + if [[ -z "$1" ]] && return 1; then + relax_set + # shellcheck disable=SC1090 + source "$1" + load_set + fi } function on_exit { @@ -66,11 +128,10 @@ function on_exit { # Run teardown script plan if it exists cd "${TESTPLANDIR}" TEARDOWN="${TESTPLANDIR}/teardown.sh" - if [ -f "${TEARDOWN}" ]; then + if [[ -f "${TEARDOWN}" ]]; then echo "Running teardown script ${TEARDOWN}" source_safely "${TEARDOWN}" fi - # TODO: do something with the output exit $rc } # ensure that teardown and other finalizing steps are always executed @@ -96,44 +157,6 @@ function docker_stats { echo } -# save current set options -function save_set { - RUN_CSIT_SAVE_SET="$-" - RUN_CSIT_SHELLOPTS="$SHELLOPTS" -} - -# load the saved set options -function load_set { - _setopts="$-" - - # bash shellopts - for i in $(echo "$SHELLOPTS" | tr ':' ' ') ; do - set +o ${i} - done - for i in $(echo "$RUN_CSIT_SHELLOPTS" | tr ':' ' ') ; do - set -o ${i} - done - - # other options - for i in $(echo "$_setopts" | sed 's/./& /g') ; do - set +${i} - done - set -${RUN_CSIT_SAVE_SET} -} - -# set options for quick bailout when error -function harden_set { - set -xeo pipefail - set +u # enabled it would probably fail too many often -} - -# relax set options so the sourced file will not fail -# the responsibility is shifted to the sourced file... -function relax_set { - set +e - set +o pipefail -} - # # main # @@ -141,29 +164,21 @@ function relax_set { # set and save options for quick failure harden_set && save_set -if [ $# -eq 0 ]; then - echo - echo "Usage: $0 plans/<project>/<functionality> [<robot-options>]" - echo - echo " <project>, <functionality>, <robot-options>: " - echo " The same values as for the '{project}-csit-{functionality}' JJB job template." - echo - exit 1 -fi - -if [ -z "$WORKSPACE" ]; then - export WORKSPACE=$(git rev-parse --show-toplevel) +if [[ -z "${WORKSPACE}" ]]; then + if (git rev-parse --show-toplevel > /dev/null 2>&1); then + WORKSPACE=$(git rev-parse --show-toplevel) + export WORKSPACE + else + WORKSPACE=$(pwd) + export WORKSPACE + fi fi -if [ -f "${WORKSPACE}/${1}/testplan.txt" ]; then - export TESTPLAN="${1}" -else +if [[ ! -f "${WORKSPACE}/${TESTPLAN}/testplan.txt" ]]; then echo "testplan not found: ${WORKSPACE}/${TESTPLAN}/testplan.txt" exit 2 fi -export TESTOPTIONS="${2}" - rm -rf "$WORKSPACE/archives/$TESTPLAN" mkdir -p "$WORKSPACE/archives/$TESTPLAN" @@ -178,7 +193,7 @@ source_safely "${ROBOT3_VENV}/bin/activate" cd "${WORKDIR}" # Add csit scripts to PATH -export PATH="${PATH}:${WORKSPACE}/docker/scripts:${WORKSPACE}/scripts:${ROBOT_VENV}/bin" +export PATH="${PATH}:${WORKSPACE}/docker/scripts:${WORKSPACE}/scripts:${ROBOT3_VENV}/bin" export SCRIPTS="${WORKSPACE}/scripts" export ROBOT_VARIABLES= @@ -199,7 +214,8 @@ docker_stats | tee "$WORKSPACE/archives/$TESTPLAN/_sysinfo-1-after-setup.txt" # Run test plan cd "$WORKDIR" echo "Reading the testplan:" -cat "${TESTPLANDIR}/testplan.txt" | egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' | sed "s|^|${WORKSPACE}/tests/|" > testplan.txt +grep -E -v '(^[[:space:]]*#|^[[:space:]]*$)' "${TESTPLANDIR}/testplan.txt" |\ + sed "s|^|${WORKSPACE}/tests/|" > testplan.txt cat testplan.txt SUITES=$( xargs -a testplan.txt ) @@ -212,10 +228,9 @@ python3 --version pip freeze python3 -m robot.run --version || : -python -m robot.run -N ${TESTPLAN} -v WORKSPACE:/tmp ${ROBOT_VARIABLES} ${TESTOPTIONS} ${SUITES} +python -m robot.run -N "${TESTPLAN}" -v WORKSPACE:/tmp "${ROBOT_VARIABLES}" "${TESTOPTIONS}" "${SUITES}" RESULT=$? load_set echo "RESULT: $RESULT" # Note that the final steps are done in on_exit function after this exit! exit $RESULT - diff --git a/jjb/portal-ng/portal-ng.yaml b/jjb/portal-ng/portal-ng.yaml new file mode 100644 index 000000000..8bc5d1351 --- /dev/null +++ b/jjb/portal-ng/portal-ng.yaml @@ -0,0 +1,15 @@ +--- +- project: + name: portal-ng-project-view + project-name: portal-ng + views: + - project-view + +- project: + name: portal-ng-info + project: portal-ng + project-name: portal-ng + build-node: centos7-builder-2c-1g + branch: master + jobs: + - gerrit-info-yaml-verify
\ No newline at end of file diff --git a/jjb/sdc/sdc-sdc-docker-base.yaml b/jjb/sdc/sdc-sdc-docker-base.yaml index 0a894db11..3798685c3 100644 --- a/jjb/sdc/sdc-sdc-docker-base.yaml +++ b/jjb/sdc/sdc-sdc-docker-base.yaml @@ -12,6 +12,7 @@ docker-pom: 'pom.xml' mvn-profile: 'default' - gerrit-maven-docker-stage: + sbom-generator: true build-node: ubuntu1804-docker-8c-8g maven-versions-plugin: true mvn-params: '-P default' diff --git a/jjb/sdc/sdc-sdc-helm-validator.yaml b/jjb/sdc/sdc-sdc-helm-validator.yaml index 26ca8fe42..987790423 100644 --- a/jjb/sdc/sdc-sdc-helm-validator.yaml +++ b/jjb/sdc/sdc-sdc-helm-validator.yaml @@ -27,6 +27,7 @@ maven-versions-plugin: true mvn-goals: 'clean deploy' - gerrit-maven-docker-stage: + sbom-generator: true build-node: ubuntu1804-docker-8c-8g maven-versions-plugin: true mvn-goals: 'clean install docker:build docker:push' diff --git a/jjb/sdc/sdc-sdc-workflow-designer.yaml b/jjb/sdc/sdc-sdc-workflow-designer.yaml index 409b1de62..68a03fcae 100644 --- a/jjb/sdc/sdc-sdc-workflow-designer.yaml +++ b/jjb/sdc/sdc-sdc-workflow-designer.yaml @@ -18,6 +18,7 @@ build-node: centos7-builder-4c-4g maven-versions-plugin: true - gerrit-maven-docker-stage: + sbom-generator: true build-node: ubuntu1804-docker-8c-8g maven-versions-plugin: true mvn-params: '-P docker -Dmaven.test.skip=true' diff --git a/jjb/sdc/sdc.yaml b/jjb/sdc/sdc.yaml index 6a754e5da..2a330ed43 100644 --- a/jjb/sdc/sdc.yaml +++ b/jjb/sdc/sdc.yaml @@ -55,6 +55,7 @@ mvn-version: 'mvn36' mvn-params: '-DskipTests=true -Dnpm.registry=https://nexus3.onap.org/repository/npm.public/npm/-/' - gerrit-maven-docker-stage: + sbom-generator: true build-node: ubuntu1804-docker-v1-8c-8g maven-versions-plugin: true mvn-params: '-P docker -DskipTests=true -Dnpm.registry=https://nexus3.onap.org/repository/npm.public/npm/-/' diff --git a/jjb/sdnc/sdnc-apps.yaml b/jjb/sdnc/sdnc-apps.yaml index b92ed487b..706184ba7 100644 --- a/jjb/sdnc/sdnc-apps.yaml +++ b/jjb/sdnc/sdnc-apps.yaml @@ -18,11 +18,11 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - gerrit-maven-docker-stage: build-node: ubuntu1804-docker-8c-8g container-public-registry: 'nexus3.onap.org:10001' @@ -31,17 +31,16 @@ maven-versions-plugin: true cron: '@daily' - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'sdnc/apps' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version : "mvn38" mvn-settings: 'sdnc-apps-settings' @@ -74,7 +73,7 @@ mvn-version : "mvn38" mvn-settings: 'sdnc-apps-settings' mvn-goals: 'clean install' - mvn-opts: '-Xmx1024m -XX:MaxPermSize=256m' + mvn-opts: '-Xmx1024m' maven-version: "mvn38" # @@ -115,6 +114,6 @@ project-name: sdnc-apps jobs: - gerrit-info-yaml-verify - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g project: sdnc/apps branch: master diff --git a/jjb/sdnc/sdnc-csit.yaml b/jjb/sdnc/sdnc-csit.yaml index c427e82f8..582fee02b 100644 --- a/jjb/sdnc/sdnc-csit.yaml +++ b/jjb/sdnc/sdnc-csit.yaml @@ -15,8 +15,6 @@ stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' - 'kohn': branch: 'kohn' mvn-settings: 'sdnc-oam-settings' diff --git a/jjb/sdnc/sdnc-northbound.yaml b/jjb/sdnc/sdnc-northbound.yaml index a2f070fc4..1330b3b79 100644 --- a/jjb/sdnc/sdnc-northbound.yaml +++ b/jjb/sdnc/sdnc-northbound.yaml @@ -8,21 +8,20 @@ cron: '@daily' - gerrit-maven-stage: sign-artifacts: true - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g cron: '' - sbom-generator: false + sbom-generator: true - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'sdnc/northbound' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version: "mvn38" mvn-settings: 'sdnc-northbound-settings' @@ -55,7 +54,7 @@ mvn-goals: 'clean install' java-version: openjdk17 mvn-version: "mvn38" - mvn-opts: '-Xmx1024m -XX:MaxPermSize=256m' + mvn-opts: '-Xmx1024m' - project: name: sdnc-northbound-clm @@ -77,6 +76,6 @@ project-name: sdnc-northbound jobs: - gerrit-info-yaml-verify - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g project: sdnc/northbound branch: master diff --git a/jjb/sdnc/sdnc-oam.yaml b/jjb/sdnc/sdnc-oam.yaml index 9ffb8296c..4c2e6f05c 100644 --- a/jjb/sdnc/sdnc-oam.yaml +++ b/jjb/sdnc/sdnc-oam.yaml @@ -9,11 +9,11 @@ cron: '@daily' mvn-params: '-Dmaven.test.skip=true' - gerrit-maven-stage: - build-node: centos7-builder-4c-4g + build-node: centos8-docker-8c-8g sign-artifacts: true mvn-params: '-Dmaven.test.skip=true' cron: '' - sbom-generator: false + sbom-generator: true - gerrit-maven-docker-stage: build-node: ubuntu1804-docker-8c-8g container-public-registry: 'nexus3.onap.org:10001' @@ -22,16 +22,15 @@ maven-versions-plugin: true cron: '@daily' - '{project-name}-gerrit-release-jobs': - build-node: centos7-docker-8c-8g + build-node: centos7-docker-2c-8g project: 'sdnc/oam' stream: - 'master': branch: 'master' - - 'jakarta': - branch: 'jakarta' + java-version: openjdk17 - 'kohn': branch: 'kohn' - java-version: openjdk17 + java-version: openjdk11 mvn-version: "mvn38" mvn-settings: 'sdnc-oam-settings' files: '**' @@ -79,6 +78,6 @@ project-name: sdnc-oam jobs: - gerrit-info-yaml-verify - build-node: centos7-builder-2c-1g + build-node: centos8-builder-2c-1g project: sdnc/oam branch: master diff --git a/jjb/testsuite/testsuite-docker.sh b/jjb/testsuite/testsuite-docker.sh index ca5e95aba..2ac64365e 100644 --- a/jjb/testsuite/testsuite-docker.sh +++ b/jjb/testsuite/testsuite-docker.sh @@ -2,7 +2,7 @@ # echo '============== CALLING SCRIPT TO CREATE DOCKER IMAGES =================' cp $WORKSPACE/docker/* . -docker -D build -t onap/testsuite . +docker -D build --no-cache -t onap/testsuite . export REPO="nexus3.onap.org:10003" DATETIME_STAMP=$(date +%Y%m%dT%H%M%S) |