diff options
Diffstat (limited to 'build')
| -rwxr-xr-x | build/create_repo.sh | 118 | ||||
| -rwxr-xr-x | build/creating_data/docker-images-collector.sh | 6 | ||||
| -rw-r--r-- | build/data_lists/additional_packages.list | 1 | ||||
| -rw-r--r-- | build/data_lists/onap_deb.list | 57 | ||||
| -rwxr-xr-x | build/docker-entrypoint.sh | 160 | ||||
| -rwxr-xr-x | build/package.py | 49 | ||||
| -rw-r--r-- | build/requirements.txt | 2 |
7 files changed, 340 insertions, 53 deletions
diff --git a/build/create_repo.sh b/build/create_repo.sh index eaf0ee30..fa53e688 100755 --- a/build/create_repo.sh +++ b/build/create_repo.sh @@ -1,26 +1,66 @@ #!/usr/bin/env bash -container_name="centos_repo" +# Set type of distribution +distro_type="$(cat /etc/*-release | grep -w "ID" | awk -F'=' '{ print $2 }' | tr -d '"')" + # Path to folder with cloned offline-installer build directory with docker_entrypoint script volume_offline_directory="$(readlink -f $(dirname ${0}))" + # Path for directory where repository will be created volume_repo_directory="$(pwd)" + # Path inside container with cloned offline-installer build directory container_offline_volume="/mnt/offline/" + # Path inside container where will be created repository container_repo_volume="/mnt/repo/" -# Docker image name and version -docker_image="centos:centos7.6.1810" -# Expected directory for RPM packages -expected_dir="resources/pkg/rpm" +# Path inside container where will be stored additional packages lists +container_list_volume="/mnt/additional-lists/" + +# Show help for using this script help () { - echo "Script for run docker container with RPM repository" - echo "usage: create_repo.sh [-d|--destination-repository output directory] [-c|--cloned-directory input directory]" - echo "-h --help: Show this help" - echo "-d --destination-repository: set path where will be stored RPM packages. Default value is current directory" - echo "-c --cloned-directory: set path where is stored this script and docker-entrypoint script (offline-installer/build directory). Fill it just when you want to use different script/datalists" - echo "If build folder from offline repository is not specified will be used default path of current folder." +cat <<EOF +Script for run docker container creating DEB or RPM repository + +Type of repository is created based on user input or if input is empty type of host OS + +usage: create_repo.sh [-d|--destination-repository output directory] [-c|--cloned-directory input directory] + [-t|--target-platform centos target platform for repository] + [-a|----additional-lists path to additional package list] +-h --help: Show this help +-d --destination-repository: set path where will be stored RPM packages. Default value is current directory +-c --cloned-directory: set path where is stored this script and docker-entrypoint script (offline-installer/build directory). Fill it just when you want to use different script/datalists +-t --target-platform: set target platform for repository (ubuntu/rhel/centos) +-a --additional-list: add additional packages list + can be used multiple times for more additional lists + +If build folder from offline repository is not specified will be used default path of current folder. +EOF +} + +# Get type of distribution +# Set Docker image name and version based on type of linux distribution +# Set expected directory for RPM/DEB packages +set_enviroment () { + case "$1" in + ubuntu) + distro_type="ubuntu" + docker_image="ubuntu:18.04" + expected_dir="resources/pkg/deb" + container_name=$1"_repo" + ;; + centos|rhel) + distro_type="rhel" + docker_image="centos:centos7.6.1810" + expected_dir="resources/pkg/rpm" + container_name=$1"_repo" + ;; + *) + echo "Unknown type of linux distribution." + exit 1 + ;; + esac } # Getting input parametters @@ -29,6 +69,7 @@ if [[ $# -eq 0 ]] ; then help # show help exit 0 fi + while [[ $# -gt 0 ]] do case "$1" in @@ -47,6 +88,16 @@ do # Sets path where will be repository created volume_repo_directory="$2" ;; + -t|--target-platform) + # Repository type (rpm/deb) + # Sets target platform for repository + target_input="$2" + ;; + -a|--additional-list) + # Array with more packages lists + # Add more packages lists to download + additional_lists+=("$2") + ;; *) # unknown option help # show help @@ -56,10 +107,28 @@ do shift;shift done -# Check if path contains expected path "resources/pkg/rpm" +# Check if user specified type of repository +# This settings have higher priority, then type of distribution +if ! test -z "$target_input" +then + set_enviroment "$target_input" +else + set_enviroment "$distro_type" +fi + +# Check if path contains expected path: +# "resources/pkg/rpm" for Rhel/CentOS or +# "resources/pkg/deb" for Ubuntu/Debian if ! [[ "/$volume_repo_directory/" = *"/$expected_dir/"* ]]; then # Create repo folder if it not exists - volume_repo_directory="$volume_repo_directory"/resources/pkg/rpm + case "$distro_type" in + ubuntu) + volume_repo_directory="$volume_repo_directory"/resources/pkg/deb + ;; + rhel) + volume_repo_directory="$volume_repo_directory"/resources/pkg/rhel + ;; + esac [ ! -d "$volume_repo_directory" ] && mkdir -p $volume_repo_directory fi @@ -72,15 +141,28 @@ if [ ! "$(docker ps -q -f name=$container_name)" ]; then # run repo container # name of container $container_name # docker entrypoint script from mounted volume - # + # with dynamic parameters + # mount additional packages lists to container + param_array=() + mounted_lists=() + param_array+=(--directory ${container_repo_volume}) + param_array+=(--list ${container_offline_volume}data_lists/) + param_array+=(--packages-lists-path ${container_list_volume}) + [[ ! ${#additional_lists[@]} -eq 0 ]] && \ + for array_list in "${additional_lists[@]}"; + do + param_array+=(--additional-list "${array_list##*/}") && \ + mounted_lists+=(-v ${array_list}:${container_list_volume}${array_list##*/}) + done + docker run -d \ --name $container_name \ -v ${volume_offline_directory}:${container_offline_volume} \ -v ${volume_repo_directory}:${container_repo_volume} \ + "${mounted_lists[@]}" \ --rm \ --entrypoint="${container_offline_volume}docker-entrypoint.sh" \ - -it ${docker_image} \ - --directory ${container_repo_volume} \ - --list ${container_offline_volume}data_lists/ - docker logs $(docker ps --filter "name=centos_repo" --format '{{.ID}}' -a) -f + -it ${docker_image} \ + "${param_array[@]}" + docker logs $(docker ps --filter "name=${container_name}" --format '{{.ID}}' -a) -f fi diff --git a/build/creating_data/docker-images-collector.sh b/build/creating_data/docker-images-collector.sh index c07de107..76ee9016 100755 --- a/build/creating_data/docker-images-collector.sh +++ b/build/creating_data/docker-images-collector.sh @@ -40,15 +40,13 @@ usage () { } parse_yaml() { -python - <<PYP -#!/usr/bin/python -from __future__ import print_function +python3 - <<PYP +#!/usr/bin/python3 import yaml import sys with open("${1}", 'r') as f: values = yaml.load(f, Loader=yaml.SafeLoader) - enabled = filter(lambda x: values[x].get('enabled', False) == True, values) print(' '.join(enabled)) PYP diff --git a/build/data_lists/additional_packages.list b/build/data_lists/additional_packages.list index 98d1dda1..e43bd52e 100644 --- a/build/data_lists/additional_packages.list +++ b/build/data_lists/additional_packages.list @@ -1,2 +1,3 @@ jq screen +git diff --git a/build/data_lists/onap_deb.list b/build/data_lists/onap_deb.list new file mode 100644 index 00000000..fcc6391d --- /dev/null +++ b/build/data_lists/onap_deb.list @@ -0,0 +1,57 @@ +docker-ce=5:18.09.5~3-0~ubuntu-bionic +docker-ce-cli=5:18.09.5~3-0~ubuntu-bionic +containerd.io=1.2.2-3 +pigz +libltdl7 +cgroupfs-mount +aufs-tools +bridge-utils +runc +ubuntu-fan +golang-docker-credential-helpers +libsecret-common +python3-docker +python3-dockerpycreds +python3-websocket +gssproxy +libbasicobjects0 +libcollection4 +libgssrpc4 +libini-config5 +libpath-utils1 +libref-array1 +libverto1 +libverto-libevent1 +keyutils +libnfsidmap2 +libtirpc1 +nfs-common +nfs-kernel-server +rpcbind +chrony +libnspr4 +build-essential +cpp +dpkg-dev +g++ +g++-7 +libcc-0 +libcc-7-dev +libgomp1 +libitm1 +libatomic1 +libasan4 +liblsan0 +libtsan0 +libubsan0 +libcilkrts5 +libmpx2 +libquadmath0 +libc6-dev +gcc +gcc-7 +libc6-dev +libc-dev +make +binutils +resolvconf diff --git a/build/docker-entrypoint.sh b/build/docker-entrypoint.sh index 14f6aaa7..b3306e26 100755 --- a/build/docker-entrypoint.sh +++ b/build/docker-entrypoint.sh @@ -1,21 +1,47 @@ #!/usr/bin/env bash +# Set type of distribution where script is running +distro_type=$(cat /etc/*-release | grep -w "ID" | awk -F'=' '{ print $2 }' | tr -d '"') +case "$distro_type" in + ubuntu) + distro_type="ubuntu" + ;; + rhel|centos) + distro_type="rhel" + ;; + *) + echo "Unknown type of linux distribution." + exit 1 + ;; +esac + # Path where will be created repository (in container) OFFLINE_REPO_DIR="" -# Path where is stored onap_rpm.list file -RPM_LIST_DIR="" +# Path where is stored onap_rpm.list and onap_deb.list file +PCKG_LIST_DIR="" + +# Path where is stored additional packages lists +ADD_LIST_DIR="" +# Show help for using this script help () { - echo -e "Docker entrypoint script for creating RPM repository\n" - echo "usage: create-repo.sh [-d|--directory output directory] [-l|--list input rpm list directory]" - echo "-h --help: Show this help" - echo "-d --directory: set path for repo directory in container" - echo -e "-l --list: set path where rpm list is stored in container\n" - echo "Both paths have to be set with shared volume between" - echo "container and host computer. Default path in container is: /tmp/" - echo "Repository will be created at: /<path>/resources/pkg/rpm/" - echo "RMP list is stored at: ./data_list/" +cat <<EOF +Docker entrypoint script for creating RPM/DEB repository based on linux distribution where script is running + +usage: create-repo.sh [-d|--directory output directory] [-l|--list input rpm/deb list directory] [-a|--additional-lists list1.list] +-h --help: Show this help +-d --directory: set path for repo directory in container +-l --list: set path where rpm or deb list is stored in container +-a --additional-list: add name of additional packages list + can be used multiple times for more additional lists +-p --packages-lists-path: set path for other additional packages lists + +Both paths have to be set with shared volume between +container and host computer. Default path in container is: /tmp/ +Repository will be created at: /<path>/resources/pkg/rhel/ +RMP/DEB list is stored at: ./data_list/ +EOF } # Getting input parametters @@ -39,8 +65,17 @@ do ;; -l|--list) # List parametter - # Sets path where is stored onap_rpm.list file - RPM_LIST_DIR="$2" + # Sets path where is stored onap_rpm.list or onap_deb.list file + PCKG_LIST_DIR="$2" + ;; + -p|--packages-lists-path) + # Path parametter + # Sets path where is stored additional packages lists + ADD_LIST_DIR="$2" + ;; + -a|--additional-list) + # Array of additional packages lists + ADDITIONAL_LISTS+=("$2") ;; *) # unknown option @@ -52,7 +87,10 @@ do done # Testing if directory parametter was used -# If not variable is sets to default value /tmp/repo/resources/pkg/rpm +# If not variable is sets to default value: +# /tmp/repo/resources/pkg/rpm +# or +# /tmp/repo/resources/pkg/deb if test -z "$OFFLINE_REPO_DIR" then OFFLINE_REPO_DIR="/tmp/repo/" @@ -60,21 +98,93 @@ fi # Testing if list parametter was used # If not variable is sets to default value /tmp/offline/data-list -if test -z "$RPM_LIST_DIR" +if test -z "$PCKG_LIST_DIR" then - RPM_LIST_DIR="/tmp/offline/data_list/" + PCKG_LIST_DIR="/tmp/offline/data_list/" +fi +# Testing if additional packages list parametter was used +# If not variable is sets to default value /tmp/additional-lists +if test -z "$PCKG_LIST_DIR" +then + PCKG_LIST_DIR="/tmp/additional-lists/" fi -# Install createrepo package for create repository in folder -# and yum-utils due to yum-config-manager for adding docker repository -yum install createrepo yum-utils -y +case "$distro_type" in + ubuntu) + # Change current working dir + pushd $OFFLINE_REPO_DIR + + # Install dpkg-deb package for create repository in folder + # Install software-properties-common to get add-apt-repository command + # Install apt-transport-https, ca-certificates, curl and gnupg-agent allowing apt to use a repository over HTTPS + apt-get update -y + apt-get install dpkg-dev apt-transport-https ca-certificates curl gnupg-agent software-properties-common -y + + # Add Docker's official GPG key: + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + apt-key fingerprint 0EBFCD88 + + # Add docker repository + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + # Temp fix of known bug + # https://bugs.launchpad.net/ubuntu/+source/aptitude/+bug/1543280 + chown _apt $OFFLINE_REPO_DIR + + # Download all packages from onap_deb.list via apt-get to repository folder + for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}');do apt-get download $i -y; done + for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}'); + do + for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/); + do apt-get download $depends -y; + done; + done + + # Download all packages with dependecies from all additional packages lists via apt-get to repository folder + if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then + for list in ${ADDITIONAL_LISTS[@]} + do + for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do apt-get download $i -y; done + for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}'); + do + for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/); + do apt-get download $depends -y; + done; + done + done + fi + + # In repository folder create gz package with deb packages + dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz + ;; + + rhel) + # Install createrepo package for create repository in folder, + # yum-utils due to yum-config-manager for adding docker repository + # and epel-release for additional packages (like jq etc.) + yum install createrepo yum-utils epel-release -y + + # Add official docker repository + yum-config-manager --add-repo=https://download.docker.com/linux/centos/7/x86_64/stable/ + + # Download all packages from onap_rpm.list via yumdownloader to repository folder + for i in $(cat ${PCKG_LIST_DIR}onap_rpm.list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done -# Add official docker repository -yum-config-manager --add-repo=https://download.docker.com/linux/centos/7/x86_64/stable/ + # Download all packages from all additional packages lists via apt-get to repository folder + if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then + for list in ${ADDITIONAL_LISTS[@]} + do + for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done + done + fi -# Download all packages from onap_rpm.list via yumdownloader to repository folder -for i in $(cat ${RPM_LIST_DIR}onap_rpm.list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done + # In repository folder create repositor + createrepo $OFFLINE_REPO_DIR + ;; -# In repository folder create repository -createrepo $OFFLINE_REPO_DIR + *) + echo "Unknown type of linux distribution." + exit 1 + ;; +esac diff --git a/build/package.py b/build/package.py index c0ca47a5..9e37d4bf 100755 --- a/build/package.py +++ b/build/package.py @@ -28,6 +28,7 @@ import glob import json import sys import os +import hashlib import tarfile import git @@ -82,20 +83,43 @@ def create_package_info_file(output_file, repository_list, tag, metadata): build_info = { 'Build_info': { 'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'), - 'Version': tag + 'Version': tag, + 'Packages': {} } } for repository in repository_list: build_info['Build_info'][ repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha - if len(metadata) != 0: - build_info['Build_info'][metadata[0]] = metadata[1] + if metadata: + for meta in metadata: + build_info['Build_info'].update(meta) with open(output_file, 'w') as outfile: json.dump(build_info, outfile, indent=4) +def add_checksum_info(output_dir): + """ + Add checksum information into package.info file + :param output_dir: directory where are packages + """ + tar_files = ['resources_package.tar', 'aux_package.tar', 'sw_package.tar'] + for tar_file in tar_files: + try: + checksum = hashlib.md5() + with open(os.path.join(output_dir, tar_file), 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + checksum.update(chunk) + with open(os.path.join(output_dir, 'package.info'), 'r') as f: + json_data = json.load(f) + json_data['Build_info']['Packages'].update({tar_file: checksum.hexdigest()}) + with open(os.path.join(output_dir, 'package.info'), 'w') as f: + json.dump(json_data, f, indent=4) + except FileNotFoundError: + pass + + def create_package(tar_content, file_name): """ Creates packages @@ -109,6 +133,20 @@ def create_package(tar_content, file_name): output_tar_file.add(src, dst) +def metadata_validation(param): + """ + Validation of metadata parameters + :param param: parameter to be checked needs to be in format key=value + """ + try: + key, value = param.split('=') + assert (key and value) + return {key: value} + except (ValueError, AssertionError): + msg = "%r is not a valid parameter. Needs to be in format key=value" % param + raise argparse.ArgumentTypeError(msg) + + def build_offline_deliverables(build_version, application_repository_url, application_repository_reference, @@ -224,6 +262,7 @@ def build_offline_deliverables(build_version, aux_package_tar_path = os.path.join(output_dir, 'aux_package.tar') create_package(aux_content, aux_package_tar_path) + add_checksum_info(output_dir) shutil.rmtree(application_dir) @@ -265,8 +304,8 @@ def run_cli(): help='overwrite files in output directory') parser.add_argument('--debug', action='store_true', default=False, help='Turn on debug output') - parser.add_argument('--add-metadata', nargs=2, - help='additional metadata added into package.info, format: key value', default=[]) + parser.add_argument('--add-metadata', nargs="+", type=metadata_validation, + help='additional metadata added into package.info, format: key=value') args = parser.parse_args() if args.debug: diff --git a/build/requirements.txt b/build/requirements.txt index 39544458..441b3fcb 100644 --- a/build/requirements.txt +++ b/build/requirements.txt @@ -1,2 +1,2 @@ docker>=3.7.2 -gitpython==2.1.11 +gitpython==3.1.0 |