summaryrefslogtreecommitdiffstats
path: root/build
diff options
context:
space:
mode:
authorBartek Grzybowski <b.grzybowski@partner.samsung.com>2020-11-20 12:51:17 +0100
committerBartek Grzybowski <b.grzybowski@partner.samsung.com>2020-11-23 08:56:24 +0100
commit7f8b8e7cdc41ef31a11516340aa58b28e23e2175 (patch)
tree208a5ec5e0b9ee13b91453e6cb66b980463d9247 /build
parentaefd99bf33417653d779b44a9e2365cdfd6b53ab (diff)
Enhance repo creation scripts idempotency by supporting caching
While 'yumdownloader' and 'apt-get download' commands do not re-download packages that are already present in working dir, scripts still need to perform package dependencies resolution which is a time consuming activity hence a feature was added to skip any further actions (download or deps resolution) on packages that are already present locally. For cold versus warm cache the gain in run time is: cold warm RPM 90s 10s DEB 7m 90s Change-Id: Ie502934cff6484844557832251e7ce0991789cfb Issue-ID: OOM-2635 Signed-off-by: Bartek Grzybowski <b.grzybowski@partner.samsung.com>
Diffstat (limited to 'build')
-rwxr-xr-xbuild/create_repo.sh21
-rwxr-xr-xbuild/docker-entrypoint.sh113
2 files changed, 113 insertions, 21 deletions
diff --git a/build/create_repo.sh b/build/create_repo.sh
index e70d69b2..f2bde32d 100755
--- a/build/create_repo.sh
+++ b/build/create_repo.sh
@@ -18,6 +18,9 @@ container_repo_volume="/mnt/repo/"
# Additional packages lists files path within container
container_list_volume="/mnt/additional-lists/"
+# Use cache by default
+drop_cache=false
+
# Show script usage
help () {
cat <<EOF
@@ -34,6 +37,7 @@ usage: create_repo.sh [OPTION]...
-t | --target-platform target repository platform type (ubuntu/rhel/centos)
-a | --additional-list additional packages list; can be used multiple times for more additional lists
-n | --container-name-suffix add custom suffix to docker container name
+ -r | --drop-cache remove cached packages (use package cache by default)
-h | --help show this help
If build folder from offline repository is not specified current one will be used by default.
@@ -82,24 +86,33 @@ do
# Directory parameter
# Set path to offline-installer build directory
volume_offline_directory="$2"
+ shift
;;
-d|--destination-repository)
# Repository directory parameter
# Set destination path for created repository
volume_repo_directory="$2"
+ shift
;;
-t|--target-platform)
# Repository type (rpm/deb)
# Set target platform for repository
target_input="$2"
+ shift
;;
-a|--additional-list)
# Array of additional packages lists
additional_lists+=("$2")
+ shift
;;
-n|--container-name-suffix)
# Set custom container name suffix
container_name_suffix="_${2}"
+ shift
+ ;;
+ -r|--drop-cache)
+ # Set flag to clean cache
+ drop_cache=true
;;
*)
# unknown option
@@ -107,7 +120,7 @@ do
exit 1
;;
esac
- shift;shift
+ shift
done
# Check if user specified repository type
@@ -148,6 +161,10 @@ then
param_array+=(--directory ${container_repo_volume})
param_array+=(--list ${container_offline_volume}data_lists/)
param_array+=(--packages-lists-path ${container_list_volume})
+ if ${drop_cache};
+ then
+ param_array+=(--drop-cache)
+ fi
[[ ! ${#additional_lists[@]} -eq 0 ]] && \
for array_list in "${additional_lists[@]}";
do
@@ -155,7 +172,7 @@ then
mounted_lists+=(-v ${array_list}:${container_list_volume}${array_list##*/})
done
- docker run --name $container_name \
+ docker run --name $container_name \
-v ${volume_offline_directory}:${container_offline_volume} \
-v ${volume_repo_directory}:${container_repo_volume} \
"${mounted_lists[@]}" \
diff --git a/build/docker-entrypoint.sh b/build/docker-entrypoint.sh
index 7d5f5b2d..81dcd84c 100755
--- a/build/docker-entrypoint.sh
+++ b/build/docker-entrypoint.sh
@@ -26,6 +26,9 @@ PCKG_LIST_DIR=""
# Path to additional packages lists
ADD_LIST_DIR=""
+# Use cache by default
+drop_cache=false
+
# Show help
help () {
cat <<EOF
@@ -37,6 +40,7 @@ usage: create-repo.sh [OPTION]...
-l | --list input rpm/deb list directory
-a | --additional-list additional packages list; can be used multiple times
-p | --packages-lists-path other additional packages lists
+ -r | --drop-cache remove cached packages (use package cache by default)
-h | --help show this help
Both paths have to be set with shared volume between
@@ -63,20 +67,28 @@ do
# Directory parameter
# Set target reposity path
OFFLINE_REPO_DIR="$2"
+ shift
;;
-l|--list)
# List parameter
# Set path containing onap_rpm.list or onap_deb.list file
PCKG_LIST_DIR="$2"
+ shift
;;
-p|--packages-lists-path)
# Path parameter
# Set path for additional packages lists
ADD_LIST_DIR="$2"
+ shift
;;
-a|--additional-list)
# Array of additional packages lists
ADDITIONAL_LISTS+=("$2")
+ shift
+ ;;
+ -r|--drop-cache)
+ # Set flag to clean cache
+ drop_cache=true
;;
*)
# unknown option
@@ -84,7 +96,7 @@ do
exit
;;
esac
- shift;shift
+ shift
done
# Testing if directory parameter was used
@@ -98,18 +110,21 @@ fi
# If not variable is set to default value /tmp/offline/data-list
if test -z "$PCKG_LIST_DIR"
then
- PCKG_LIST_DIR="/tmp/offline/data_list/"
+ PCKG_LIST_DIR="/tmp/offline/data_list"
fi
# Testing if additional packages list parameter was used
# If not variable is set to default value /tmp/additional-lists
if test -z "$PCKG_LIST_DIR"
then
- PCKG_LIST_DIR="/tmp/additional-lists/"
+ PCKG_LIST_DIR="/tmp/additional-lists"
fi
-# Clean target repo dir for idempotency
-rm -rf ${OFFLINE_REPO_DIR}/*
+# Clean target repo dir if --drop-cache set
+if ${drop_cache};
+then
+ rm -rf ${OFFLINE_REPO_DIR}/*
+fi
case "$distro_type" in
ubuntu)
@@ -133,22 +148,54 @@ case "$distro_type" in
# https://bugs.launchpad.net/ubuntu/+source/aptitude/+bug/1543280
chown _apt $OFFLINE_REPO_DIR
- # Download all packages from onap_deb.list via apt-get to repository folder
- for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}');do apt-get download $i -y; done
- for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}');
- do
- for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/);
- do apt-get download $depends -y;
- done;
- done
+ # Create tmp file for package list
+ list_file=$(mktemp)
+
+ # Enumerate packages that are already downloaded
+ for package in $(cat ${PCKG_LIST_DIR}/onap_deb.list);
+ do
+ # If package name contains explicit version info cut the version string off for further processing
+ p=$(echo $package |sed -r 's/=.*//')
+ # Add package to download list only if it's not already there
+ if [ $(ls ${p}_*.deb 2>/dev/null | wc -l) -eq 0 ];
+ then
+ echo ${package} >> ${list_file}
+ fi
+ done
+
+ # Download all packages via apt-get to repository folder
+ for i in $(cat ${list_file});do echo apt-get download $i -y; done
+ for i in $(cat ${list_file});
+ do
+ for depends in $(apt-cache depends $i | grep -E 'Depends' | grep -v 'Depends:.*>$' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/);
+ do
+echo apt-get download $depends -y;
+ done;
+ done
# Download all packages with dependencies from all additional packages lists via apt-get to repository folder
if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then
for list in ${ADDITIONAL_LISTS[@]}
do
- for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do apt-get download $i -y; done
- for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');
- do
+
+ # Create tmp file for package list
+ list_file=$(mktemp)
+
+ # Enumerate packages that are already downloaded
+ for package in $(cat ${ADD_LIST_DIR}/${list});
+ do
+ # If package name contains explicit version info cut the version string off for further processing
+ p=$(echo $package |sed -r 's/=.*//')
+ # Add package to download list only if it's not already there
+ if [ $(ls ${p}_*.deb 2>/dev/null | wc -l) -eq 0 ];
+ then
+ echo ${package} >> ${list_file}
+ fi
+ done
+
+ for i in $(cat ${list_file});do apt-get download $i -y; done
+ for i in $(cat ${list_file});
+ do
for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/);
do apt-get download $depends -y;
done;
@@ -169,14 +216,42 @@ case "$distro_type" in
# Add official docker repository
yum-config-manager --add-repo=https://download.docker.com/linux/centos/7/x86_64/stable/
+ # Create tmp file for package list
+ list_file=$(mktemp)
+
+ # Enumerate packages that are already downloaded
+ for package in $(cat ${PCKG_LIST_DIR}/onap_rpm.list);
+ do
+ # Add package to download list only if it's not already there
+ if [ ! -f ${OFFLINE_REPO_DIR}/${package}.rpm ];
+ then
+ echo ${package} >> ${list_file}
+ fi
+ done
+
# Download all packages from onap_rpm.list via yumdownloader to repository folder
- for i in $(cat ${PCKG_LIST_DIR}onap_rpm.list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done
+ for i in $(cat ${list_file});do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done
- # Download all packages from all additional packages lists via apt-get to repository folder
+ # Download all packages from all additional packages lists via yumdownloader to repository folder
if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then
for list in ${ADDITIONAL_LISTS[@]}
do
- for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done
+ # Create tmp file for additional package list
+ list_file=$(mktemp)
+ # Enumerate packages that are already downloaded
+ for package in $(cat ${ADD_LIST_DIR}/${list});
+ do
+ # Add package to download list only if it's not already there
+ if [ ! -f ${OFFLINE_REPO_DIR}/${package}.rpm ];
+ then
+ echo ${package} >> ${list_file}
+ fi
+ done
+
+ for i in $(cat ${list_file});
+ do
+ yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y
+ done
done
fi