diff options
5 files changed, 171 insertions, 19 deletions
diff --git a/external-schema-repo-generator/generator/Makefile b/external-schema-repo-generator/generator/Makefile index 93f3226..2966572 100644 --- a/external-schema-repo-generator/generator/Makefile +++ b/external-schema-repo-generator/generator/Makefile @@ -3,7 +3,7 @@ include environment.config all: generate install generate: clean - ./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(SNIPPET_FILENAME) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY) + ./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(SNIPPET_FILENAME) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY) $(5G_API_REPOSITORY_URL_HTTPS) install: ./install.sh $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY) diff --git a/external-schema-repo-generator/generator/bin_packing.awk b/external-schema-repo-generator/generator/bin_packing.awk new file mode 100644 index 0000000..5e82ceb --- /dev/null +++ b/external-schema-repo-generator/generator/bin_packing.awk @@ -0,0 +1,48 @@ +# ============LICENSE_START======================================================= +# OOM +# ================================================================================ +# Copyright (C) 2020-2021 Nokia. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +function first_fit(v, file) { + # find first bin that can accomodate the volume + for (i=1; i<=n; ++i) { + if (b[i] > v) { + b[i] -= v + bc[i]++ + cmd="mkdir -p "tmpLocation"/"schema"-subdir-"i"/OpenAPI" + system(cmd) + cmd="mv "file" " tmpLocation"/"schema "-subdir-" i "/OpenAPI" + system(cmd) + return + } + } + # no bin found, create new bin + if (i > n) { + b[++n] = c - v + bc[n]++ + cmd="mkdir -p "tmpLocation"/"schema"-subdir-"n"/OpenAPI" + system(cmd) + cmd="mv "file" " tmpLocation"/"schema "-subdir-"n"/OpenAPI" + system(cmd) + } + return +} +BEGIN{ if( (c+0) == 0) exit } +{ first_fit($1,$2) } +END { print "REPORT:" + print "Created",n,"directories" + for(i=1;i<=n;++i) { + print "- "tmpLocation"/"schema"-subdir-"i,":", c-b[i],"bytes",bc[i],"files" + } +} diff --git a/external-schema-repo-generator/generator/environment.config b/external-schema-repo-generator/generator/environment.config index 95e94ba..86f70ad 100644 --- a/external-schema-repo-generator/generator/environment.config +++ b/external-schema-repo-generator/generator/environment.config @@ -9,8 +9,18 @@ EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX=external-repo-configmap SNIPPET_FILENAME=ves-snippet.md # Source repository configuration +5G_API_REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/all/5G_APIs.git REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/sa5/MnS.git -REPOSITORY_BRANCH="SA88-Rel16 SA89-Rel17" +#REPOSITORY_BRANCH="Rel-16-SA-91" +#REPOSITORY_BRANCH="Rel-16-SA-91:TSG91-Rel16" +#REPOSITORY_BRANCH="SA88-Rel16" +# User can put here 5G_API specific repository tag name related to OPEN API schema branch after colon +# for example: Rel-16-SA-91:TSG91-Rel16 +# Having such information script will clone this specific 5G API branch and merge it with OPEN API schema +# If 5G_API specific repository branch after colon will not be set by user, then script will parse OPEN API if any +# reference to 5G_API specific repository is present and parse 5G_API branch from this reference. +# Next clone parsed 5G API branch and merge it with OPEN API schema +REPOSITORY_BRANCH="SA88-Rel16 Rel-16-SA-91:TSG91-Rel16 Rel-17-SA-91" SCHEMAS_LOCATION=OpenAPI # Vendor description diff --git a/external-schema-repo-generator/generator/generate.sh b/external-schema-repo-generator/generator/generate.sh index 29ddf85..8a75b57 100755 --- a/external-schema-repo-generator/generator/generate.sh +++ b/external-schema-repo-generator/generator/generate.sh @@ -29,12 +29,13 @@ configmap_name=$6 snippet_filename=$7 specs_directory=$8 generation_directory=$9 +_5g_api_repo_url=${10} # Constants SCHEMA_MAP_FILENAME="schema-map.json" SCHEMA_MAP_NAME="schema-map" SUCCESS_CODE=0 -TREE=tree +TREE=blob INDENTATION_LEVEL_1=1 INDENTATION_LEVEL_2=2 INDENTATION_LEVEL_3=3 @@ -73,19 +74,88 @@ clone_repo() { # $1 - branch name clone_branch() { check_arguments $# $EXPECTED_1_ARG - if [ -d $tmp_location/"$1" ]; then - echo "Skipping cloning repository." - echo "Branch $1 has already been cloned in the directory ./$tmp_location/$1" - echo "To redownload branch remove ./$tmp_location/$1." + configmapSize=800000 + branch="" + # check if OPEN API branch value provided by user contains 5G API branch after colon for example: Rel-16-SA-91:TSG91-Rel16 + if [ -z "${1##*":"*}" ]; then + OpenApiBranch=${1%:*} + _5GApiBranch=${1#"$OpenApiBranch"} + _5GApiBranch=${_5GApiBranch#:} + if [ -d $tmp_location/"$OpenApiBranch" ]; then + echo "Skipping cloning repository." + echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch" + echo "To redownload branch remove ./$tmp_location/$OpenApiBranch." + else + echo "Cloning repository from branch $OpenApiBranch" + git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null + OpenApiBranchResult=$? + # clones 5G API branch moves schemas to OPEN API, splits folder to folder less than 1mb + echo "Cloning repository from branch $_5GApiBranch" + git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null + _5GApiBranchResult=$? + mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location/ + dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1) + if [ "$configmapSize" -le "$dirSize" ]; then + find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk + # adds each folder as valid branch to generate separate configmap (less then 1 mb) for each folder + for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* | grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done + else + branch=$OpenApiBranch + fi + # fix schema file references both remote and local against folder structure for whole schema + fix_schema_references $tmp_location $OpenApiBranch $schemas_location + if [ $OpenApiBranchResult -ne $SUCCESS_CODE ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] ; then + echo "Problem with cloning branch $OpenApiBranch." + echo "Branch $OpenApiBranch= will not be added to spec." + else + valid_branches="${valid_branches} $branch" + echo "valid_branches: $valid_branches" + fi + fi else - echo "Cloning repository from branch $1" - git clone --quiet --single-branch --branch "$1" "$repo_url" "$tmp_location/$1" 2>/dev/null - result=$? - if [ $result -ne $SUCCESS_CODE ] ; then - echo "Problem with cloning branch $1." - echo "Branch $1 will not be added to spec." + OpenApiBranch=$1 + if [ -d $tmp_location/"$OpenApiBranch" ]; then + echo "Skipping cloning repository." + echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch" + echo "To redownload branch remove ./$tmp_location/$OpenApiBranch." else - valid_branches="${valid_branches} $1" + echo "Cloning repository from branch $OpenApiBranch" + git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null + OpenApiBranchResult=$? + branch="" + # script checks schema files if they don’t contain remote refs to 5G API even user didn't define it after colon for example: Rel-16-SA-91:TSG91-Rel16 + # if contain then script parse repo tag from reference checkouts it, split to folders, resolve refs and cerates configmap for each folder + _5GApiBranch=$(check5GApiBranchExistenceInRefs "$tmp_location/$OpenApiBranch/$schemas_location") + if [ ! -z "$_5GApiBranch" ]; then + echo "Cloning repository from branch $_5GApiBranch" + git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null + _5GApiBranchResult=$? + mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location + dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1) + if [ "$configmapSize" -le "$dirSize" ]; then + find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk + for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* | grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done + else + branch=$OpenApiBranch + fi + fix_schema_references $tmp_location $OpenApiBranch $schemas_location + if [ ! -z "$_5GApiBranch" ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] && [ $OpenApiBranchResult -ne $SUCCESS_CODE ] ; then + echo "Problem with cloning branch $OpenApiBranch." + echo "Branch $OpenApiBranch= will not be added to spec." + else + valid_branches="${valid_branches} $branch" + echo "valid_branches: $valid_branches" + fi + #if schema files don't contain remote refs to 5G API script goes with old path and doesn't checkout 5G API branch + else + if [ $OpenApiBranchResult -ne $SUCCESS_CODE ] ; then + echo "Problem with cloning branch $OpenApiBranch." + echo "Branch $OpenApiBranch= will not be added to spec." + else + valid_branches="${valid_branches} $OpenApiBranch" + echo "valid_branches: $valid_branches" + fi + fi fi fi } @@ -125,7 +195,7 @@ add_schemas_from_branch() { check_arguments $# $EXPECTED_1_ARG schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}') for schema in $schemas; do - echo "$1-$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1 + echo "$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1 cat "$schema" | indent_string $INDENTATION_LEVEL_2 done } >> "$branch_configmap_filename" @@ -164,8 +234,9 @@ add_mappings_from_branch() { for schema in $schemas; do repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev) schema_repo_path=$(echo "$schema" | cut -d/ -f4-) + schema_repo_path_fixed=$(echo "$schema_repo_path" | sed -r "s%"-subdir-[0-9]"%%" ) public_url_schemas_location=${repo_url%.*} - public_url=$public_url_schemas_location/$TREE/$schema_repo_path + public_url=$public_url_schemas_location/$TREE/$schema_repo_path_fixed local_url=$vendor/$repo_endpoint/$TREE/$schema_repo_path echo "{" | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename" @@ -182,7 +253,7 @@ create_snippet() { base_mounts_path="/opt/app/VESCollector/etc/externalRepo" base_mounts_name="external-repo-$vendor-schemas" mounts_paths=" - - mountPath: $base_mounts_path/schema-map + - mountPath: $base_mounts_path name: $base_mounts_name" config_maps=" @@ -271,9 +342,9 @@ generate_specs() { # todo add check of global env whether script should be ran main() { - check_arguments $arguments_number $EXPECTED_9_ARGS - move_to_generation_directory + check_arguments $arguments_number $EXPECTED_10_ARGS clone_repo + move_to_generation_directory generate_specs create_snippet move_to_starting_directory diff --git a/external-schema-repo-generator/generator/utils.sh b/external-schema-repo-generator/generator/utils.sh index de32515..fad8616 100644 --- a/external-schema-repo-generator/generator/utils.sh +++ b/external-schema-repo-generator/generator/utils.sh @@ -24,6 +24,7 @@ EXPECTED_1_ARG=1 EXPECTED_2_ARGS=2 EXPECTED_4_ARGS=4 EXPECTED_9_ARGS=9 +EXPECTED_10_ARGS=10 # Checks whether number of arguments is valid @@ -41,3 +42,25 @@ move_to_starting_directory() { echo "Moving back to: $starting_directory" cd $starting_directory } + +fix_schema_references(){ + schemas=$(grep -Eo "http.*[^\/]+(.yaml)" $1/$2-*/$3/* | sort -u) + for schema in $schemas; do + schemaFilePath=$(echo "$schema" | awk -F ":https:" '{print $1}') + remotePath=$(echo "$schema" | awk -F ".yaml:" '{print $2}') + fileName=$(echo "$remotePath" | grep -Eo "([^/]\w*.yaml)") + sed -i "s%${remotePath}%${fileName}%g" $schemaFilePath + done + schemas=$(grep -Eo "(\w*.yaml)" $1/$2-*/$3/* | sort -u) + for schema in $schemas; do + schemaFilePath=$(echo "$schema" | awk -F ":" '{print $1}') + wrongPath=$(echo "$schema" | awk -F ".yaml:" '{print $2}') + fileName="../..$(ls -d $1/$2-*/$3/* | grep $wrongPath | awk -F "$1" '{print $2}')" + sed -i "s%${wrongPath}%${fileName}%g" $schemaFilePath + done +} + + +check5GApiBranchExistenceInRefs () { + echo $(sed -n '/5G_APIs/p' $1/*.yaml | awk -F "raw/|blob/" '{print $2}' | awk -F "/" '{print $1}' | uniq) +} |