summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--external-schema-repo-generator/generator/Makefile2
-rw-r--r--external-schema-repo-generator/generator/bin_packing.awk48
-rw-r--r--external-schema-repo-generator/generator/environment.config12
-rwxr-xr-xexternal-schema-repo-generator/generator/generate.sh105
-rw-r--r--external-schema-repo-generator/generator/utils.sh23
5 files changed, 171 insertions, 19 deletions
diff --git a/external-schema-repo-generator/generator/Makefile b/external-schema-repo-generator/generator/Makefile
index 93f3226..2966572 100644
--- a/external-schema-repo-generator/generator/Makefile
+++ b/external-schema-repo-generator/generator/Makefile
@@ -3,7 +3,7 @@ include environment.config
all: generate install
generate: clean
- ./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(SNIPPET_FILENAME) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY)
+ ./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(SNIPPET_FILENAME) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY) $(5G_API_REPOSITORY_URL_HTTPS)
install:
./install.sh $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY)
diff --git a/external-schema-repo-generator/generator/bin_packing.awk b/external-schema-repo-generator/generator/bin_packing.awk
new file mode 100644
index 0000000..5e82ceb
--- /dev/null
+++ b/external-schema-repo-generator/generator/bin_packing.awk
@@ -0,0 +1,48 @@
+# ============LICENSE_START=======================================================
+# OOM
+# ================================================================================
+# Copyright (C) 2020-2021 Nokia. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+function first_fit(v, file) {
+ # find first bin that can accomodate the volume
+ for (i=1; i<=n; ++i) {
+ if (b[i] > v) {
+ b[i] -= v
+ bc[i]++
+ cmd="mkdir -p "tmpLocation"/"schema"-subdir-"i"/OpenAPI"
+ system(cmd)
+ cmd="mv "file" " tmpLocation"/"schema "-subdir-" i "/OpenAPI"
+ system(cmd)
+ return
+ }
+ }
+ # no bin found, create new bin
+ if (i > n) {
+ b[++n] = c - v
+ bc[n]++
+ cmd="mkdir -p "tmpLocation"/"schema"-subdir-"n"/OpenAPI"
+ system(cmd)
+ cmd="mv "file" " tmpLocation"/"schema "-subdir-"n"/OpenAPI"
+ system(cmd)
+ }
+ return
+}
+BEGIN{ if( (c+0) == 0) exit }
+{ first_fit($1,$2) }
+END { print "REPORT:"
+ print "Created",n,"directories"
+ for(i=1;i<=n;++i) {
+ print "- "tmpLocation"/"schema"-subdir-"i,":", c-b[i],"bytes",bc[i],"files"
+ }
+}
diff --git a/external-schema-repo-generator/generator/environment.config b/external-schema-repo-generator/generator/environment.config
index 95e94ba..86f70ad 100644
--- a/external-schema-repo-generator/generator/environment.config
+++ b/external-schema-repo-generator/generator/environment.config
@@ -9,8 +9,18 @@ EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX=external-repo-configmap
SNIPPET_FILENAME=ves-snippet.md
# Source repository configuration
+5G_API_REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/all/5G_APIs.git
REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/sa5/MnS.git
-REPOSITORY_BRANCH="SA88-Rel16 SA89-Rel17"
+#REPOSITORY_BRANCH="Rel-16-SA-91"
+#REPOSITORY_BRANCH="Rel-16-SA-91:TSG91-Rel16"
+#REPOSITORY_BRANCH="SA88-Rel16"
+# User can put here 5G_API specific repository tag name related to OPEN API schema branch after colon
+# for example: Rel-16-SA-91:TSG91-Rel16
+# Having such information script will clone this specific 5G API branch and merge it with OPEN API schema
+# If 5G_API specific repository branch after colon will not be set by user, then script will parse OPEN API if any
+# reference to 5G_API specific repository is present and parse 5G_API branch from this reference.
+# Next clone parsed 5G API branch and merge it with OPEN API schema
+REPOSITORY_BRANCH="SA88-Rel16 Rel-16-SA-91:TSG91-Rel16 Rel-17-SA-91"
SCHEMAS_LOCATION=OpenAPI
# Vendor description
diff --git a/external-schema-repo-generator/generator/generate.sh b/external-schema-repo-generator/generator/generate.sh
index 29ddf85..8a75b57 100755
--- a/external-schema-repo-generator/generator/generate.sh
+++ b/external-schema-repo-generator/generator/generate.sh
@@ -29,12 +29,13 @@ configmap_name=$6
snippet_filename=$7
specs_directory=$8
generation_directory=$9
+_5g_api_repo_url=${10}
# Constants
SCHEMA_MAP_FILENAME="schema-map.json"
SCHEMA_MAP_NAME="schema-map"
SUCCESS_CODE=0
-TREE=tree
+TREE=blob
INDENTATION_LEVEL_1=1
INDENTATION_LEVEL_2=2
INDENTATION_LEVEL_3=3
@@ -73,19 +74,88 @@ clone_repo() {
# $1 - branch name
clone_branch() {
check_arguments $# $EXPECTED_1_ARG
- if [ -d $tmp_location/"$1" ]; then
- echo "Skipping cloning repository."
- echo "Branch $1 has already been cloned in the directory ./$tmp_location/$1"
- echo "To redownload branch remove ./$tmp_location/$1."
+ configmapSize=800000
+ branch=""
+ # check if OPEN API branch value provided by user contains 5G API branch after colon for example: Rel-16-SA-91:TSG91-Rel16
+ if [ -z "${1##*":"*}" ]; then
+ OpenApiBranch=${1%:*}
+ _5GApiBranch=${1#"$OpenApiBranch"}
+ _5GApiBranch=${_5GApiBranch#:}
+ if [ -d $tmp_location/"$OpenApiBranch" ]; then
+ echo "Skipping cloning repository."
+ echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
+ echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
+ else
+ echo "Cloning repository from branch $OpenApiBranch"
+ git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
+ OpenApiBranchResult=$?
+ # clones 5G API branch moves schemas to OPEN API, splits folder to folder less than 1mb
+ echo "Cloning repository from branch $_5GApiBranch"
+ git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
+ _5GApiBranchResult=$?
+ mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location/
+ dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
+ if [ "$configmapSize" -le "$dirSize" ]; then
+ find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
+ # adds each folder as valid branch to generate separate configmap (less then 1 mb) for each folder
+ for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* | grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
+ else
+ branch=$OpenApiBranch
+ fi
+ # fix schema file references both remote and local against folder structure for whole schema
+ fix_schema_references $tmp_location $OpenApiBranch $schemas_location
+ if [ $OpenApiBranchResult -ne $SUCCESS_CODE ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] ; then
+ echo "Problem with cloning branch $OpenApiBranch."
+ echo "Branch $OpenApiBranch= will not be added to spec."
+ else
+ valid_branches="${valid_branches} $branch"
+ echo "valid_branches: $valid_branches"
+ fi
+ fi
else
- echo "Cloning repository from branch $1"
- git clone --quiet --single-branch --branch "$1" "$repo_url" "$tmp_location/$1" 2>/dev/null
- result=$?
- if [ $result -ne $SUCCESS_CODE ] ; then
- echo "Problem with cloning branch $1."
- echo "Branch $1 will not be added to spec."
+ OpenApiBranch=$1
+ if [ -d $tmp_location/"$OpenApiBranch" ]; then
+ echo "Skipping cloning repository."
+ echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
+ echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
else
- valid_branches="${valid_branches} $1"
+ echo "Cloning repository from branch $OpenApiBranch"
+ git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
+ OpenApiBranchResult=$?
+ branch=""
+ # script checks schema files if they don’t contain remote refs to 5G API even user didn't define it after colon for example: Rel-16-SA-91:TSG91-Rel16
+ # if contain then script parse repo tag from reference checkouts it, split to folders, resolve refs and cerates configmap for each folder
+ _5GApiBranch=$(check5GApiBranchExistenceInRefs "$tmp_location/$OpenApiBranch/$schemas_location")
+ if [ ! -z "$_5GApiBranch" ]; then
+ echo "Cloning repository from branch $_5GApiBranch"
+ git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
+ _5GApiBranchResult=$?
+ mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location
+ dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
+ if [ "$configmapSize" -le "$dirSize" ]; then
+ find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
+ for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* | grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
+ else
+ branch=$OpenApiBranch
+ fi
+ fix_schema_references $tmp_location $OpenApiBranch $schemas_location
+ if [ ! -z "$_5GApiBranch" ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] && [ $OpenApiBranchResult -ne $SUCCESS_CODE ] ; then
+ echo "Problem with cloning branch $OpenApiBranch."
+ echo "Branch $OpenApiBranch= will not be added to spec."
+ else
+ valid_branches="${valid_branches} $branch"
+ echo "valid_branches: $valid_branches"
+ fi
+ #if schema files don't contain remote refs to 5G API script goes with old path and doesn't checkout 5G API branch
+ else
+ if [ $OpenApiBranchResult -ne $SUCCESS_CODE ] ; then
+ echo "Problem with cloning branch $OpenApiBranch."
+ echo "Branch $OpenApiBranch= will not be added to spec."
+ else
+ valid_branches="${valid_branches} $OpenApiBranch"
+ echo "valid_branches: $valid_branches"
+ fi
+ fi
fi
fi
}
@@ -125,7 +195,7 @@ add_schemas_from_branch() {
check_arguments $# $EXPECTED_1_ARG
schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}')
for schema in $schemas; do
- echo "$1-$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1
+ echo "$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1
cat "$schema" | indent_string $INDENTATION_LEVEL_2
done
} >> "$branch_configmap_filename"
@@ -164,8 +234,9 @@ add_mappings_from_branch() {
for schema in $schemas; do
repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
schema_repo_path=$(echo "$schema" | cut -d/ -f4-)
+ schema_repo_path_fixed=$(echo "$schema_repo_path" | sed -r "s%"-subdir-[0-9]"%%" )
public_url_schemas_location=${repo_url%.*}
- public_url=$public_url_schemas_location/$TREE/$schema_repo_path
+ public_url=$public_url_schemas_location/$TREE/$schema_repo_path_fixed
local_url=$vendor/$repo_endpoint/$TREE/$schema_repo_path
echo "{" | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename"
@@ -182,7 +253,7 @@ create_snippet() {
base_mounts_path="/opt/app/VESCollector/etc/externalRepo"
base_mounts_name="external-repo-$vendor-schemas"
mounts_paths="
- - mountPath: $base_mounts_path/schema-map
+ - mountPath: $base_mounts_path
name: $base_mounts_name"
config_maps="
@@ -271,9 +342,9 @@ generate_specs() {
# todo add check of global env whether script should be ran
main() {
- check_arguments $arguments_number $EXPECTED_9_ARGS
- move_to_generation_directory
+ check_arguments $arguments_number $EXPECTED_10_ARGS
clone_repo
+ move_to_generation_directory
generate_specs
create_snippet
move_to_starting_directory
diff --git a/external-schema-repo-generator/generator/utils.sh b/external-schema-repo-generator/generator/utils.sh
index de32515..fad8616 100644
--- a/external-schema-repo-generator/generator/utils.sh
+++ b/external-schema-repo-generator/generator/utils.sh
@@ -24,6 +24,7 @@ EXPECTED_1_ARG=1
EXPECTED_2_ARGS=2
EXPECTED_4_ARGS=4
EXPECTED_9_ARGS=9
+EXPECTED_10_ARGS=10
# Checks whether number of arguments is valid
@@ -41,3 +42,25 @@ move_to_starting_directory() {
echo "Moving back to: $starting_directory"
cd $starting_directory
}
+
+fix_schema_references(){
+ schemas=$(grep -Eo "http.*[^\/]+(.yaml)" $1/$2-*/$3/* | sort -u)
+ for schema in $schemas; do
+ schemaFilePath=$(echo "$schema" | awk -F ":https:" '{print $1}')
+ remotePath=$(echo "$schema" | awk -F ".yaml:" '{print $2}')
+ fileName=$(echo "$remotePath" | grep -Eo "([^/]\w*.yaml)")
+ sed -i "s%${remotePath}%${fileName}%g" $schemaFilePath
+ done
+ schemas=$(grep -Eo "(\w*.yaml)" $1/$2-*/$3/* | sort -u)
+ for schema in $schemas; do
+ schemaFilePath=$(echo "$schema" | awk -F ":" '{print $1}')
+ wrongPath=$(echo "$schema" | awk -F ".yaml:" '{print $2}')
+ fileName="../..$(ls -d $1/$2-*/$3/* | grep $wrongPath | awk -F "$1" '{print $2}')"
+ sed -i "s%${wrongPath}%${fileName}%g" $schemaFilePath
+ done
+}
+
+
+check5GApiBranchExistenceInRefs () {
+ echo $(sed -n '/5G_APIs/p' $1/*.yaml | awk -F "raw/|blob/" '{print $2}' | awk -F "/" '{print $1}' | uniq)
+}