[UTILS] Fix and resolve remote references in standard defined domain

Add functionality to scripts that:
- Looks in cloned schema file for remote refs.
- Downloads remote refs to schema directory
- Splits schema directory directories less than 1mb
- Replace remote and local refs locations with local locations
and current directory structure
- Creates for each directory separate configmap less than 1mb

Signed-off-by: Krzysztof Kuzmicki <krzysztof.kuzmicki@nokia.com>
Issue-ID: DCAEGEN2-2893 DCAEGEN2-2894 DCAEGEN2-2895
Change-Id: Ia55e2fb29eb2fa6575821f12af5e22f0f1d8ff5a

5 files changed, 171 insertions, 19 deletions

diff --git a/external-schema-repo-generator/generator/Makefile b/external-schema-repo-generator/generator/Makefile
index 93f3226..2966572 100644
--- a/external-schema-repo-generator/generator/Makefile
+++ b/external-schema-repo-generator/generator/Makefile
@@ -3,7 +3,7 @@ include environment.config
 all: generate install
 
 generate: clean
-	./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(SNIPPET_FILENAME) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY)
+	./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(SNIPPET_FILENAME) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY)  $(5G_API_REPOSITORY_URL_HTTPS)
 
 install:
 	./install.sh $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY)
diff --git a/external-schema-repo-generator/generator/bin_packing.awk b/external-schema-repo-generator/generator/bin_packing.awk
new file mode 100644
index 0000000..5e82ceb
--- /dev/null
+++ b/external-schema-repo-generator/generator/bin_packing.awk
@@ -0,0 +1,48 @@
+# ============LICENSE_START=======================================================
+# OOM
+# ================================================================================
+# Copyright (C) 2020-2021 Nokia. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+function first_fit(v, file) {
+    # find first bin that can accomodate the volume
+    for (i=1; i<=n; ++i) {
+        if (b[i] > v) {
+            b[i] -= v
+            bc[i]++
+            cmd="mkdir -p "tmpLocation"/"schema"-subdir-"i"/OpenAPI"
+            system(cmd)
+            cmd="mv "file" " tmpLocation"/"schema  "-subdir-" i "/OpenAPI"
+            system(cmd)
+            return
+        }
+    }
+    # no bin found, create new bin
+    if (i > n) {
+        b[++n] = c - v
+        bc[n]++
+        cmd="mkdir -p "tmpLocation"/"schema"-subdir-"n"/OpenAPI"
+        system(cmd)
+        cmd="mv "file" " tmpLocation"/"schema  "-subdir-"n"/OpenAPI"
+        system(cmd)
+    }
+    return
+}
+BEGIN{ if( (c+0) == 0) exit }
+{ first_fit($1,$2) }
+END { print "REPORT:"
+    print "Created",n,"directories"
+    for(i=1;i<=n;++i) {
+	    print "- "tmpLocation"/"schema"-subdir-"i,":", c-b[i],"bytes",bc[i],"files"
+    }
+}
diff --git a/external-schema-repo-generator/generator/environment.config b/external-schema-repo-generator/generator/environment.config
index 95e94ba..86f70ad 100644
--- a/external-schema-repo-generator/generator/environment.config
+++ b/external-schema-repo-generator/generator/environment.config
@@ -9,8 +9,18 @@ EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX=external-repo-configmap
 SNIPPET_FILENAME=ves-snippet.md
 
 # Source repository configuration
+5G_API_REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/all/5G_APIs.git
 REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/sa5/MnS.git
-REPOSITORY_BRANCH="SA88-Rel16 SA89-Rel17"
+#REPOSITORY_BRANCH="Rel-16-SA-91"
+#REPOSITORY_BRANCH="Rel-16-SA-91:TSG91-Rel16"
+#REPOSITORY_BRANCH="SA88-Rel16"
+# User can put here 5G_API specific repository tag name related to OPEN API schema branch after colon
+# for example: Rel-16-SA-91:TSG91-Rel16
+# Having such information script will clone this specific 5G API branch and merge it with OPEN API schema
+# If 5G_API specific repository branch after colon will not be set by user, then script will parse OPEN API if any
+# reference to 5G_API specific repository is present and parse 5G_API branch from this reference.
+# Next clone parsed 5G API branch and merge it with OPEN API schema
+REPOSITORY_BRANCH="SA88-Rel16 Rel-16-SA-91:TSG91-Rel16 Rel-17-SA-91"
 SCHEMAS_LOCATION=OpenAPI
 
 # Vendor description
diff --git a/external-schema-repo-generator/generator/generate.sh b/external-schema-repo-generator/generator/generate.sh
index 29ddf85..8a75b57 100755
--- a/external-schema-repo-generator/generator/generate.sh
+++ b/external-schema-repo-generator/generator/generate.sh
@@ -29,12 +29,13 @@ configmap_name=$6
 snippet_filename=$7
 specs_directory=$8
 generation_directory=$9
+_5g_api_repo_url=${10}
 
 # Constants
 SCHEMA_MAP_FILENAME="schema-map.json"
 SCHEMA_MAP_NAME="schema-map"
 SUCCESS_CODE=0
-TREE=tree
+TREE=blob
 INDENTATION_LEVEL_1=1
 INDENTATION_LEVEL_2=2
 INDENTATION_LEVEL_3=3
@@ -73,19 +74,88 @@ clone_repo() {
 # $1 - branch name
 clone_branch() {
   check_arguments $# $EXPECTED_1_ARG
-  if [ -d $tmp_location/"$1" ]; then
-    echo "Skipping cloning repository."
-    echo "Branch $1 has already been cloned in the directory ./$tmp_location/$1"
-    echo "To redownload branch remove ./$tmp_location/$1."
+  configmapSize=800000
+  branch=""
+  # check if OPEN API branch value provided by user contains 5G API branch after colon for example: Rel-16-SA-91:TSG91-Rel16
+  if [ -z "${1##*":"*}" ]; then
+    OpenApiBranch=${1%:*}
+    _5GApiBranch=${1#"$OpenApiBranch"}
+    _5GApiBranch=${_5GApiBranch#:}
+    if [ -d $tmp_location/"$OpenApiBranch" ]; then
+      echo "Skipping cloning repository."
+      echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
+      echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
+    else
+      echo "Cloning repository from branch $OpenApiBranch"
+      git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
+      OpenApiBranchResult=$?
+      # clones 5G API branch moves schemas to OPEN API, splits folder to folder less than 1mb
+      echo "Cloning repository from branch $_5GApiBranch"
+      git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
+      _5GApiBranchResult=$?
+      mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location/
+      dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
+      if [ "$configmapSize" -le "$dirSize"  ]; then
+	find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
+	# adds each folder as valid branch to generate separate configmap (less then 1 mb) for each folder
+	for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* |  grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
+      else
+	branch=$OpenApiBranch
+      fi
+      # fix schema file references both remote and local against folder structure for whole schema
+      fix_schema_references $tmp_location $OpenApiBranch $schemas_location
+      if  [ $OpenApiBranchResult -ne $SUCCESS_CODE ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] ; then
+        echo "Problem with cloning branch $OpenApiBranch."
+        echo "Branch $OpenApiBranch= will not be added to spec."
+      else
+        valid_branches="${valid_branches} $branch"
+        echo "valid_branches: $valid_branches"
+      fi
+    fi
   else
-    echo "Cloning repository from branch $1"
-    git clone --quiet --single-branch --branch "$1" "$repo_url" "$tmp_location/$1" 2>/dev/null
-    result=$?
-    if [ $result -ne $SUCCESS_CODE ] ; then
-      echo "Problem with cloning branch $1."
-      echo "Branch $1 will not be added to spec."
+    OpenApiBranch=$1
+    if [ -d $tmp_location/"$OpenApiBranch" ]; then
+      echo "Skipping cloning repository."
+      echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
+      echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
     else
-      valid_branches="${valid_branches} $1"
+      echo "Cloning repository from branch $OpenApiBranch"
+      git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
+      OpenApiBranchResult=$?
+      branch=""
+      # script checks schema files if they don’t contain remote refs to 5G API even user didn't define it after colon for example: Rel-16-SA-91:TSG91-Rel16
+      # if contain then script parse repo tag from reference checkouts it, split to folders, resolve refs and cerates configmap for each folder
+      _5GApiBranch=$(check5GApiBranchExistenceInRefs "$tmp_location/$OpenApiBranch/$schemas_location")
+      if [ ! -z "$_5GApiBranch" ]; then
+	echo "Cloning repository from branch $_5GApiBranch"
+        git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
+	_5GApiBranchResult=$?
+        mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location
+        dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
+        if [ "$configmapSize" -le "$dirSize"  ]; then
+          find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
+          for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* |  grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
+        else
+          branch=$OpenApiBranch
+        fi
+        fix_schema_references $tmp_location $OpenApiBranch $schemas_location
+	if  [ ! -z "$_5GApiBranch" ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] && [ $OpenApiBranchResult -ne $SUCCESS_CODE ]  ; then
+          echo "Problem with cloning branch $OpenApiBranch."
+          echo "Branch $OpenApiBranch= will not be added to spec."
+        else
+          valid_branches="${valid_branches} $branch"
+          echo "valid_branches: $valid_branches"
+        fi
+      #if schema files don't contain remote refs to 5G API script goes with old path and doesn't checkout 5G API branch
+      else
+        if  [ $OpenApiBranchResult -ne $SUCCESS_CODE ] ; then
+          echo "Problem with cloning branch $OpenApiBranch."
+          echo "Branch $OpenApiBranch= will not be added to spec."
+        else
+          valid_branches="${valid_branches} $OpenApiBranch"
+	  echo "valid_branches: $valid_branches"
+        fi
+      fi
     fi
   fi
 }
@@ -125,7 +195,7 @@ add_schemas_from_branch() {
   check_arguments $# $EXPECTED_1_ARG
   schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}')
   for schema in $schemas; do
-    echo "$1-$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1
+    echo "$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1
     cat "$schema" | indent_string $INDENTATION_LEVEL_2
   done
 } >> "$branch_configmap_filename"
@@ -164,8 +234,9 @@ add_mappings_from_branch() {
   for schema in $schemas; do
     repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
     schema_repo_path=$(echo "$schema" | cut -d/ -f4-)
+    schema_repo_path_fixed=$(echo "$schema_repo_path" | sed -r "s%"-subdir-[0-9]"%%" )
     public_url_schemas_location=${repo_url%.*}
-    public_url=$public_url_schemas_location/$TREE/$schema_repo_path
+    public_url=$public_url_schemas_location/$TREE/$schema_repo_path_fixed
     local_url=$vendor/$repo_endpoint/$TREE/$schema_repo_path
 
     echo "{" | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename"
@@ -182,7 +253,7 @@ create_snippet() {
   base_mounts_path="/opt/app/VESCollector/etc/externalRepo"
   base_mounts_name="external-repo-$vendor-schemas"
   mounts_paths="
-        - mountPath: $base_mounts_path/schema-map
+        - mountPath: $base_mounts_path
           name: $base_mounts_name"
 
   config_maps="
@@ -271,9 +342,9 @@ generate_specs() {
 
 # todo add check of global env whether script should be ran
 main() {
-  check_arguments $arguments_number $EXPECTED_9_ARGS
-  move_to_generation_directory
+  check_arguments $arguments_number $EXPECTED_10_ARGS
   clone_repo
+  move_to_generation_directory
   generate_specs
   create_snippet
   move_to_starting_directory
diff --git a/external-schema-repo-generator/generator/utils.sh b/external-schema-repo-generator/generator/utils.sh
index de32515..fad8616 100644
--- a/external-schema-repo-generator/generator/utils.sh
+++ b/external-schema-repo-generator/generator/utils.sh
@@ -24,6 +24,7 @@ EXPECTED_1_ARG=1
 EXPECTED_2_ARGS=2
 EXPECTED_4_ARGS=4
 EXPECTED_9_ARGS=9
+EXPECTED_10_ARGS=10
 
 
 # Checks whether number of arguments is valid
@@ -41,3 +42,25 @@ move_to_starting_directory() {
   echo "Moving back to: $starting_directory"
   cd $starting_directory
 }
+
+fix_schema_references(){
+  schemas=$(grep -Eo "http.*[^\/]+(.yaml)" $1/$2-*/$3/* | sort -u)
+  for schema in $schemas; do
+    schemaFilePath=$(echo "$schema" | awk  -F ":https:" '{print $1}')
+    remotePath=$(echo "$schema" | awk  -F ".yaml:" '{print $2}')
+    fileName=$(echo "$remotePath" |  grep -Eo "([^/]\w*.yaml)")
+    sed -i "s%${remotePath}%${fileName}%g" $schemaFilePath
+  done
+  schemas=$(grep -Eo "(\w*.yaml)" $1/$2-*/$3/* | sort -u)
+  for schema in $schemas; do
+    schemaFilePath=$(echo "$schema" | awk  -F ":" '{print $1}')
+    wrongPath=$(echo "$schema" | awk  -F ".yaml:" '{print $2}')
+    fileName="../..$(ls -d $1/$2-*/$3/* | grep $wrongPath | awk  -F "$1" '{print $2}')"
+    sed -i "s%${wrongPath}%${fileName}%g" $schemaFilePath
+  done
+}
+
+
+check5GApiBranchExistenceInRefs () {
+  echo $(sed -n '/5G_APIs/p' $1/*.yaml | awk  -F "raw/|blob/" '{print $2}' | awk -F "/"  '{print $1}' | uniq)
+}