path: root/external-schema-repo-generator/generator/generate.sh

#!/bin/sh

# ============LICENSE_START=======================================================
# OOM
# ================================================================================
# Copyright (C) 2020-2021 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#      http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============LICENSE_END=========================================================

. ./utils.sh

# Arguments renaming
arguments_number=$#
repo_url=$1
branches=$2
schemas_location=$3
vendor=$4
configmap_filename=$5
configmap_name=$6
snippet_filename=$7
specs_directory=$8
generation_directory=$9
_5g_api_repo_url=${10}

# Constants
SCHEMA_MAP_FILENAME="schema-map.json"
SCHEMA_MAP_NAME="schema-map"
SUCCESS_CODE=0
TREE=blob
INDENTATION_LEVEL_1=1
INDENTATION_LEVEL_2=2
INDENTATION_LEVEL_3=3
INDENTATION_LEVEL_4=4
INDENTATION_LEVEL_5=5

# Variables
tmp_location=$(mktemp -d)
valid_branches=""

# Create and move to directory for storing generated files
move_to_generation_directory() {
  mkdir "$generation_directory"
  cd ./"$generation_directory"
}

# Indents each line of string by adding indent_size*indent_string spaces on the beginning
# Optional argument is indent_string level, default: 1
# correct usage example:
# echo "Sample Text" | indent_string 2
indent_string() {
  indent_size=2
  indent_string=1
  if [ -n "$1" ]; then indent_string=$1; fi
  pr -to $(expr "$indent_string" \* "$indent_size")
}

# Clones all branches selected in $BRANCH from $repo_url
clone_repo() {
  for actual_branch in $branches; do
    clone_branch "$actual_branch"
  done
}

# Clones single branch $1 from $repo_url.
# $1 - branch name
clone_branch() {
  check_arguments $# $EXPECTED_1_ARG
  configmapSize=800000
  branch=""
  # check if OPEN API branch value provided by user contains 5G API branch after colon for example: Rel-16-SA-91:TSG91-Rel16
  if [ -z "${1##*":"*}" ]; then
    OpenApiBranch=${1%:*}
    _5GApiBranch=${1#"$OpenApiBranch"}
    _5GApiBranch=${_5GApiBranch#:}
    if [ -d $tmp_location/"$OpenApiBranch" ]; then
      echo "Skipping cloning repository."
      echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
      echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
    else
      echo "Cloning repository from branch $OpenApiBranch"
      git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
      OpenApiBranchResult=$?
      # clones 5G API branch moves schemas to OPEN API, splits folder to folder less than 1mb
      echo "Cloning repository from branch $_5GApiBranch"
      git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
      _5GApiBranchResult=$?
      mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location/
      dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
      if [ "$configmapSize" -le "$dirSize"  ]; then
	find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
	# adds each folder as valid branch to generate separate configmap (less then 1 mb) for each folder
	for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* |  grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
      else
	branch=$OpenApiBranch
      fi
      # fix schema file references both remote and local against folder structure for whole schema
      fix_schema_references $tmp_location $OpenApiBranch $schemas_location
      if  [ $OpenApiBranchResult -ne $SUCCESS_CODE ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] ; then
        echo "Problem with cloning branch $OpenApiBranch."
        echo "Branch $OpenApiBranch= will not be added to spec."
      else
        valid_branches="${valid_branches} $branch"
        echo "valid_branches: $valid_branches"
      fi
    fi
  else
    OpenApiBranch=$1
    if [ -d $tmp_location/"$OpenApiBranch" ]; then
      echo "Skipping cloning repository."
      echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
      echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
    else
      echo "Cloning repository from branch $OpenApiBranch"
      git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
      OpenApiBranchResult=$?
      branch=""
      # script checks schema files if they don’t contain remote refs to 5G API even user didn't define it after colon for example: Rel-16-SA-91:TSG91-Rel16
      # if contain then script parse repo tag from reference checkouts it, split to folders, resolve refs and cerates configmap for each folder
      _5GApiBranch=$(check5GApiBranchExistenceInRefs "$tmp_location/$OpenApiBranch/$schemas_location")
      if [ ! -z "$_5GApiBranch" ]; then
	echo "Cloning repository from branch $_5GApiBranch"
        git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
	_5GApiBranchResult=$?
        mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location
        dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
        if [ "$configmapSize" -le "$dirSize"  ]; then
          find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
          for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* |  grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
        else
          branch=$OpenApiBranch
        fi
        fix_schema_references $tmp_location $OpenApiBranch $schemas_location
	if  [ ! -z "$_5GApiBranch" ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] && [ $OpenApiBranchResult -ne $SUCCESS_CODE ]  ; then
          echo "Problem with cloning branch $OpenApiBranch."
          echo "Branch $OpenApiBranch= will not be added to spec."
        else
          valid_branches="${valid_branches} $branch"
          echo "valid_branches: $valid_branches"
        fi
      #if schema files don't contain remote refs to 5G API script goes with old path and doesn't checkout 5G API branch
      else
        if  [ $OpenApiBranchResult -ne $SUCCESS_CODE ] ; then
          echo "Problem with cloning branch $OpenApiBranch."
          echo "Branch $OpenApiBranch= will not be added to spec."
        else
          valid_branches="${valid_branches} $OpenApiBranch"
	  echo "valid_branches: $valid_branches"
        fi
      fi
    fi
  fi
}

# Creates file with name $configmap_filename
# Inserts ConfigMap metadata and sets name as $configmap_name
# $1 - branch name
add_config_map_metadata() {
  branch_configmap_filename="${configmap_filename}-$1.yaml"
  branch_configmap_name=$( echo "$configmap_name-$1" | tr '[:upper:]' '[:lower:]' )
  echo "Creating ConfigMap spec file: $branch_configmap_filename"
  cat << EOF > "$branch_configmap_filename"
apiVersion: v1
kind: ConfigMap
metadata:
  name: $branch_configmap_name
  labels:
    name: $branch_configmap_name
  namespace: onap
data:
EOF
}

# For each selected branch:
#   clones the branch from repository,
#   adds schemas from branch to ConfigMap spec
# $1 - branch name
add_schemas() {
  echo "Adding schemas from branch $1 to spec"
  add_schemas_from_branch "$1"
}

# Adds schemas from single branch to spec
# $1 - branch name
add_schemas_from_branch() {
  branch_configmap_filename="${configmap_filename}-$1.yaml"
  check_arguments $# $EXPECTED_1_ARG
  schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}')
  for schema in $schemas; do
    echo "$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1
    cat "$schema" | indent_string $INDENTATION_LEVEL_2
  done
} >> "$branch_configmap_filename"

move_to_spec_directory() {
  mkdir "$specs_directory"
  cd ./"$specs_directory"
}

# Generates mapping file for collected schemas directly in spec
# $1 - schema map name
generate_mapping_file() {
  schema_map_filename="${configmap_filename}-$1.yaml"
  echo "Generating mapping file in spec"
  echo "$SCHEMA_MAP_FILENAME"": |-" | indent_string $INDENTATION_LEVEL_1 >> "$schema_map_filename"
  echo "[" | indent_string $INDENTATION_LEVEL_2 >> "$schema_map_filename"

  for actual_branch in $valid_branches; do
    echo "Adding mappings from branch: $actual_branch"
    add_mappings_from_branch $actual_branch $schema_map_filename
  done

  truncate -s-2 "$schema_map_filename"
  echo "" >> "$schema_map_filename"
  echo "]" | indent_string $INDENTATION_LEVEL_2 >> "$schema_map_filename"
}

# Adds mappings from single branch directly to spec
# $1 - branch name
# $2 - schema map file name
add_mappings_from_branch() {
  check_arguments $# $EXPECTED_2_ARGS
  schema_map_filename="$2"
  schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}' )

  for schema in $schemas; do
    repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
    schema_repo_path=$(echo "$schema" | cut -d/ -f4-)
    schema_repo_path_fixed=$(echo "$schema_repo_path" | sed -r "s%"-subdir-[0-9]"%%" )
    public_url_schemas_location=${repo_url%.*}
    public_url=$public_url_schemas_location/$TREE/$schema_repo_path_fixed
    local_url=$vendor/$repo_endpoint/$TREE/$schema_repo_path

    echo "{" | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename"
    echo "\"publicURL\": \"$public_url\"," | indent_string $INDENTATION_LEVEL_4 >> "$schema_map_filename"
    echo "\"localURL\": \"$local_url\"" | indent_string $INDENTATION_LEVEL_4 >> "$schema_map_filename"
    echo "}," | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename"
  done
}

# Create snippet file to describe how to connect mount ConfigMaps in VES
create_snippet() {
  echo "Generating snippets in file: $snippet_filename"
  generate_entries
  base_mounts_path="/opt/app/VESCollector/etc/externalRepo"
  base_mounts_name="external-repo-$vendor-schemas"
  mounts_paths="
        - mountPath: $base_mounts_path
          name: $base_mounts_name"

  config_maps="
      - configMap:
          defaultMode: 420
          name: $configmap_name-$SCHEMA_MAP_NAME
        name: $base_mounts_name"

  for actual_branch in $valid_branches; do

    actual_branch_name=$( echo "$actual_branch" | tr '[:upper:]' '[:lower:]' )

    repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
    local_url=$vendor/$repo_endpoint/$TREE/$actual_branch/$schemas_location
    mounts_paths="$mounts_paths
        - mountPath: $base_mounts_path/$local_url
          name: $base_mounts_name-$actual_branch_name"

    config_maps="$config_maps
      - configMap:
          defaultMode: 420
          name: $configmap_name-$actual_branch_name
        name: $base_mounts_name-$actual_branch_name"
  done


  cat << EOF > "$snippet_filename"
Snippets for mounting ConfigMap in DCAE VESCollector Deployment
=========================================================================

## Description
These snippets will override existing in VESCollector schemas and mapping file.

No extra configuration in VESCollector is needed with these snippets.

## Snippets
#### spec.template.spec.containers[0].volumeMounts
\`\`\`$mounts_paths
\`\`\`

#### spec.template.spec.volumes
\`\`\`$config_maps
\`\`\`
EOF
}

generate_entries() {
  for actual_branch in $valid_branches; do
    schemas=$(ls -g $tmp_location/$actual_branch/$schemas_location/*.yaml | awk '{print $NF}')
    for schema in $schemas; do
      repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
      schema_repo_path=$(echo "$schema" | cut -d/ -f4-)

      key="$actual_branch-$(basename "$schema")"
      path=$vendor/$repo_endpoint/$TREE/$schema_repo_path
      schemas_entries="$schemas_entries- key: $key\n  path: $path\n"
    done
  done
  schemas_entries=$(echo "$schemas_entries" | indent_string $INDENTATION_LEVEL_5)
}

# Generate specs for branch
# $1 - branch name
generate_specs_for_branch() {
  check_arguments $# $EXPECTED_1_ARG
  add_config_map_metadata $1
  add_schemas $1
}

# Generate specs for schema map
generate_specs_for_schema_map() {
  add_config_map_metadata "$SCHEMA_MAP_NAME"
  generate_mapping_file "$SCHEMA_MAP_NAME"
}

# Generate specs for all releases and for schema map
generate_specs() {
  move_to_spec_directory
  for actual_branch in $valid_branches; do
    generate_specs_for_branch $actual_branch
  done
  generate_specs_for_schema_map
  cd ..
}


# todo add check of global env whether script should be ran
main() {
  check_arguments $arguments_number $EXPECTED_10_ARGS
  clone_repo
  move_to_generation_directory
  generate_specs
  create_snippet
  move_to_starting_directory
}

main