From 1898455d2fdd4c4f1f4914119e123babed60ce61 Mon Sep 17 00:00:00 2001 From: Bartosz Gardziejewski Date: Tue, 9 Feb 2021 11:04:39 +0100 Subject: Update script to generate spec per branch Signed-off-by: Bartosz Gardziejewski Issue-ID: DCAEGEN2-2619 Change-Id: I8308af63b726a36f2661566cdd08a1f366052d57 --- external-schema-repo-generator/Makefile | 8 - external-schema-repo-generator/README.md | 100 ++++---- external-schema-repo-generator/environment.config | 15 -- external-schema-repo-generator/generate.sh | 227 ----------------- external-schema-repo-generator/generator/Makefile | 12 + external-schema-repo-generator/generator/clean.sh | 42 +++ .../generator/environment.config | 17 ++ .../generator/generate.sh | 282 +++++++++++++++++++++ .../generator/install.sh | 114 +++++++++ external-schema-repo-generator/generator/utils.sh | 43 ++++ external-schema-repo-generator/install.sh | 58 ----- 11 files changed, 567 insertions(+), 351 deletions(-) delete mode 100644 external-schema-repo-generator/Makefile delete mode 100644 external-schema-repo-generator/environment.config delete mode 100755 external-schema-repo-generator/generate.sh create mode 100644 external-schema-repo-generator/generator/Makefile create mode 100755 external-schema-repo-generator/generator/clean.sh create mode 100644 external-schema-repo-generator/generator/environment.config create mode 100755 external-schema-repo-generator/generator/generate.sh create mode 100755 external-schema-repo-generator/generator/install.sh create mode 100644 external-schema-repo-generator/generator/utils.sh delete mode 100755 external-schema-repo-generator/install.sh diff --git a/external-schema-repo-generator/Makefile b/external-schema-repo-generator/Makefile deleted file mode 100644 index 5bc4605..0000000 --- a/external-schema-repo-generator/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -include environment.config - -generate: - ./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(SPEC_CONFIGMAP_FILENAME) $(K8S_CONFIGMAP_NAME) $(SNIPPET_FILENAME) - -install: - ./install.sh $(SPEC_CONFIGMAP_FILENAME) $(K8S_CONFIGMAP_NAME) - diff --git a/external-schema-repo-generator/README.md b/external-schema-repo-generator/README.md index 1ac06b2..8e959ce 100644 --- a/external-schema-repo-generator/README.md +++ b/external-schema-repo-generator/README.md @@ -2,8 +2,8 @@ StndDefined schemas ConfigMap generator ======================================= ## Description -StndDefined schemas Kubernetes ConfigMap generator is a Makefile with two targets running shell scripts: 'generate' and -'install'. This Makefile may be used by VES Collector users to generate and install ConfigMap containing schemas +StndDefined schemas Kubernetes ConfigMap generator is a Makefile with shell scripts: 'generate', 'install', 'clean' ane 'utils'. +This Makefile may be used by VES Collector users to generate and install ConfigMap containing schemas and mapping file for stndDefined validation in VES pod. Additionally script creates file with snippets containing auto-generated configuration of volumes finally mounted in VES deployment. Process of generation of ConfigMap spec file is configurable via environment.config file. @@ -25,10 +25,12 @@ as schemas and attached to ConfigMap spec. ### Generator tool files integration It is recommended to consider files of this tool as a unity and not split them when moving generator through -environments. Generator tool files that are required are: +environments. Generator tool files that are required are located in `generator` directory: - Makefile - install.sh - generate.sh +- utils.sh +- clean.sh - environment.config ## Instruction @@ -37,8 +39,10 @@ environments. Generator tool files that are required are: Before running any target from Makefile, configuration in *environment.config* must be properly prepared. Description of the configurable properties is below. -- **SPEC_CONFIGMAP_FILENAME** - Filename name of ConfigMap spec that will be generated. -- **K8S_CONFIGMAP_NAME** - Kubernetes name of ConfigMap that will be generated and installed. +- **GENERATION_DIRECTORY** - Directory in which generated files will be located. +- **EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX** - Prefix for file name of ConfigMap spec, that will be generated. +- **EXTERNAL_REPO_CONFIGMAP_DIRECTORY** - Directory inside *GENERATION_DIRECTORY*, in which generated specs will be located. +- **EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX** - Prefix for Kubernetes name of ConfigMap, that will be generated and installed. - **SNIPPET_FILENAME** - Filename of snippet with autogenerated content that should be added to VES deployment. - **REPOSITORY_URL_HTTPS** - URL to remote Git repository which lets cloning repository via HTTPS. @@ -53,13 +57,26 @@ VES. ### Running commands -To run ConfigMap spec generation as well as snippet file used for mounting ConfigMap to specific Deployment use: + +#### All + +To generate and install ConfigMaps run: + +``` +make all +``` + +#### Generation + +To run ConfigMaps spec generation as well as snippet file used for mounting ConfigMap to specific Deployment use: ``` make generate ``` -To run ConfigMap installation in Kubernetes use: +#### Installation + +To run ConfigMaps installation in Kubernetes use: **NOTE**: Remember about running this command on RKE node. @@ -67,16 +84,23 @@ To run ConfigMap installation in Kubernetes use: make install ``` -**NOTE**: It is possible that ConfigMap with selected K8S_CONFIGMAP_NAME already exists in Kubernetes. In such situation -either regenerate spec with new K8S_CONFIGMAP_NAME or remove existing ConfigMap from Kubernetes and install spec again. -To remove ConfigMap from Kubernetes use: +**NOTE**: It is possible that ConfigMap with selected K8S_CONFIGMAP_NAME already exists in Kubernetes. +In such situation existing ConfigMap will be replaced. + +**NOTE**: If ConfigMap is bigger than one megabyte it won't be installed +and warning will be displayed during installation. + +#### Cleaning + +To remove generated files use: + ``` -kubectl -n onap delete configmap +make clean ``` ## ConfigMap validation -After running the script ConfigMap spec file is generated in current working directory. -Spec file can be manually validated via any text editor. The last file included in spec is schema-map.json file with +After running the script ConfigMap spec file is generated in selected (be default `genereted`) directory. +Spec files can be manually validated via any text editor. The last generated file includes in spec is schema-map.json file with mappings of external URLs to prepared local URLs. To check whether it has been created use command: @@ -85,11 +109,11 @@ To check whether it has been created use command: kubectl -n onap get configmap | grep ``` -A ConfigMap with configured name should be visible on the list. +A ConfigMaps with configured name should be visible on the list. ## Mounting ConfigMap into VES Collector -To mount created ConfigMap in VES, its deployment must be edited. It can be done with: +To mount created ConfigMaps in VES, its deployment must be edited. It can be done with: ``` kubectl -n onap edit deployment dep-dcae-ves-collector ``` @@ -110,16 +134,18 @@ when pasting snippets. Use ``:set paste `` to turn paste mode on. To close paste ``` volumeMounts: - - ... - - mountPath: /opt/app/VESCollector/etc/externalRepoCustom - name: custom-3gpp-schemas + - mountPath: /opt/app/VESCollector/etc/externalRepo + name: custom-3gpp-schemas + - mountPath: /opt/app/VESCollector/etc/externalRepo/3gpp/rep/sa5/MnS/tree/branch/OpenAPI + name: branch-custom-3gpp-schemas + - ... ``` - mountPath - Directory context for schemas. **NOTE**: must be the same as configuration of VES Collector property *collector.externalSchema.schemasLocation* in *collector.properties*. This property might be modified via Consul UI, later after changes in deployment. - - name - Name of ConfigMap volume. Must be the same as set in the 2. step in *name* field. + - name - Name of ConfigMap volume. For schema map must be the same as set in the 2. step in *name* field. 2. Add volumes element @@ -131,32 +157,20 @@ when pasting snippets. Use ``:set paste `` to turn paste mode on. To close paste ``` volumes: - - configMap: - defaultMode: 420 - items: - - key: schema-map.json - path: schema-map.json - - key: SA88-Rel16-faultMnS.yaml - path: 3gpp/rep/sa5/data-models/SA88-Rel16/OpenAPI/faultMnS.yaml - - ... - name: stnd-defined-configmap - name: custom-3gpp-schemas - - ... + - configMap: + defaultMode: 420 + name: schema-map-stnd-defined-configmap + name: custom-3gpp-schemas + - configMap: + defaultMode: 420 + name: branch-stnd-defined-configmap + name: branch-custom-3gpp-schemas + - ... ``` Fields description: - - name - Name of ConfigMap volume. Must be the same as set in the 1. step in *name* field. - - configMap.name - name of installed Kubernetes ConfigMap described in K8S_CONFIGMAP_NAME configuration setting - - configMap.items[].key - name of mounted file from installed previously ConfigMap. - **NOTE**: Not every schema from ConfigMap must be listed in *items* (warning message will be logged on VES startup if - so), but mapping file named as described in MAPPING_FILE_NAME setting is required for correct stndDefined validation - in VES. - - configMap.items[].path: Relative path from *mountPath* from step 1 which describes location of schema location in - VES container. - **NOTE 1**: For correct schemas detection in VES Collector *path* of each schema must be the same as its localURL in - mapping file. Mapping file is included as the last file in generated ConfigMap spec. - **NOTE 2**: For correct mapping file detection in VES Collector its *path* must be the same as in property - *collector.externalSchema.mappingFileLocation* in *collector.properties*. This property might be modified via Consul - UI, later after changes in deployment. + - name - Name of ConfigMap volume. Consist of branch name and *name* field. Must be same as name in 1. step + - configMap.name - name of installed Kubernetes ConfigMap consist of schema-map or branch name + and K8S_CONFIGMAP_NAME from configuration setting 3. Save and close an editor, K8S will automatically detect changes, terminate old VES Pod and deploy new one with mounted ConfigMap. Correctness of new VES Pod initialization and mounting ConfigMap can be tracked using diff --git a/external-schema-repo-generator/environment.config b/external-schema-repo-generator/environment.config deleted file mode 100644 index 074a463..0000000 --- a/external-schema-repo-generator/environment.config +++ /dev/null @@ -1,15 +0,0 @@ -# Configuration file for externalSchemaRepoGenerator Makefile -# Remember about escaping special characters, e.g. in REPOSITORY_URL - -# ConfigMap configuration -SPEC_CONFIGMAP_FILENAME=stndDefined-schemas-configmap-spec.yaml -K8S_CONFIGMAP_NAME=stnd-defined-configmap -SNIPPET_FILENAME=ves-snippet.md - -# Source repository configuration -REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/sa5/MnS.git -REPOSITORY_BRANCH="SA88-Rel16 SA89-Rel17" -SCHEMAS_LOCATION=OpenAPI - -# Vendor description -VENDOR_NAME=3gpp \ No newline at end of file diff --git a/external-schema-repo-generator/generate.sh b/external-schema-repo-generator/generate.sh deleted file mode 100755 index 731bcbd..0000000 --- a/external-schema-repo-generator/generate.sh +++ /dev/null @@ -1,227 +0,0 @@ -#!/bin/sh - -# ============LICENSE_START======================================================= -# OOM -# ================================================================================ -# Copyright (C) 2020 Nokia. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= - - -# Arguments renaming -arguments_number=$# -repo_url=$1 -branches=$2 -schemas_location=$3 -vendor=$4 -configmap_filename=$5 -configmap_name=$6 -snippet_filename=$7 - -# Constants -SCHEMA_MAP_FILENAME="schema-map.json" -SUCCESS_CODE=0 -TREE=tree -EXPECTED_1_ARG=1 -EXPECTED_7_ARGS=7 -INDENTATION_LEVEL_1=1 -INDENTATION_LEVEL_2=2 -INDENTATION_LEVEL_3=3 -INDENTATION_LEVEL_4=4 -INDENTATION_LEVEL_5=5 - -# Variables -tmp_location=$(mktemp -d) -valid_branches="" - -# Indents each line of string by adding indent_size*indent_string spaces on the beginning -# Optional argument is indent_string level, default: 1 -# correct usage example: -# echo "Sample Text" | indent_string 2 -indent_string() { - indent_size=2 - indent_string=1 - if [ -n "$1" ]; then indent_string=$1; fi - pr -to $(expr "$indent_string" \* "$indent_size") -} - -# Checks whether number of arguments is valid -# $1 is actual number of arguments -# $2 is expected number of arguments -check_arguments() { - if [ "$1" -ne "$2" ]; then - echo "Incorrect number of arguments" - exit 1 - fi -} - -# Clones all branches selected in $BRANCH from $repo_url -clone_repo() { - for actual_branch in $branches; do - clone_branch "$actual_branch" - done -} - -# Clones single branch $1 from $repo_url. -# $1 - branch name -clone_branch() { - check_arguments $# $EXPECTED_1_ARG - if [ -d $tmp_location/"$1" ]; then - echo "Skipping cloning repository." - echo "Branch $1 has already been cloned in the directory ./$tmp_location/$1" - echo "To redownload branch remove ./$tmp_location/$1." - else - echo "Cloning repository from branch $1" - git clone --quiet --single-branch --branch "$1" "$repo_url" "$tmp_location/$1" 2>/dev/null - result=$? - if [ $result -ne $SUCCESS_CODE ] ; then - echo "Problem with cloning branch $1." - echo "Branch $1 will not be added to spec." - else - valid_branches="${valid_branches} $1" - fi - fi -} - -# Creates file with name $configmap_filename -# Inserts ConfigMap metadata and sets name as $configmap_name -add_config_map_metadata() { - echo "Creating ConfigMap spec file: $configmap_filename" - cat << EOF > "$configmap_filename" -apiVersion: v1 -kind: ConfigMap -metadata: - name: $configmap_name - labels: - name: $configmap_name - namespace: onap -data: -EOF -} - -# For each selected branch: -# clones the branch from repository, -# adds schemas from branch to ConfigMap spec -add_schemas() { - for actual_branch in $valid_branches; do - echo "Adding schemas from branch $actual_branch to spec" - add_schemas_from_branch "$actual_branch" - done -} - -# Adds schemas from single branch to spec -# $1 - branch name -add_schemas_from_branch() { - check_arguments $# $EXPECTED_1_ARG - schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}') - for schema in $schemas; do - echo "$1-$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1 - cat "$schema" | indent_string $INDENTATION_LEVEL_2 - done -} >> "$configmap_filename" - -# Generates mapping file for collected schemas directly in spec -generate_mapping_file() { - echo "Generating mapping file in spec" - echo "$SCHEMA_MAP_FILENAME"": |-" | indent_string $INDENTATION_LEVEL_1 >> "$configmap_filename" - echo "[" | indent_string $INDENTATION_LEVEL_2 >> "$configmap_filename" - - for actual_branch in $valid_branches; do - echo "Adding mappings from branch: $actual_branch" - add_mappings_from_branch "$actual_branch" - done - - truncate -s-2 "$configmap_filename" - echo "" >> "$configmap_filename" - echo "]" | indent_string $INDENTATION_LEVEL_2 >> "$configmap_filename" -} - -# Adds mappings from single branch directly to spec -# $1 - branch name -add_mappings_from_branch() { - check_arguments $# $EXPECTED_1_ARG - schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}' ) - - for schema in $schemas; do - repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev) - schema_repo_path=$(echo "$schema" | cut -d/ -f4-) - public_url_schemas_location=${repo_url%.*} - public_url=$public_url_schemas_location/$TREE/$schema_repo_path - local_url=$vendor/$repo_endpoint/$TREE/$schema_repo_path - - echo "{" | indent_string $INDENTATION_LEVEL_3 >> "$configmap_filename" - echo "\"publicURL\": \"$public_url\"," | indent_string $INDENTATION_LEVEL_4 >> "$configmap_filename" - echo "\"localURL\": \"$local_url\"" | indent_string $INDENTATION_LEVEL_4 >> "$configmap_filename" - echo "}," | indent_string $INDENTATION_LEVEL_3 >> "$configmap_filename" - done -} - -create_snippet() { - echo "Generating snippets in file: $snippet_filename" - generate_entries - - cat << EOF > "$snippet_filename" -Snippets for mounting ConfigMap in DCAE VESCollector Deployment -========================================================================= - -## Description -These snippets will override existing in VESCollector schemas and mapping file. - -No extra configuration in VESCollector is needed with these snippets. - -## Snippets -#### spec.template.spec.containers[0].volumeMounts -\`\`\` - - mountPath: /opt/app/VESCollector/etc/externalRepo - name: custom-$vendor-schemas -\`\`\` - -#### spec.template.spec.volumes -\`\`\` - - configMap: - defaultMode: 420 - items: - - key: $SCHEMA_MAP_FILENAME - path: schema-map.json -$schemas_entries - name: $configmap_name - name: custom-$vendor-schemas -\`\`\` -EOF -} - -generate_entries() { - for actual_branch in $valid_branches; do - schemas=$(ls -g $tmp_location/$actual_branch/$schemas_location/*.yaml | awk '{print $NF}') - for schema in $schemas; do - repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev) - schema_repo_path=$(echo "$schema" | cut -d/ -f4-) - - key="$actual_branch-$(basename "$schema")" - path=$vendor/$repo_endpoint/$TREE/$schema_repo_path - schemas_entries="$schemas_entries- key: $key\n path: $path\n" - done - done - schemas_entries=$(echo "$schemas_entries" | indent_string $INDENTATION_LEVEL_5) -} - -# todo add check of global env whether script should be ran -main() { - check_arguments $arguments_number $EXPECTED_7_ARGS - clone_repo - add_config_map_metadata - add_schemas - generate_mapping_file - create_snippet -} - -main \ No newline at end of file diff --git a/external-schema-repo-generator/generator/Makefile b/external-schema-repo-generator/generator/Makefile new file mode 100644 index 0000000..93f3226 --- /dev/null +++ b/external-schema-repo-generator/generator/Makefile @@ -0,0 +1,12 @@ +include environment.config + +all: generate install + +generate: clean + ./generate.sh $(REPOSITORY_URL_HTTPS) $(REPOSITORY_BRANCH) $(SCHEMAS_LOCATION) $(VENDOR_NAME) $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(SNIPPET_FILENAME) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY) + +install: + ./install.sh $(EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX) $(EXTERNAL_REPO_CONFIGMAP_DIRECTORY) $(GENERATION_DIRECTORY) + +clean: + ./clean.sh $(GENERATION_DIRECTORY) diff --git a/external-schema-repo-generator/generator/clean.sh b/external-schema-repo-generator/generator/clean.sh new file mode 100755 index 0000000..8a91d18 --- /dev/null +++ b/external-schema-repo-generator/generator/clean.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +# ============LICENSE_START======================================================= +# OOM +# ================================================================================ +# Copyright (C) 2021 Nokia. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +. ./utils.sh + +# Arguments renaming +arguments_number=$# +generation_directory=$1 + +remove_generated_directory() { + check_arguments $arguments_number $EXPECTED_1_ARG + echo "Removing generated directory: $1" + rm -rf $1 +} + +main() { + check_arguments $arguments_number $EXPECTED_1_ARG + directory_to_remove="./$generation_directory/" + if [ -d "$directory_to_remove" ] + then + remove_generated_directory $directory_to_remove + else + echo "Nothing to clean. No directory: $directory_to_remove" + fi +} + +main diff --git a/external-schema-repo-generator/generator/environment.config b/external-schema-repo-generator/generator/environment.config new file mode 100644 index 0000000..95e94ba --- /dev/null +++ b/external-schema-repo-generator/generator/environment.config @@ -0,0 +1,17 @@ +# Configuration file for externalSchemaRepoGenerator Makefile +# Remember about escaping special characters, e.g. in REPOSITORY_URL + +# ConfigMap configuration +GENERATION_DIRECTORY=generated +EXTERNAL_REPO_CONFIGMAP_FILENAME_PREFIX=external-repo-configmap-spec +EXTERNAL_REPO_CONFIGMAP_DIRECTORY=specs +EXTERNAL_REPO_CONFIGMAP_NAME_PREFIX=external-repo-configmap +SNIPPET_FILENAME=ves-snippet.md + +# Source repository configuration +REPOSITORY_URL_HTTPS=https\://forge.3gpp.org/rep/sa5/MnS.git +REPOSITORY_BRANCH="SA88-Rel16 SA89-Rel17" +SCHEMAS_LOCATION=OpenAPI + +# Vendor description +VENDOR_NAME=3gpp diff --git a/external-schema-repo-generator/generator/generate.sh b/external-schema-repo-generator/generator/generate.sh new file mode 100755 index 0000000..29ddf85 --- /dev/null +++ b/external-schema-repo-generator/generator/generate.sh @@ -0,0 +1,282 @@ +#!/bin/sh + +# ============LICENSE_START======================================================= +# OOM +# ================================================================================ +# Copyright (C) 2020-2021 Nokia. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +. ./utils.sh + +# Arguments renaming +arguments_number=$# +repo_url=$1 +branches=$2 +schemas_location=$3 +vendor=$4 +configmap_filename=$5 +configmap_name=$6 +snippet_filename=$7 +specs_directory=$8 +generation_directory=$9 + +# Constants +SCHEMA_MAP_FILENAME="schema-map.json" +SCHEMA_MAP_NAME="schema-map" +SUCCESS_CODE=0 +TREE=tree +INDENTATION_LEVEL_1=1 +INDENTATION_LEVEL_2=2 +INDENTATION_LEVEL_3=3 +INDENTATION_LEVEL_4=4 +INDENTATION_LEVEL_5=5 + +# Variables +tmp_location=$(mktemp -d) +valid_branches="" + +# Create and move to directory for storing generated files +move_to_generation_directory() { + mkdir "$generation_directory" + cd ./"$generation_directory" +} + +# Indents each line of string by adding indent_size*indent_string spaces on the beginning +# Optional argument is indent_string level, default: 1 +# correct usage example: +# echo "Sample Text" | indent_string 2 +indent_string() { + indent_size=2 + indent_string=1 + if [ -n "$1" ]; then indent_string=$1; fi + pr -to $(expr "$indent_string" \* "$indent_size") +} + +# Clones all branches selected in $BRANCH from $repo_url +clone_repo() { + for actual_branch in $branches; do + clone_branch "$actual_branch" + done +} + +# Clones single branch $1 from $repo_url. +# $1 - branch name +clone_branch() { + check_arguments $# $EXPECTED_1_ARG + if [ -d $tmp_location/"$1" ]; then + echo "Skipping cloning repository." + echo "Branch $1 has already been cloned in the directory ./$tmp_location/$1" + echo "To redownload branch remove ./$tmp_location/$1." + else + echo "Cloning repository from branch $1" + git clone --quiet --single-branch --branch "$1" "$repo_url" "$tmp_location/$1" 2>/dev/null + result=$? + if [ $result -ne $SUCCESS_CODE ] ; then + echo "Problem with cloning branch $1." + echo "Branch $1 will not be added to spec." + else + valid_branches="${valid_branches} $1" + fi + fi +} + +# Creates file with name $configmap_filename +# Inserts ConfigMap metadata and sets name as $configmap_name +# $1 - branch name +add_config_map_metadata() { + branch_configmap_filename="${configmap_filename}-$1.yaml" + branch_configmap_name=$( echo "$configmap_name-$1" | tr '[:upper:]' '[:lower:]' ) + echo "Creating ConfigMap spec file: $branch_configmap_filename" + cat << EOF > "$branch_configmap_filename" +apiVersion: v1 +kind: ConfigMap +metadata: + name: $branch_configmap_name + labels: + name: $branch_configmap_name + namespace: onap +data: +EOF +} + +# For each selected branch: +# clones the branch from repository, +# adds schemas from branch to ConfigMap spec +# $1 - branch name +add_schemas() { + echo "Adding schemas from branch $1 to spec" + add_schemas_from_branch "$1" +} + +# Adds schemas from single branch to spec +# $1 - branch name +add_schemas_from_branch() { + branch_configmap_filename="${configmap_filename}-$1.yaml" + check_arguments $# $EXPECTED_1_ARG + schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}') + for schema in $schemas; do + echo "$1-$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1 + cat "$schema" | indent_string $INDENTATION_LEVEL_2 + done +} >> "$branch_configmap_filename" + +move_to_spec_directory() { + mkdir "$specs_directory" + cd ./"$specs_directory" +} + +# Generates mapping file for collected schemas directly in spec +# $1 - schema map name +generate_mapping_file() { + schema_map_filename="${configmap_filename}-$1.yaml" + echo "Generating mapping file in spec" + echo "$SCHEMA_MAP_FILENAME"": |-" | indent_string $INDENTATION_LEVEL_1 >> "$schema_map_filename" + echo "[" | indent_string $INDENTATION_LEVEL_2 >> "$schema_map_filename" + + for actual_branch in $valid_branches; do + echo "Adding mappings from branch: $actual_branch" + add_mappings_from_branch $actual_branch $schema_map_filename + done + + truncate -s-2 "$schema_map_filename" + echo "" >> "$schema_map_filename" + echo "]" | indent_string $INDENTATION_LEVEL_2 >> "$schema_map_filename" +} + +# Adds mappings from single branch directly to spec +# $1 - branch name +# $2 - schema map file name +add_mappings_from_branch() { + check_arguments $# $EXPECTED_2_ARGS + schema_map_filename="$2" + schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}' ) + + for schema in $schemas; do + repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev) + schema_repo_path=$(echo "$schema" | cut -d/ -f4-) + public_url_schemas_location=${repo_url%.*} + public_url=$public_url_schemas_location/$TREE/$schema_repo_path + local_url=$vendor/$repo_endpoint/$TREE/$schema_repo_path + + echo "{" | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename" + echo "\"publicURL\": \"$public_url\"," | indent_string $INDENTATION_LEVEL_4 >> "$schema_map_filename" + echo "\"localURL\": \"$local_url\"" | indent_string $INDENTATION_LEVEL_4 >> "$schema_map_filename" + echo "}," | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename" + done +} + +# Create snippet file to describe how to connect mount ConfigMaps in VES +create_snippet() { + echo "Generating snippets in file: $snippet_filename" + generate_entries + base_mounts_path="/opt/app/VESCollector/etc/externalRepo" + base_mounts_name="external-repo-$vendor-schemas" + mounts_paths=" + - mountPath: $base_mounts_path/schema-map + name: $base_mounts_name" + + config_maps=" + - configMap: + defaultMode: 420 + name: $configmap_name-$SCHEMA_MAP_NAME + name: $base_mounts_name" + + for actual_branch in $valid_branches; do + + actual_branch_name=$( echo "$actual_branch" | tr '[:upper:]' '[:lower:]' ) + + repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev) + local_url=$vendor/$repo_endpoint/$TREE/$actual_branch/$schemas_location + mounts_paths="$mounts_paths + - mountPath: $base_mounts_path/$local_url + name: $base_mounts_name-$actual_branch_name" + + config_maps="$config_maps + - configMap: + defaultMode: 420 + name: $configmap_name-$actual_branch_name + name: $base_mounts_name-$actual_branch_name" + done + + + cat << EOF > "$snippet_filename" +Snippets for mounting ConfigMap in DCAE VESCollector Deployment +========================================================================= + +## Description +These snippets will override existing in VESCollector schemas and mapping file. + +No extra configuration in VESCollector is needed with these snippets. + +## Snippets +#### spec.template.spec.containers[0].volumeMounts +\`\`\`$mounts_paths +\`\`\` + +#### spec.template.spec.volumes +\`\`\`$config_maps +\`\`\` +EOF +} + +generate_entries() { + for actual_branch in $valid_branches; do + schemas=$(ls -g $tmp_location/$actual_branch/$schemas_location/*.yaml | awk '{print $NF}') + for schema in $schemas; do + repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev) + schema_repo_path=$(echo "$schema" | cut -d/ -f4-) + + key="$actual_branch-$(basename "$schema")" + path=$vendor/$repo_endpoint/$TREE/$schema_repo_path + schemas_entries="$schemas_entries- key: $key\n path: $path\n" + done + done + schemas_entries=$(echo "$schemas_entries" | indent_string $INDENTATION_LEVEL_5) +} + +# Generate specs for branch +# $1 - branch name +generate_specs_for_branch() { + check_arguments $# $EXPECTED_1_ARG + add_config_map_metadata $1 + add_schemas $1 +} + +# Generate specs for schema map +generate_specs_for_schema_map() { + add_config_map_metadata "$SCHEMA_MAP_NAME" + generate_mapping_file "$SCHEMA_MAP_NAME" +} + +# Generate specs for all releases and for schema map +generate_specs() { + move_to_spec_directory + for actual_branch in $valid_branches; do + generate_specs_for_branch $actual_branch + done + generate_specs_for_schema_map + cd .. +} + + +# todo add check of global env whether script should be ran +main() { + check_arguments $arguments_number $EXPECTED_9_ARGS + move_to_generation_directory + clone_repo + generate_specs + create_snippet + move_to_starting_directory +} + +main diff --git a/external-schema-repo-generator/generator/install.sh b/external-schema-repo-generator/generator/install.sh new file mode 100755 index 0000000..e6c8bbc --- /dev/null +++ b/external-schema-repo-generator/generator/install.sh @@ -0,0 +1,114 @@ +#!/bin/sh + +# ============LICENSE_START======================================================= +# OOM +# ================================================================================ +# Copyright (C) 2020-2021 Nokia. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +. ./utils.sh + +# Arguments renaming +arguments_number=$# +spec_configmap_filename=$1 +k8s_configmap_name=$2 +specs_directory=$3 +generation_directory=$4 + +# Constants +MAX_CONFIG_MAP_UPDATE_SIZE=262144 # If ConfigMaps is bigger then this value (in Bytes) it can not be updated +MAX_SPEC_SIZE=1048576 # 1MB + +# Alias +alias kubectl_onap="kubectl -n onap" + +# Checks whether ConfigMap exists +# When file does not exist exits with return code 1 +# $1 - name of spec_configmap_filename +check_if_spec_exists() { + check_arguments $# $EXPECTED_1_ARG + spec_filename="$1" + if [ ! -f "$spec_filename" ]; then + echo "Spec file $spec_filename does not exist." + # todo add location of spec with filename + exit 1 + fi +} + +# If spec file is to big to be apply it needs to be created +# If ConfigMap with same name exists, iot needs do be destroyed +# $1 - name of spec file +create_config_map() { + echo "ConfigMap spec file is too long for 'kubectl apply'. Actual spec length: $spec_size, max spec length: $MAX_CONFIG_MAP_UPDATE_SIZE" + echo "Creating new ConfigMap $k8s_configmap_name" + kubectl_onap replace --force -f "$spec_filename" +} + +# Install ConfigMap from spec +# $1 - name of spec file +# $2 - size of spec file +install_config_map() { + check_arguments $# $EXPECTED_2_ARGS + spec_filename="$1" + spec_size="$2" + if [ "$spec_size" -le $MAX_CONFIG_MAP_UPDATE_SIZE ]; then + echo "Applying ConfigMap $k8s_configmap_name" + kubectl_onap apply -f "$spec_filename" + else + create_config_map + fi +} + +# Uploads ConfigMap spec to Kubernetes +# $1 - name of spec_configmap_filename +upload_config_map() { + check_arguments $# $EXPECTED_1_ARG + spec_filename="$1" + spec_size=$(stat --printf="%s" "$spec_filename") + if [ "$spec_size" -le "$MAX_SPEC_SIZE" ]; then + install_config_map $spec_filename $spec_size + else + echo "WARNING!!!!" + echo " Config file is to big to be installed" + echo " Config file size is: $spec_size Bytes" + echo " Max size is: $MAX_SPEC_SIZE Bytes" + fi +} + +# install all specs located in generated specs directory +# $1 - branch name +install_all_spec_in_directory() { + FILES="./*" + for f in $FILES + do + echo "installing $f" + check_if_spec_exists $f + upload_config_map $f + done +} + +# Moving to directory containing specs +move_to_specs_directory() { + target_directory="$generation_directory/$specs_directory" + echo "Moving to directory containing specs: $target_directory" + cd ./"$target_directory" +} + +main() { + check_arguments $arguments_number $EXPECTED_4_ARGS + move_to_specs_directory + install_all_spec_in_directory + move_to_starting_directory +} + +main diff --git a/external-schema-repo-generator/generator/utils.sh b/external-schema-repo-generator/generator/utils.sh new file mode 100644 index 0000000..de32515 --- /dev/null +++ b/external-schema-repo-generator/generator/utils.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# ============LICENSE_START======================================================= +# OOM +# ================================================================================ +# Copyright (C) 2021 Nokia. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +# Variables +starting_directory="$PWD" + +# Constants +EXPECTED_1_ARG=1 +EXPECTED_2_ARGS=2 +EXPECTED_4_ARGS=4 +EXPECTED_9_ARGS=9 + + +# Checks whether number of arguments is valid +# $1 is actual number of arguments +# $2 is expected number of arguments +check_arguments() { + if [ "$1" -ne "$2" ]; then + echo "Incorrect number of arguments" + exit 1 + fi +} + +# Go back to directory from which script was called +move_to_starting_directory() { + echo "Moving back to: $starting_directory" + cd $starting_directory +} diff --git a/external-schema-repo-generator/install.sh b/external-schema-repo-generator/install.sh deleted file mode 100755 index 2f926b6..0000000 --- a/external-schema-repo-generator/install.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh - -# ============LICENSE_START======================================================= -# OOM -# ================================================================================ -# Copyright (C) 2020 Nokia. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= - -# Arguments renaming -spec_configmap_filename=$1 -k8s_configmap_name=$2 - -# Constants -MAX_SPEC_SIZE=262144 - -# Alias -alias kubectl_onap="kubectl -n onap" - -# Checks whether ConfigMap $spec_configmap_filename exists -# When file does not exist exits with return code 1 -checkIfSpecExists() { - if [ ! -f "$spec_configmap_filename" ]; then - echo "Spec file $spec_configmap_filename does not exist." - # todo add location of spec with filename - exit 1 - fi -} - -# Uploads ConfigMap spec $spec_configmap_filename to Kubernetes -uploadConfigMap() { - spec_size=$(stat --printf="%s" "$spec_configmap_filename") - if [ "$spec_size" -ge $MAX_SPEC_SIZE ]; then - echo "ConfigMap spec file is too long for 'kubectl apply'. Actual spec length: $spec_size, max spec length: $MAX_SPEC_SIZE" - echo "Creating new ConfigMap $k8s_configmap_name" - kubectl_onap create -f "$spec_configmap_filename" - else - echo "Applying ConfigMap $k8s_configmap_name" - kubectl_onap apply -f "$spec_configmap_filename" - fi -} - -main() { - checkIfSpecExists - uploadConfigMap -} - -main - -- cgit 1.2.3-korg