path: root/policy-db-migrator/src/main/docker/db-migrator

#!/usr/bin/env sh

# ============LICENSE_START=======================================================
# ONAP
# ================================================================================
# Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
# Modifications Copyright (C) 2021 Nordix Foundation.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============LICENSE_END=========================================================

# #####################################################################
#
# Upgrade/Downgrade SQL File Name Format:
#
#     <VERSION>-<pdp|feature-name>[-description](.upgrade|.downgrade).sql
#
# This tool operates on a migration working directory at
#
#    $POLICY_HOME/etc/db/migration
#
# Upgrade/Downgrade files for each schema (aka database) names to be maintained
# by this tool are located at
#
#    $POLICY_HOME/etc/db/migration/<schema-name>/sql
#
# The nature of the migration directories is dynamic.
# A number of environment variables needs to be initialized
# prior to running db-migrator.
# These vaiables specifiy the connection details for the database
# to be upgraded/downgraded.
#
# The repository of upgrade/downgrade scripts is located in the
# /home/policy/sql directory.
# Two additional scripts have been provided to prepare the directories/files
# needed to perform the upgrade/downgrade.
#
# At any given time the following invariant must be preserved in any given
# $POLICY_HOME/etc/db/migration/<schema-name>/sql directory
#
#     There is only upgrade scripts, or only downgrade scripts, or none.
#
# #####################################################################

source ${POLICY_HOME}/etc/profile.d/env.sh

METADATA_DB=migration
METADATA_TABLE=${METADATA_DB}.db_metadata_versions
MIGRATION_DIR=${POLICY_HOME}/etc/db/migration
ZERO_VERSION="0"
BASE_VERSION="0800"
UPGRADE_SQL_SUFFIX=".upgrade.sql"
DOWNGRADE_SQL_SUFFIX=".downgrade.sql"

SQL_QUOTES="SET SESSION SQL_MODE=ANSI_QUOTES;"

#####################################################
# usage
#####################################################

function usage() {
    echo
    echo -e "syntax: $(basename "$0") "
    echo -e "\t -s <schema-name> "
    echo -e "\t [-b <migration-dir>] "
    echo -e "\t [-f <from-version>]"
    echo -e "\t [-t <target-version>]"
    echo -e "\t -o <operations> "
    echo
    echo -e "\t where <operations>=upgrade|downgrade|auto|version|erase|report"
    echo
    echo
    echo -e "Configuration Options:"
    echo -e "\t -s|--schema|--database:  schema to operate on ('ALL' to apply on all)"
    echo -e "\t -b|--basedir: overrides base DB migration directory"
    echo -e "\t -f|--from: overrides current release version for operations"
    echo -e "\t -t|--target: overrides target release to upgrade/downgrade"
    echo
    echo -e "Operations:"
    echo -e "\t upgrade: upgrade operation"
    echo -e "\t downgrade: performs a downgrade operation"
    echo -e "\t auto: autonomous operation, determines upgrade or downgrade"
    echo -e "\t version: returns current version, and in conjunction if '-f' sets the current version"
    echo -e "\t erase: erase all data related <schema> (use with care)"
    echo -e "\t report: migration detailed report on an schema"
    echo -e "\t ok: is the migration status valid"
    echo
    echo
}

#####################################################
# current_schema
#####################################################

function current_schema
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- current_schema --"
        set -x
    fi

    local rc
    local query="SELECT count(table_name) from information_schema.tables where table_schema='${SCHEMA}'"

    TABLES_IN_SCHEMA=$(${MYSQL} --skip-column-names --silent --execute "${query}")
    if [ ${TABLES_IN_SCHEMA} -gt 0 ] && [ "${BASE_VERSION}" \> "${CURRENT_RELEASE}" ]; then
        set_current_release "${BASE_VERSION}"
        return $?
    fi

    return 0
}

#####################################################
# ensure global metadata
#####################################################

function ensure_metadata
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- ensure_metadata --"
        set -x
    fi

    local sql rc

    sql="CREATE DATABASE IF NOT EXISTS ${METADATA_DB};"
    ${MYSQL} --execute "${sql}"
    rc=$?
    if [ ${rc} -ne 0 ]; then
        return ${rc}
    fi

    sql="CREATE TABLE IF NOT EXISTS ${METADATA_TABLE} "
    sql=${sql}"(name VARCHAR(60) NOT NULL, version VARCHAR(20), "
    sql=${sql}"PRIMARY KEY(name));"
    ${MYSQL} --execute "${sql}"
    return $?
}


#####################################################
# ensure metadata on a per schema basis
#####################################################

function ensure_metadata_schema
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- ensure_metadata_schema --"
        set -x
    fi

    local sql rc

    sql="CREATE TABLE IF NOT EXISTS ${METADATA_HISTORY} "
    sql=${sql}"(ID bigint(20) unsigned NOT NULL AUTO_INCREMENT, script VARCHAR(80) NOT NULL, "
    sql=${sql}"operation VARCHAR(10), from_version VARCHAR(20), to_version VARCHAR(20), tag VARCHAR(20), "
    sql=${sql}"success VARCHAR(1), atTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, "
    sql=${sql}"PRIMARY KEY(ID));"
    ${MYSQL} --execute "${sql}"
    rc=$?
    if [ ${rc} -ne 0 ]; then
        return ${rc}
    fi

    sql="CREATE DATABASE IF NOT EXISTS ${SCHEMA_DB};"
    ${MYSQL} --execute "${sql}"
    return $?
}


#####################################################
# target_release
#####################################################

function target_release
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- target_release --"
        set -x
    fi

    local sql sqlName upgradeSqls downgradeSqls

    TARGET_UPGRADE_RELEASE=${ZERO_VERSION}
    TARGET_DOWNGRADE_RELEASE=${ZERO_VERSION}

    upgradeSqls=$(ls -v -r "${UPGRADE_DIR}"/*/upgrade/*.sql 2> /dev/null)
    for sql in ${upgradeSqls}; do
        TARGET_UPGRADE_RELEASE="$(basename $(dirname $(dirname $sql)))"
        break
    done

    # default unless overriden
    TARGET_DOWNGRADE_RELEASE="${ZERO_VERSION}"
}

#####################################################
# is_upgrade
#####################################################

function is_upgrade
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- is_upgrade --"
        set -x
    fi

    local upgradeSqls

    upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2> /dev/null)
    if [ -z "${upgradeSqls}" ]; then
        return 1
    else
        return 0
    fi
}


#####################################################
# is_downgrade
#####################################################

function is_downgrade
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- is_downgrade --"
        set -x
    fi

    local downgradeSqls

    downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2> /dev/null)
    if [ -z "${downgradeSqls}" ]; then
        return 1
    else
        return 0
    fi
}


#####################################################
# set_current_release
#####################################################

function set_current_release
{
        if [ "${DEBUG}" = "y" ]; then
                echo "-- set_current_release --"
                set -x
        fi

        CURRENT_RELEASE="${1}"

        local sql
        sql="INSERT INTO ${METADATA_TABLE} (name, version) "
        sql=${sql}"VALUES('${SCHEMA}', '${CURRENT_RELEASE}') "
        sql=${sql}"ON DUPLICATE KEY UPDATE version='${CURRENT_RELEASE}';"

        ${MYSQL} --execute "${sql}"
        return $?
}

#####################################################
# current_release
#####################################################

function current_release
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- current_release --"
        set -x
    fi

    local rc
    local query="SELECT version FROM ${METADATA_TABLE} WHERE name='${SCHEMA}'"

    CURRENT_RELEASE=$(${MYSQL} --skip-column-names --silent --execute "${query}")
    if [ -z "${CURRENT_RELEASE}" ]; then
        set_current_release "${ZERO_VERSION}"
        return $?
    fi

    return 0
}

#####################################################
# previous_release
#####################################################

function previous_release
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- previous_release --"
        set -x
    fi
    local current_release="${1}"
    local current_release_int_val previous_release_int_val previous_release

    if [ $current_release == $ZERO_VERSION ] || [ $current_release == $BASE_VERSION ]; then
         PREVIOUS_RELEASE=$ZERO_VERSION
    else
         current_release_int_val=$(echo $current_release  | awk '{$0=int($0)}1')
         let previous_release_int_val=$current_release_int_val-100
         if [ $previous_release_int_val -lt 1000 ]; then
                 previous_release="0"$previous_release_int_val
         else
                 previous_release=$previous_release_int_val
         fi
         PREVIOUS_RELEASE=$previous_release
    fi

    return 0
}


#####################################################
# last_operation_status
#####################################################

function last_operation_status
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- last_operation_status --"
        set -x
    fi

    local query rc row

    query="SELECT script, operation, from_version, to_version, success FROM  "
    query=${query}"${METADATA_HISTORY} WHERE id="
    query=${query}"(SELECT MAX(id) from ${METADATA_HISTORY}) "
    query=${query}"AND success=0"

    row=$(${MYSQL} --skip-column-names --silent --execute "${query}")

    if [ "${row}" != "" ]; then
        LAST_SCRIPT=$(echo $row | awk '{print $1}')
        LAST_OPERATION=$(echo $row | awk '{print $2}')
        LAST_FROM_VERSION=$(echo $row | awk '{print $3}')
        LAST_TO_VERSION=$(echo $row | awk '{print $4}')
        LAST_STATUS=$(echo $row | awk '{print $5}')
        rc=0
    else
        rc=1
    fi

    return $rc;
}

#####################################################
# execute sql script history
#####################################################

function track_script
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- track_script $* --"
        set -x
    fi

    local script="${1}" operation="${2}" success="${3}" from_version="${4}" to_version="${5}" tag="${6}"
    if [ $operation == "downgrade" ]; then
       to_version=${TARGET_DOWNGRADE_RELEASE}
    fi

    local sql="INSERT INTO ${METADATA_HISTORY}(script,operation,from_version,to_version,tag,success,atTime) "
    sql=${sql}"VALUES ('${script}','${operation}','${from_version}','${to_version}','${tag}','${success}',now()) "

    ${MYSQL} --execute "${sql}"
    return $?
}


#####################################################
# execute sql script
#####################################################

function run_script
{
    if [ "${DEBUG}" == "y" ]; then
        echo "-- run_script $* --"
        set -x
    fi

    local operation="${1}" script="${2}" scriptPath="${3}" fromVersion="${4}" toVersion="${5}" tag="${6}"

    echo
    echo "> ${operation} ${script}"

    ${MYSQL} ${SCHEMA} --verbose < "${scriptPath}"
    local rc=$?
    if [ ${rc} -ne 0 ]; then
        success="0"
    else
        success="1"
    fi

    track_script "${script}" "${operation}" "${success}" "${fromVersion}" "${toVersion}" "${tag}"

    return ${rc}
}

#####################################################
# upgrade
#####################################################

function upgrade
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- upgrade --"
        set -x
    fi

    local sqlName sqlFile schemaVersion upgradeSqls rc tagDate tag retry compare
    tagDate=$(date +%d%m%y%H%M%S)
    retry=0

    echo "upgrade: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
    last_operation_status
    rc=$?
    if [ $rc -eq 0 ]; then
         echo "Partial $LAST_OPERATION detected"
         retry=1
         if [ "${LAST_OPERATION}" == 'upgrade' ]; then
              CURRENT_RELEASE="${LAST_FROM_VERSION}"
              # less than
              compare="lt"
              echo "Upgrade will attempt to resume at $LAST_SCRIPT"
         else
              CURRENT_RELEASE="${LAST_TO_VERSION}"
              # less than or equal
              compare="le"
              echo "Upgrade will attempt to resume after $LAST_SCRIPT"
         fi
    fi

    if [ ${CURRENT_RELEASE} \< ${TARGET_UPGRADE_RELEASE} ]; then
        upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2> /dev/null)
        for sqlFile in ${upgradeSqls}; do
            sqlName=$(basename "${sqlFile}")
            schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
            previous_release $schemaVersion
            if [ "${schemaVersion}" -gt "${CURRENT_RELEASE}" ] && \
                [ "${schemaVersion}" -le "${TARGET_UPGRADE_RELEASE}" ]; then
                tag=${tagDate}"${schemaVersion}u"
                if [ $retry -eq 1 ] && \
                    # Compare the numeric portion of the filename because shell doesn't support string comparison
                    [ $(echo ${sqlName}| awk -F- '{print $1}') -${compare} $(echo ${LAST_SCRIPT}| awk -F- '{print $1}') ]; then
                     rc=1
                else
                     rc=0
                fi
                if [ ${rc} -eq 0 ]; then
                    run_script "upgrade" "${sqlName}" "${sqlFile}" "${PREVIOUS_RELEASE}" "${schemaVersion}" "${tag}"
                    rc=$?
                    if [ ${rc} -ne 0 ]; then
                        echo "${SCHEMA}: upgrade aborted at ${schemaVersion} by script ${sqlName}"
                        set_current_release "${schemaVersion}"
                        return ${rc}
                    fi
                fi
            fi
        done

        set_current_release "${TARGET_UPGRADE_RELEASE}"
    fi

    return 0
}

#####################################################
# downgrade
#####################################################

function downgrade
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- downgrade --"
        set -x
    fi

    local sqlName sqlFile schemaVersion downgradeSqls rc tagDate tag retry compare
    tagDate=$(date +%d%m%y%H%M%S)
    retry=0

    echo "downgrade: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
    last_operation_status
    rc=$?
    if [ $rc -eq 0 ]; then
         echo "Partial $LAST_OPERATION detected"
         retry=1
         if [ "${LAST_OPERATION}" == 'downgrade' ]; then
              CURRENT_RELEASE="${LAST_FROM_VERSION}"
              # greater than
              compare="gt"
              echo "Downgrade will attempt to resume at $LAST_SCRIPT"
         else
              CURRENT_RELEASE="${LAST_TO_VERSION}"
              # greater than or equal to
              compare="ge"
              echo "Downgrade will attempt to resume before $LAST_SCRIPT"
         fi
    fi

    if [ ${CURRENT_RELEASE} \> ${TARGET_DOWNGRADE_RELEASE} ]; then
        downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2> /dev/null)
        for sqlFile in ${downgradeSqls}; do
            sqlName=$(basename "${sqlFile}")
            schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
            if [ "${schemaVersion}" -le "${CURRENT_RELEASE}" ] && \
                [ "${schemaVersion}" -gt "${TARGET_DOWNGRADE_RELEASE}" ]; then
                tag=${tagDate}"${schemaVersion}d"
                if [ $retry -eq 1 ] && \
                    # Compare the numeric portion of the filename because shell doesn't support string comparison
                    [ $(echo ${sqlName}| awk -F- '{print $1}') -${compare} $(echo ${LAST_SCRIPT}| awk -F- '{print $1}') ]; then
                     rc=1
                else
                     rc=0
                fi
                if [ ${rc} -eq 0 ]; then
                    run_script "downgrade" "${sqlName}" "${sqlFile}" "${schemaVersion}" "${PREVIOUS_RELEASE}" ${tag}
                    rc=$?
                    if [ ${rc} -ne 0 ]; then
                        echo "${SCHEMA}: downgrade aborted at ${schemaVersion} by script ${sqlName}"
                        set_current_release "${schemaVersion}"
                        return ${rc}
                    fi
                fi
            fi
        done

        set_current_release "${TARGET_DOWNGRADE_RELEASE}"
    fi

    return 0
}

#####################################################
# erase
#####################################################

function erase
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- erase --"
        set -x
    fi

    local updateMetadata="UPDATE ${METADATA_TABLE} SET version='${ZERO_VERSION}';"
    ${MYSQL} --execute "${updateMetadata}"

    local deleteHistory="DELETE FROM ${METADATA_HISTORY};"
    ${MYSQL} --execute "${deleteHistory}"

    local dropDB="DROP DATABASE IF EXISTS ${SCHEMA_DB}";
    ${MYSQL} --execute "${dropDB}"
}

#####################################################
# report
#####################################################

function report
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- report --"
        set -x
    fi

    local versionSql="SELECT * FROM ${METADATA_TABLE} WHERE name='${SCHEMA}';"
    ${MYSQL} --execute "${versionSql}"

    local historySql="SELECT * FROM ${METADATA_HISTORY} ORDER BY id, atTime ASC;"
    ${MYSQL} --execute "${historySql}"

    okay
}

function okay
{
    if [ "${DEBUG}" = "y" ]; then
        echo "-- okay --"
        set -x
    fi

    local rc=0
    if is_upgrade; then
        if [ "${CURRENT_RELEASE}" = "${TARGET_UPGRADE_RELEASE}" ]; then
            echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
        else
            echo "${SCHEMA}: upgrade available: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
            rc=1
        fi
    else
        if [ "${CURRENT_RELEASE}" = "${TARGET_DOWNGRADE_RELEASE}" ]; then
            echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
        else
            echo "${SCHEMA}: downgrade available: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
            rc=1
        fi
    fi

    return ${rc}
}

#####################################################
# MAIN
#####################################################

if [ "${DEBUG}" = "y" ]; then
    echo "-- $0 $* --"
    set -x
fi
until [ -z "$1" ]; do
    case $1 in
        -s|--schema|--database)    shift
             SCHEMA=$1
             ;;
        -b|--basedir)   shift
             MIGRATION_DIR=$1
             ;;
        -t|--target)    shift
             INPUT_TARGET_RELEASE=$1
             ;;
        -f|--from)      shift
             INPUT_FROM_RELEASE=$1
             ;;
        -o|--operation) shift
             OPERATION=$1
             ;;
        *)              usage
             exit 1
             ;;
    esac
    shift
done

case ${OPERATION} in
    upgrade)    ;;
    downgrade)  ;;
    auto)       ;;
    version)    ;;
    erase)      ;;
    report)     ;;
    ok)         ;;
    *)          echo "error: invalid operation provided"
                usage
                exit 1
                ;;
esac

if [ -z "${SCHEMA}" ]; then
    echo "error: a database name must be provided"
    usage
    exit 2
fi

source "${POLICY_HOME}"/etc/profile.d/env.sh

if [ -z "${SQL_HOST}" ] || [ -z "${SQL_USER}" ] || [ -z "${SQL_PASSWORD}" ]; then
    echo "error: no database has been set up"
    exit 4
fi

MYSQL="mysql -u${SQL_USER} -p${SQL_PASSWORD} -h ${SQL_HOST}";
if ! ${MYSQL} -h"${SQL_HOST}" --execute "show databases;" > /dev/null 2>&1; then
    echo "error: No DB connectivity to ${SQL_HOST} for ${SQL_USER}"
    exit 5
fi

if [ "${SCHEMA}" = "ALL" ]; then
    SCHEMA="*"
fi

SCHEMA_S=$(ls -d "${MIGRATION_DIR}"/${SCHEMA}/ 2> /dev/null)
if [ -z "${SCHEMA_S}" ]; then
    echo "error: no databases available"
    exit 0
fi

if ! ensure_metadata; then
    echo "error: migration metadata not accessible"
    exit 7
fi

rc=0
for dbPath in ${SCHEMA_S}; do
    SCHEMA=$(basename "${dbPath}")
    SCHEMA_DB="\`${SCHEMA}\`"
    UPGRADE_DIR="${MIGRATION_DIR}"/"${SCHEMA}"/sql
    DOWNGRADE_DIR=${UPGRADE_DIR}
    METADATA_HISTORY="${METADATA_DB}.\`${SCHEMA}_change_history\`"
    TARGET_RELEASE=${INPUT_TARGET_RELEASE}

    if is_upgrade && is_downgrade; then
        echo "${SCHEMA}: failure: invalid configuration: ${UPGRADE_SQL_SUFFIX} and "\
            "${DOWNGRADE_SQL_SUFFIX} exist under ${DOWNGRADE_DIR}"
        rc=1
        continue
    fi

    if [ "${operation}" = "auto" ]; then
        if is_upgrade; then
            operation=upgrade
        else
            operation=downgrade
        fi
    fi

    if ! ensure_metadata_schema; then
        echo "${SCHEMA}: failure: metadata not accessible for this schema"
        continue
    fi

    if [ -z "${TARGET_RELEASE}" ]; then
        target_release
    else
        # user asked to override
        TARGET_UPGRADE_RELEASE="${TARGET_RELEASE}"
        TARGET_DOWNGRADE_RELEASE="${TARGET_RELEASE}"
    fi

    if [ -z "${CURRENT_RELEASE}" ]; then
        if ! current_release; then
            echo "${SCHEMA}: failure: cannot obtain current release"
            continue
        fi
    else
        if ! set_current_release "${CURRENT_RELEASE}"; then
            echo "${SCHEMA}: failure: cannot set current release"
            continue
        fi
    fi

    # Check if the schema has already been installed
    current_schema

    if [ -n "${INPUT_FROM_RELEASE}" ]; then
        if [ "${CURRENT_RELEASE}" \> "${INPUT_FROM_RELEASE}" ] || [ "${CURRENT_RELEASE}" \< "${INPUT_FROM_RELEASE}" ]; then
            echo "${SCHEMA}: On version ${CURRENT_RELEASE} cannot ${OPERATION} from ${INPUT_FROM_RELEASE}"
            continue
        else
            CURRENT_RELEASE=${INPUT_FROM_RELEASE}
        fi
    fi

    case ${OPERATION} in
        upgrade)     if upgrade; then
                         echo "${SCHEMA}: OK: upgrade (${CURRENT_RELEASE})"
                     else
                         rc=1
                         echo "${SCHEMA}: failure: upgrade to release ${TARGET_UPGRADE_RELEASE} (${CURRENT_RELEASE})"
                     fi
                     ;;
        downgrade)   if downgrade; then
                         echo "${SCHEMA}: OK: downgrade (${CURRENT_RELEASE})"
                     else
                        rc=1
                        echo "${SCHEMA}: failure: downgrade to release ${TARGET_DOWNGRADE_RELEASE} (${CURRENT_RELEASE})"
                     fi
                     ;;
        version)     echo "${SCHEMA}: ${CURRENT_RELEASE}"
                     ;;
        erase)       erase
                     ;;
        report)      report
                     ;;
        ok)          okay
                     ;;
        esac

done
exit $rc