diff options
Diffstat (limited to 'pgaas/src/stage')
| -rwxr-xr-x | pgaas/src/stage/opt/app/pgaas/bin/check_cluster | 28 | ||||
| -rwxr-xr-x | pgaas/src/stage/opt/app/pgaas/bin/dcae_admin_db.py | 74 | ||||
| -rwxr-xr-x | pgaas/src/stage/opt/app/pgaas/bin/isrw | 13 | ||||
| -rwxr-xr-x | pgaas/src/stage/opt/app/pgaas/bin/list_masters | 46 | ||||
| -rw-r--r-- | pgaas/src/stage/opt/app/pgaas/bin/makefile | 8 | ||||
| -rw-r--r-- | pgaas/src/stage/opt/app/pgaas/bin/start_maintenance_mode | 48 | ||||
| -rw-r--r-- | pgaas/src/stage/opt/app/pgaas/bin/stop_maintenance_mode | 28 | ||||
| -rw-r--r-- | pgaas/src/stage/opt/app/pgaas/man/check_cluster.man | 12 | ||||
| -rw-r--r-- | pgaas/src/stage/opt/app/pgaas/man/makefile | 4 | ||||
| -rw-r--r-- | pgaas/src/stage/opt/app/pgaas/man/start_maintenance_mode.man | 28 | ||||
| -rw-r--r-- | pgaas/src/stage/opt/app/pgaas/man/stop_maintenance_mode.man | 24 |
11 files changed, 291 insertions, 22 deletions
diff --git a/pgaas/src/stage/opt/app/pgaas/bin/check_cluster b/pgaas/src/stage/opt/app/pgaas/bin/check_cluster index 0af6d95..c55d973 100755 --- a/pgaas/src/stage/opt/app/pgaas/bin/check_cluster +++ b/pgaas/src/stage/opt/app/pgaas/bin/check_cluster @@ -23,7 +23,7 @@ # -v verbose # # DESCRIPTION -# Loop through the nodes in the cluster, using pgwget to determine how many are masters, secondaries, or not up. +# Loop through the nodes in the cluster, using pgwget to determine how many are masters, secondaries, down for maintenance, or not up. # Complain about certain situations. # If there are multiple masters, and this not the first master in the list, then: # run pg_ctl_restart @@ -44,19 +44,22 @@ usage() { exec 1>&2 [ $# -gt 0 ] && echo "$@" - echo "Usage: $0 [-v] [-l] [-t timeout]" + echo "Usage: $0 [-v] [-l] [-t timeout] [-d file]" echo -e " -l do not check localhost first (and restarting the service if necessary)" echo -e " -t timeout set how long to wait when accessing the servers" echo -e " -v verbose" + echo -e " -d file duplicate the status output to the given file" exit 1 } VERBOSE=false TIMEOUT=10 TESTLOCAL=: -while getopts lt:v c +DFILE= +while getopts d:lt:v c do case "$c" in + d ) DFILE=$OPTARG ;; l ) TESTLOCAL=false ;; t ) TIMEOUT=$OPTARG ;; v ) VERBOSE=: ;; @@ -70,13 +73,16 @@ master_count=0 secondary_count=0 total_count=0 down_count=0 +maintenance_count=0 -DOWNS= MASTERS= SECONDARIES= +MAINTENANCES= +DOWNS= MSEP= SSEP= +BSEP= DSEP= HOSTNAME=$(hostname -f) FOUNDPREVIOUSMASTER= @@ -86,7 +92,7 @@ if $TESTLOCAL then isrw=`pgwget --tries=1 --read-timeout=$TIMEOUT --quiet -O/dev/stdout http://localhost:8000/isrw` case "$isrw" in - Master | Secondary ) ;; + Master | Secondary | Maintenance ) ;; * ) echo "$(date)|WARNING|RESTARTED|Local iDNS-responder.py not responding. Restarting." ps -fu postgres | grep "python3 /opt/app/postgresql-prep/bin/iDNS-responder.py" | grep -v grep | awk '{print "kill " $2}' | sh @@ -118,6 +124,12 @@ do SECONDARIES="$SECONDARIES$SSEP$i" SSEP=" " ;; + Maintenance ) + (( maintenance_count = maintenance_count + 1 )) + (( total_count = total_count + 1 )) + MAINTENANCES="$MAINTENANCES$BSEP$i" + BSEP=" " + ;; * ) DOWNS="$DOWNS$DSEP$i" DSEP=" " @@ -130,7 +142,11 @@ done (( up_count = master_count + secondary_count )) date=$(date) -echo "$date|INFO|masters=$master_count $MASTERS|secondaries=$secondary_count $SECONDARIES|down=$down_count $DOWNS|" +output="$date|INFO|masters=$master_count $MASTERS|secondaries=$secondary_count $SECONDARIES|maintenance=$maintenance_count $MAINTENANCES|down=$down_count $DOWNS|" +echo "$output" +if [ -n "$DFILE" ] +then (umask 022; echo "$output" > $DFILE.tmp && mv $DFILE.tmp $DFILE) +fi FORCEROOFF=/var/run/postgresql/force-ro-off if [ $master_count -lt 1 ] diff --git a/pgaas/src/stage/opt/app/pgaas/bin/dcae_admin_db.py b/pgaas/src/stage/opt/app/pgaas/bin/dcae_admin_db.py index 80da148..9d5b05d 100755 --- a/pgaas/src/stage/opt/app/pgaas/bin/dcae_admin_db.py +++ b/pgaas/src/stage/opt/app/pgaas/bin/dcae_admin_db.py @@ -121,6 +121,7 @@ def usage(msg = None): print("dcae_admin_db.py [options] suspend") print("dcae_admin_db.py [options] resume") print("dcae_admin_db.py [options] test") + print("dcae_admin_db.py [options] newdb dbname admin-pswd user-pswd viewer-pswd") print("") print("options:") print("-H / --dbhost= - host name, defaults to CFG['dcae_admin_db_hostname']") @@ -282,6 +283,10 @@ def main(): if len(args) != 1: usage("too many arguments") testOperations(keyedOptions) + elif args[0] == "newdb": + if len(args) != 5: + usage("wrong number of arguments") + newDb(keyedOptions, args[1], args[2], args[3], args[4]) else: usage("unrecognized operation '%s'" % args[0]) @@ -313,11 +318,14 @@ def testOperations(options): ret = runProgram(cmd) # pg_ctl: no server running # pg_ctl: server is running (PID: 13988) + # does /var/run/postgresql/inmaintenance exist? -> YELLOW cmdRepmgr = ["pgrep", "repmgrd"] retRepmgr = runProgram(cmdRepmgr) msg = "????" - if re.search("no server running", ret): + if os.path.isfile("/var/run/postgresql/inmaintenance"): + msg = "YELLOW: in maintenance mode" + elif re.search("no server running", ret): msg = "RED: no PG server running" elif re.search("server is running", ret) and re.search("[0-9]+", retRepmgr): msg = "GREEN" @@ -377,6 +385,13 @@ def configurationChanged(options, jsonFile): if verbose: dumpJSON(inp, "modified JSON") + setupJsonDatabases(options, inp) + +def setupDictionaryDatabases(options, inp): + """ + Set up the databases listed in the dictionary + """ + # trace("version=%s" % requireJSON("version", inp, "version")) requireJSON("databases", inp, "databases") con = None @@ -392,6 +407,63 @@ def configurationChanged(options, jsonFile): con.commit() con.close() +def newDb(options, dbName, adminPswd, userPswd, viewerPswd): + """ + Given the database name and passwords, set up a database and corresponding users. + For example, with dbname="foo", adminPswd="fooa", userPswd="foou" and viewerPswd="foov", + act the same as if we had received the json configuration: + { + 'databases': { + 'foo': { + 'ownerRole': 'foo_admin', + 'roles': { + 'foo_admin': { + 'password': 'fooa', + 'role': 'admin' + }, + 'foo_user': { + 'password': 'foou', + 'role': 'writer' + }, + 'foo_viewer': { + 'password': 'foov', + 'role': 'reader' + } + } + } + } + } + """ + if not re.match("^[A-Za-z][A-Za-z0-9_]*$", dbName): + errorPrint("'%s' is not a valid database name" % dbName) + return + + adminName = dbName + "_admin" + userName = dbName + "_user" + viewerName = dbName + "_viewer" + + setupDictionaryDatabases(options, { + 'databases': { + dbName: { + 'ownerRole': adminName, + 'roles': { + adminName: { + 'password': adminPswd, + 'role': 'admin' + }, + userName: { + 'password': userPswd, + 'role': 'writer' + }, + viewerName: { + 'password': viewerPswd, + 'role': 'reader' + } + } + } + } + }) + def dumpJSON(js, msg): tracePrint("vvvvvvvvvvvvvvvv %s" % msg) tracePrint(json.dumps(js, indent=4)) diff --git a/pgaas/src/stage/opt/app/pgaas/bin/isrw b/pgaas/src/stage/opt/app/pgaas/bin/isrw index 4ec3dcb..817030d 100755 --- a/pgaas/src/stage/opt/app/pgaas/bin/isrw +++ b/pgaas/src/stage/opt/app/pgaas/bin/isrw @@ -22,7 +22,12 @@ fi export PATH=$PATH:/opt/java/jdk/jdk170/bin:/opt/app/cdf/bin:/opt/app/pgaas/bin:$PATH -case `show_pg_is_in_recovery` in - *f* ) echo "Master" ;; - *t* ) echo "Secondary" ;; -esac +if [ -f /var/run/postgresql/inmaintenance ] +then + echo "Maintenance" +else + case `show_pg_is_in_recovery` in + *f* ) echo "Master" ;; + *t* ) echo "Secondary" ;; + esac +fi diff --git a/pgaas/src/stage/opt/app/pgaas/bin/list_masters b/pgaas/src/stage/opt/app/pgaas/bin/list_masters index d922739..9e22cd1 100755 --- a/pgaas/src/stage/opt/app/pgaas/bin/list_masters +++ b/pgaas/src/stage/opt/app/pgaas/bin/list_masters @@ -27,17 +27,53 @@ else CFGDIR=/opt/app/postgresql-config/ fi PATH=$PGDIR/bin:$CDF/bin:/opt/app/pgaas/bin:/opt/app/postgresql-prep/bin:$PATH +usage() +{ + exec 1>&2 + [ $# -gt 0 ] && echo "$@" + echo "Usage: $0 [-r] [-v]" + echo -e " -r check /ro instead of /rw" + echo -e " -v verbose" + exit 1 +} + +VERBOSE=false +CMD=rw +QUIET=--quiet +OUTPUT=-O/dev/null +while getopts rvQ c +do + case "$c" in + r ) CMD=ro ;; + Q ) QUIET= OUTPUT= ;; + v ) VERBOSE=: ;; + \?) usage ;; + esac +done +shift $(($OPTIND - 1)) + # loop through the nodes in the cluster, using pgwget to determine if any are a master. Save in $@ for i in $(getpropvalue -n pgnodes | sed 's/|/ /g') do - if pgwget --quiet -O/dev/null http://$i:8000/rw + $VERBOSE && echo -n "Checking $i" + if pgwget $QUIET $OUTPUT http://$i:8000/$CMD then set -- "$@" $i fi done echo "$@" -case $# in - 1 ) exit 0 ;; # one master exists and is running - 0 ) exit 1 ;; # no masters exist - * ) exit 2 ;; # more than one master exist +case "$CMD" in + rw ) + case $# in + 1 ) exit 0 ;; # one master exists and is running + 0 ) exit 1 ;; # no masters exist + * ) exit 2 ;; # more than one master exist + esac + ;; + ro ) + case $# in + 0 ) exit 1 ;; # no masters exist + * ) exit 0 ;; # one or more masters+secondaries exist + esac + ;; esac diff --git a/pgaas/src/stage/opt/app/pgaas/bin/makefile b/pgaas/src/stage/opt/app/pgaas/bin/makefile index 877f8ba..f3ff097 100644 --- a/pgaas/src/stage/opt/app/pgaas/bin/makefile +++ b/pgaas/src/stage/opt/app/pgaas/bin/makefile @@ -26,7 +26,7 @@ SHFILES= check_cluster isrw list_masters \ runpsql runpsqll startpsql setpropvalue show_pg_is_in_recovery show_pg_stat_activity show_pg_stat_archiver show_pg_stat_bgwriter \ show_pg_stat_database show_pg_stat_database_conflicts show_pg_statio_user_functions show_pg_statio_user_indexes \ show_pg_statio_user_sequences show_pg_statio_user_tables show_pg_stat_user_indexes show_pg_stat_user_tables \ - update_var_run_isrw + update_var_run_isrw startbackup stopbackup stage: rm -rf $(STAGEDIR)/$(DISTPATH)/bin @@ -40,8 +40,10 @@ stage: ;; \ makefile | *~ ) ;; \ * ) \ - cp $$i $(STAGEDIR)/$(DISTPATH)/bin/$$i; \ - chmod a+x $(STAGEDIR)/$(DISTPATH)/bin/$$i; \ + if [ -f $$i ]; then \ + cp $$i $(STAGEDIR)/$(DISTPATH)/bin/$$i; \ + chmod a+x $(STAGEDIR)/$(DISTPATH)/bin/$$i; \ + fi; \ ;; \ esac; \ done diff --git a/pgaas/src/stage/opt/app/pgaas/bin/start_maintenance_mode b/pgaas/src/stage/opt/app/pgaas/bin/start_maintenance_mode new file mode 100644 index 0000000..d1c1827 --- /dev/null +++ b/pgaas/src/stage/opt/app/pgaas/bin/start_maintenance_mode @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this code except in compliance +# with the License. You may obtain a copy of the License +# at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +PATH=/opt/app/pgaas/bin:/opt/app/postgresql-prep/bin:$PATH + +INMAINTENANCE=/var/run/postgresql/inmaintenance +ISRW=/var/run/postgresql/isrw + +if [ -f $INMAINTENANCE ] +then + echo "Already in maintenance mode" 1>&2 + exit 1 +fi + +case `< $ISRW` in + Master ) echo "This is a Master system. It cannot be set to maintenance mode" 1>&2 + exit 2 + ;; +esac + +echo "Setting maintenance mode" +echo inmaintenance > $INMAINTENANCE + +echo "Shutting down the PostgreSQL server" +pg_ctl_stop + +# the following loop is to allow iDNS to get a chance to mark this system as being down +echo "Sleeping for 20 seconds" +for i in `seq 20` +do + echo -n " $i" +done +echo " Done" + + + + diff --git a/pgaas/src/stage/opt/app/pgaas/bin/stop_maintenance_mode b/pgaas/src/stage/opt/app/pgaas/bin/stop_maintenance_mode new file mode 100644 index 0000000..fc582ba --- /dev/null +++ b/pgaas/src/stage/opt/app/pgaas/bin/stop_maintenance_mode @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this code except in compliance +# with the License. You may obtain a copy of the License +# at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +PATH=/opt/app/pgaas/bin:/opt/app/postgresql-prep/bin:$PATH + +INMAINTENANCE=/var/run/postgresql/inmaintenance + +if [ ! -f $INMAINTENANCE ] +then + echo "Not in maintenance mode" 1>&2 + exit 1 +fi + +echo "Restarting PostgreSQL server" +pg_ctl_start +echo "Returning server to normal rotation" +rm -f $INMAINTENANCE diff --git a/pgaas/src/stage/opt/app/pgaas/man/check_cluster.man b/pgaas/src/stage/opt/app/pgaas/man/check_cluster.man index dc6fcce..ea8f5e9 100644 --- a/pgaas/src/stage/opt/app/pgaas/man/check_cluster.man +++ b/pgaas/src/stage/opt/app/pgaas/man/check_cluster.man @@ -14,7 +14,10 @@ .SH NAME check_cluster - check the state of the cluster .SH SYNOPSIS -check_cluster [-v] [-l] [-t timeout] +check_cluster [-d file] [-v] [-l] [-t timeout] +.HP 20 +-d file +duplicate the status output to the given file .HP 20 -l do not check localhost first (and restarting the service if necessary) @@ -25,10 +28,13 @@ set how long to wait when accessing the servers -v verbose .SH DESCRIPTION -Loop through the nodes in the cluster, using pgwget to determine how many are masters, secondaries, or not up. +Loop through the nodes in the cluster, using pgwget to determine how many are +masters, secondaries, in maintenance, or not up. Complain about certain situations. If there are multiple masters, and this not the first master in the list, then: run pg_ctl_restart -prevent /ro from returning true +to prevent /ro from returning true +.PP +When \-d is used, the filename will have ".tmp" appended, writing the output to that temp filename, and then renaming the ".tmp" file to the given filename. diff --git a/pgaas/src/stage/opt/app/pgaas/man/makefile b/pgaas/src/stage/opt/app/pgaas/man/makefile index 9cd2090..92f6292 100644 --- a/pgaas/src/stage/opt/app/pgaas/man/makefile +++ b/pgaas/src/stage/opt/app/pgaas/man/makefile @@ -93,5 +93,9 @@ show_pg_stat_user_tables.1: show_pg_stat_user_tables.man DATE=$$(date +%Y-%m-%d); sed "1s/ {{DATE}} / $$DATE /" < $< | nroff -Tlp -man > $@ startpsql.1: startpsql.man DATE=$$(date +%Y-%m-%d); sed "1s/ {{DATE}} / $$DATE /" < $< | nroff -Tlp -man > $@ +start_maintenance_mode.1: start_maintenance_mode.man + DATE=$$(date +%Y-%m-%d); sed "1s/ {{DATE}} / $$DATE /" < $< | nroff -Tlp -man > $@ +stop_maintenance_mode.1: stop_maintenance_mode.man + DATE=$$(date +%Y-%m-%d); sed "1s/ {{DATE}} / $$DATE /" < $< | nroff -Tlp -man > $@ update_var_run_isrw.1: update_var_run_isrw.man DATE=$$(date +%Y-%m-%d); sed "1s/ {{DATE}} / $$DATE /" < $< | nroff -Tlp -man > $@ diff --git a/pgaas/src/stage/opt/app/pgaas/man/start_maintenance_mode.man b/pgaas/src/stage/opt/app/pgaas/man/start_maintenance_mode.man new file mode 100644 index 0000000..1906afa --- /dev/null +++ b/pgaas/src/stage/opt/app/pgaas/man/start_maintenance_mode.man @@ -0,0 +1,28 @@ +'\" Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +'\" +'\" Licensed under the Apache License, Version 2.0 (the "License"); +'\" you may not use this code except in compliance +'\" with the License. You may obtain a copy of the License +'\" at http://www.apache.org/licenses/LICENSE-2.0 +'\" +'\" Unless required by applicable law or agreed to in writing, software +'\" distributed under the License is distributed on an "AS IS" BASIS, +'\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +'\" implied. See the License for the specific language governing +'\" permissions and limitations under the License. +.TH start_maintenance_mode 1PG {{DATE}} OpenECOMP OpenECOMP +.SH NAME +start_maintenance_mode - convert this system to maintenance mode +.SH SYNOPSIS +start_maintenance_mode +.SH DESCRIPTION +Take this system out of the normal rotation and mark it as being in maintenance mode. +This command may only be used on a secondary system and not on a master system. +After this command is run, the PostgreSQL server will be brought down, but may be +restarted manually using the pg_ctl_start. +Note that the health checks will not regard this system as being in the normal +rotation until the command stop_maintenance_mode is executed, allowing the DBA to perform +whatever is needed to perform a backup or other maintenance routines, +such as copying the database files. +.SH FILES +/var/run/postgresql/inmaintenance diff --git a/pgaas/src/stage/opt/app/pgaas/man/stop_maintenance_mode.man b/pgaas/src/stage/opt/app/pgaas/man/stop_maintenance_mode.man new file mode 100644 index 0000000..bc97745 --- /dev/null +++ b/pgaas/src/stage/opt/app/pgaas/man/stop_maintenance_mode.man @@ -0,0 +1,24 @@ +'\" Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +'\" +'\" Licensed under the Apache License, Version 2.0 (the "License"); +'\" you may not use this code except in compliance +'\" with the License. You may obtain a copy of the License +'\" at http://www.apache.org/licenses/LICENSE-2.0 +'\" +'\" Unless required by applicable law or agreed to in writing, software +'\" distributed under the License is distributed on an "AS IS" BASIS, +'\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +'\" implied. See the License for the specific language governing +'\" permissions and limitations under the License. +.TH stop_maintenance_mode 1PG {{DATE}} OpenECOMP OpenECOMP +.SH NAME +stop_maintenance_mode - convert this system to maintenance mode +.SH SYNOPSIS +stop_maintenance_mode +.SH DESCRIPTION +Mark this system as no longer being in maintenance mode, and +return this system back to the normal rotation. +This command may only be used on a secondary system and not on a master system. +This command will also restart the PostgreSQL server. +.SH FILES +/var/run/postgresql/inmaintenance |