Init dcae.pgaas

Change-Id: Ieef6b600f4cbb0bf4ee3910c1bfc6b36773cd2d2
Signed-off-by: lj1412 <lji@research.att.com>

32 files changed, 1444 insertions, 0 deletions

diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/gen-repmgr-info b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/gen-repmgr-info
new file mode 100755
index 0000000..b210e58
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/gen-repmgr-info
@@ -0,0 +1,247 @@
+#!/usr/bin/perl
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this code except in compliance
+# with the License. You may obtain a copy of the License
+# at http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software  
+# distributed under the License is distributed on an "AS IS" BASIS,  
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  
+# implied. See the License for the specific language governing  
+# permissions and limitations under the License. 
+
+
+# NAME
+#	gen-repmgr-info - extract information about the system appropriate for use with repmgr
+#
+# DESCRIPTION
+#	gen-repmgr-info -n hosts options...
+#
+# for a list of hosts such as "uiopzxc5pepstg00.grant.example.com|uiopzxc5pepstg01.grant.example.com|uiopmtc6njpstg00.grant.example.com|uiopmtc6njpstg01.grant.example.com"
+# extract various pieces of information about the list. For example, generate a list of repmgr node numbers like this:
+# uiopzxc5pepstg00.grant.example.com 100
+# uiopzxc5pepstg01.grant.example.com 101
+# uiopmtc6njpstg00.grant.example.com 200
+# uiopmtc6njpstg01.grant.example.com 201
+
+use strict vars;
+
+use Getopt::Std;
+use Digest::SHA qw(sha256_hex);
+
+sub usage {
+  my $msg = shift;
+  print "$msg\n" if $msg;
+  print "Usage: $0 -n 'node|node|node|...' [-S] [-s site] [-L] [-l node] [-c node] [-C node] [-m] [-M node] [-e] [-v] [-p]\n";
+  print "-n list of nodes (FQDNs), |-separated\n";
+  print "-S show list of all sites and their node # values\n";
+  print "-s site\tshow the node # value for a given site\n";
+  print "-L show list of all nodes and their node # values\n";
+  print "-l node\tshow the node # value for a given node\n";
+  print "-C node\tshow the machine name that a given node should cascade from, or DEFAULT\n";
+  print "-c node\tshow the machine node # that a given node should cascade from, or DEFAULT\n";
+  print "-e node\tshow the list of nodes on the same site as the given node, |-separated\n";
+  print "-m\twhich system is the 'master'\n";
+  print "-M node\twhich system matches the given node, taking FQDN into consideration\n";
+  print "-v\tverbose\n";
+  print "-p\tprint the node names in sorted order\n";
+  exit 1;
+}
+
+
+my %optargs;
+getopts('C:c:e:E:Ll:M:mn:pPSs:v', \%optargs) or usage();
+my $verbose = $optargs{v};
+
+my $pgnodes = $optargs{n} or usage("-n is required");
+
+# generate the data about the nodes
+my @pgnodes = sort split(/[|]/, $pgnodes);
+
+# @sites contains the list of all site names.
+# For uiopzxc5pepstg01.grant.example.com, the sitename will be uiopzxc5pepstg.
+my @sites = genSites();
+my %pgnodesToSite = genPgnodeToSites();
+
+# The %siteValues contains 100, 200, etc for each site name
+# print "\nsites=" . join("\n", @sites);
+my %siteValues = genSiteValues();
+
+# The %pgnodeValues contains 100, 101, 200, 201, etc for each node name
+my %pgnodeValues = genPgnodeValues();
+# The %valuesToPgnode contains node names for each value
+my %valuesToPgnode = genValuesToPgnodes();
+
+if ($optargs{L}) {
+  for my $pgnode (@pgnodes) {
+    print "$pgnode $pgnodeValues{$pgnode}\n";
+  }
+}
+
+if ($optargs{S}) {
+  for my $site (@sites) {
+    print "$site $siteValues{$site}\n";
+  }
+}
+
+if ($optargs{s}) {
+  for my $site (@sites) {
+    print "$siteValues{$site}\n" if $site eq $optargs{s};
+  }
+}
+
+if ($optargs{l}) {
+  for my $pgnode (@pgnodes) {
+    print "$pgnodeValues{$pgnode}\n" if $pgnode eq $optargs{l};
+  }
+}
+
+if ($optargs{c}) {
+  my $pgnode = $optargs{c};
+  my $pgnodeValue = $pgnodeValues{$pgnode};
+  my $masterValue = int($pgnodeValue / 100) * 100;
+  if (($masterValue > 100) && (($masterValue % 100) > 0)) {
+    print "$masterValue\n";
+  } else {
+    print "DEFAULT\n";
+  }
+}
+
+if ($optargs{C}) {
+  my $pgnode = $optargs{C};
+  my $pgnodeValue = $pgnodeValues{$pgnode};
+  my $masterValue = int($pgnodeValue / 100) * 100;
+  # print "pgnode=$pgnode, pgnodeValue=$pgnodeValue, masterValue=$masterValue\n";
+  if (($pgnodeValue % 100) > 0) {
+    print "$valuesToPgnode{$masterValue}\n";
+  } else {
+    print "DEFAULT\n";
+  }
+}
+
+sub enodes {
+  my $pgnodeSearch = $optargs{e};
+  my $siteSearch = $pgnodesToSite{$pgnodeSearch};
+  my $ret = "";
+  # print "looking for $pgnodeSearch in $siteSearch\n";
+  my $sep = "";
+  for my $pgnode (@pgnodes) {
+    my $site = $pgnodesToSite{$pgnode};
+    # print "looking at $pgnode in $site\n";
+    if ($site eq $siteSearch) {
+      $ret .= "$sep$pgnode";
+      $sep = "|";
+    }
+  }
+  return $ret;
+}
+
+if ($optargs{e}) {
+  my $ret = enodes();
+  print "$ret\n";
+}
+
+if ($optargs{E}) {
+  print sha256_hex(enodes()) . "\n";
+}
+
+if ($optargs{m}) {
+    print "$pgnodes[0]\n";
+}
+
+if ($optargs{M}) {
+  my $node = $optargs{M};
+  if ($node =~ /[.]/) {
+    print "$node\n";
+  } else {
+    my $found;
+    for my $pgnode (@pgnodes) {
+      if ($pgnode =~ /^$node[.]/) {
+	print $node;
+	$found = 1;
+	last;
+      }
+    }
+  }
+}
+
+sub pnodes {
+  return join("|", @pgnodes);
+}
+
+if ($optargs{p}) {
+  print pnodes() . "\n";
+}
+
+if ($optargs{P}) {
+  print sha256_hex(pnodes()) . "\n";
+}
+
+# for a given node name uiopzxc5pepstg01.grant.example.com, the return uiopzxc5pepstg.
+sub nodeToSite {
+  my $site = shift;
+  $site =~ s/[.].*//;
+  $site =~ s/\d*$//;
+  return $site;
+}
+
+# from a list of nodes, generate the sorted list of sites
+sub genSites {
+  my %sites = ();
+  # print "pgnodes=" . join("\n", @pgnodes);
+  for my $pgnode (@pgnodes) {
+    my $site = nodeToSite($pgnode);
+    $sites{$site} = $site;
+  }
+  my @sites = sort keys %sites;
+  return @sites;
+}
+
+# from a list of nodes, generate a mapping from them to their sites
+sub genPgnodeToSites {
+  my %sites = ();
+  for my $pgnode (@pgnodes) {
+    $sites{$pgnode} = nodeToSite($pgnode);
+  }
+  return %sites;
+}
+
+# generate the 100, 200, etc for each site name
+sub genSiteValues {
+  my %siteValues;
+  for (my $i = 0; $i <= $#sites; $i++) {
+    $siteValues{$sites[$i]} = ($i+1) * 100;
+  }
+  # print "\nsiteValues=\n"; for my $site (@sites) { print "$site $siteValues{$site}\n"; }
+  return %siteValues;
+}
+
+sub genPgnodeValues {
+  my %pgnodeValues;
+  my $i = 0;
+  my $lastSite = '';
+  for my $pgnode (@pgnodes) {
+    my $thisSite = nodeToSite($pgnode);
+    if ($thisSite eq $lastSite) {
+      $i++;
+    } else {
+      $i = 0;
+    }
+    $lastSite = $thisSite;
+    $pgnodeValues{$pgnode} = $siteValues{$thisSite} + $i;
+  }
+  # print "\nnodeValues=\n"; for my $pgnode (@pgnodes) { print "$pgnode $pgnodeValues{$pgnode}\n"; }
+  return %pgnodeValues;
+}
+
+sub genValuesToPgnodes {
+  my %valuesToPgnode;
+  for my $pgnode (keys %pgnodeValues) {
+    my $value = $pgnodeValues{$pgnode};
+    $valuesToPgnode{$value} = $pgnode;
+  }
+  return %valuesToPgnode;
+}
+
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/iDNS-responder.py b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/iDNS-responder.py
new file mode 100755
index 0000000..b6c5174
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/iDNS-responder.py
@@ -0,0 +1,1025 @@
+#!/usr/bin/env python3
+# -*- indent-tabs-mode: nil -*-
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this code except in compliance
+# with the License. You may obtain a copy of the License
+# at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+
+import http.server
+import time, os, sys, re, subprocess, traceback, html, base64
+import psycopg2
+# TODO - move lots of code to a common library to share with other python modules
+# sys.path.append("/opt/app/postgres-prep/lib")
+# import dbtools
+
+getLogDict = { }
+def openLogFile(fname):
+    """
+    Open a log file for append and remember the file descriptor.
+    Remember its inode/dev pair.
+    If either changes, reopen it.
+    """
+    reopen = False
+    try:
+        curstat = os.stat(fname)
+    except:
+        reopen = True
+    global getLogDict
+    # print("top: reopen(%s)=%s" % (fname, reopen))
+    if not reopen and getLogDict.get(fname):
+        # print("found getLogDict.get(" + fname + ")")
+        d = getLogDict[fname]
+        fd = d["fd"] if "fd" in d else None
+        oldstat = d["stat"] if "stat" in d else None
+        if fd is None:
+            reopen = True
+        elif oldstat is None:
+            reopen = True
+        elif oldstat.st_ino != curstat.st_ino or oldstat.st_dev != curstat.st_dev:
+            reopen = True
+    if reopen or not getLogDict.get(fname):
+        # print("closing old fd")
+        oldd = getLogDict.get(fname)
+        if oldd is not None:
+            oldfd = oldd.get("fd")
+            if oldfd is not None:
+                oldfd.close()
+        # print("reopening " + fname)
+        fd = open(fname, "a")
+        st = os.stat(fname)
+        getLogDict[fname] = { "fd": fd, "stat": st }
+    return getLogDict[fname]["fd"]
+
+debugOn = False
+testOn = False
+
+if len(sys.argv) > 1:
+    debugOn = True
+    testOn = True
+
+else:
+    sys.stderr = openLogFile("/opt/app/log/postgresql/idns/error.log")
+
+HOST_NAME = os.popen("hostname -f").readlines()[0].strip()
+PORT_NUMBER = 8000
+
+validPerDbTables = [ "pg_tables", "pg_indexes", "pg_views" ]
+topButton = "&nbsp;<font size='1'><a href='#'>^</a></font>"
+
+def traceMsg(msg):
+    """ print a trace message. By default, this goes to trace.out """
+    file = sys.stderr if testOn else openLogFile("/opt/app/log/postgresql/idns/trace.log")
+    print(time.asctime(), msg, file=file)
+    file.flush()
+
+def errTrace(msg):
+    """ print an error message. By default, sys.stderr is rerouted to error.log """
+    file = sys.stderr if testOn else openLogFile("/opt/app/log/postgresql/idns/error.log")
+    sys.stderr = file
+    print(time.asctime(), msg, file=file)
+    file.flush()
+
+def debugTrace(msg):
+    """ print a debug message. By default, this goes to debug.log """
+    if debugOn:
+        file = sys.stderr if testOn else openLogFile("/opt/app/log/postgresql/idns/debug.log")
+        print(time.asctime(), msg, file=file)
+        file.flush()
+
+def readFile(file, defStr = None, mode = "r"):
+    """ read a file and return its contents """
+    ret = defStr
+    try:
+        with open(file, mode) as f:
+            ret = f.read()
+    except Exception as e:
+        if defStr is not None:
+            ret = defStr
+            pass
+        else:
+            raise e
+    return ret
+
+def readFileBinary(file, defStr = None):
+    return readFile(file, defStr = defStr, mode = "rb")
+
+def readFileHtml(file, defStr = None):
+    """ read a file and return its contents, escaping anything important to HTML """
+    return html.escape(readFile(file, defStr))
+
+def readPipe(cmd, ignoreError = False):
+    """ read a pipe and return its contents """
+    ret = ""
+    try:
+        with os.popen(cmd) as p:
+            ret = p.read()
+    except Exception as e:
+        if ignoreError:
+            pass
+        else:
+            raise e
+    return ret
+
+def readPipeHtml(file, defStr = None):
+    """ read a pipe and return its contents, escaping anything important to HTML """
+    return html.escape(readPipe(file, defStr))
+
+def readFileOrGz(file, defStr = None):
+    """ read a file and return its contents. If the file ends in .gz, use gunzip on it """
+    if file.endswith(".gz"):
+        return readPipe("gunzip < '" + file + "'", defStr)
+    else:
+        return readFile(file, defStr)
+
+def readFileOrGzHtml(file, defStr = None):
+    """ read a file and return its contents, escaping anything important to HTML. If the file ends in .gz, use gunzip on it """
+    return html.escape(readFileOrGz(file, defStr))
+
+def getCdfPropValue(nm, encrypted=False, cfg="/opt/app/cdf/lib/cdf.cfg", dflt=None):
+    """
+    Return a value from the configuration file /opt/app/cdf/lib/cdf.cfg
+    """
+    return getPropValue(nm=nm, encrypted=encrypted, cfg=cfg, dflt=dflt)
+
+def getPgaasPropValue(nm, encrypted=False, cfg="/opt/app/pgaas/lib/pgaas.cfg", dflt=None):
+    """
+    Return a value from the configuration file /opt/app/pgaas/lib/pgaas.cfg
+    """
+    return getPropValue(nm=nm, encrypted=encrypted, cfg=cfg, dflt=dflt)
+
+getPropDict = { }
+
+def getPropValue(nm, encrypted=False, cfg=None, dflt=None):
+    """
+    Return a value from the specified configuration file 
+    """
+    if cfg is None:
+        return None
+    global getPropDict
+    if getPropDict.get(cfg):
+        savedDate = getPropDict[cfg]
+        debugTrace("getPropValue: savedDate[" + cfg + "]=" + str(savedDate))
+        cfgDate = os.path.getmtime(cfg)
+        debugTrace("getPropValue: cfgDate=" + str(cfgDate))
+        if float(savedDate) >= float(cfgDate): # cfg has not changed
+            val = getPropDict.get(cfg + ":" + nm)
+            debugTrace("getPropValue: val=" + str(val))
+            if val is not None:
+                debugTrace("getPropValue: getPropValue(saved) => '%s'" % str(val))
+                return val
+        else: # clear out any previously saved keys
+            cfgcolon = cfg + ":"
+            for k in list(getPropDict.keys()):
+                if re.match(cfgcolon, k):
+                    del getPropDict[k]
+    getPropValueProgram = '/opt/app/cdf/bin/getpropvalue'
+    if encrypted:
+        cmd = [getPropValueProgram, "-f", cfg, "-x", "-n", nm]
+    else:
+        cmd = [getPropValueProgram, "-f", cfg, "-n", nm]
+    debugTrace("getPropValue: cmd=%s" % str(cmd))
+
+    try:
+        with subprocess.Popen(cmd,shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) as p:
+            (origString, stderrString) = p.communicate()
+    except Exception as e:
+        traceback.print_exc()
+        errTrace("Error decoding string because {0}".format(e))
+        return None
+    else:
+        if stderrString:
+            if not re.search("Configuration property .* must be defined", stderrString.decode('utf-8')): # and dflt is not None:
+                errTrace("Error decoding string because: {0} ".format(stderrString))
+            return dflt
+        else:
+            debugTrace("getPropValue() => '%s'" % str(origString))
+            getPropDict[cfg] = os.path.getmtime(cfg)
+            val = origString.decode('utf-8').rstrip('\n')
+            debugTrace("getPropValue() => '%s'" % val)
+            getPropDict[cfg + ":" + nm] = val
+            return val
+
+def checkFileAge(full_path,number_of_days):
+    """
+    return True if the file is >= number_of_days old from right now
+    """
+    time_n_days_ago = time.time() - (number_of_days * 24 * 60 * 60)
+    stat = os.stat(full_path)
+    return time_n_days_ago >= stat.st_mtime
+
+def jumpTable(prefix, *args):
+    """
+    Return a string consisting of a series of <a href='#prefix-xxx'>xxx</a>.
+    Include <font size='1'></font> around all of it.
+    """
+    header = "<font size='1'>"
+    sep = ""
+    for table in args:
+        header = header + sep + "<a href='#" + prefix + "-" + table + "'>" + table + "</a> "
+        sep = " | "
+    header = header + "</font>"
+    return header
+
+def addFilenameHrefs(prefix, str):
+    """
+    for each line in a list of filenames, change the last two elements of the path to an anchor.
+    """
+    ret = ""
+    for line in str.splitlines():
+        line = re.sub("/([^/]+)/([^/]+)$", '/\g<1>' + "/<a href='" + prefix + '\g<1>/\g<2>' + "'>" + '\g<2>' + "</a>", line)
+        ret = ret + line + "\n"
+    return ret
+
+def ifEmpty(str, defStr):
+    """ if a string is empty, return the defStr in its place """
+    if str is None or str == "":
+        str = defStr
+    return str
+
+def isExe(fname):
+    """ check if a path exists and is executable """
+    return os.path.exists(fname) and os.access(fname, os.X_OK)
+
+class MyHandler(http.server.BaseHTTPRequestHandler):
+
+    def isServerUp(self):
+        """
+        Check if the postgres server is up and running by calling pg_ctl and
+        looking for "server is running" (or "no server running").
+        Then call ps -fu postgres and make sure we're not waiting on a master:
+        postgres  20815  20812  0 15:52 ?        00:00:00 postgres: startup process   waiting for 000000010000000000000001
+        """
+        PGCTLPATH1 = "/usr/lib/postgresql/9.5/bin/pg_ctl"
+        PGCTLPATH2 = "/opt/app/postgresql-9.5.2/bin/pg_ctl"
+        if isExe(PGCTLPATH1):
+            statusLines = readPipe(PGCTLPATH1 + " status -D /dbroot/pgdata/main/")
+        else:
+            statusLines = readPipe(PGCTLPATH2 + " status -D /dbroot/pgdata/main/")
+        debugTrace("isServerUp(): statusLines = %s" % statusLines)
+        psLines = readPipe("ps -fu postgres")
+        debugTrace("isServerUp(): ps -fu postgres = %s" % psLines)
+        ret = len(statusLines) > 0 and re.search("server is running", statusLines, re.MULTILINE) and not re.search("startup process\\s+waiting", psLines, re.MULTILINE)
+        debugTrace("isServerUp(): returning = %s" % ret)
+        return ret
+
+    def isRepmgrdUp(self):
+        """
+        Check if the repmgrd server is up and running by calling "pgrep repmgrd" and
+        looking for a process id.
+        """
+        statusLines = readPipe("pgrep repmgrd")
+        debugTrace("isServerUp(): statusLines = %s" % statusLines)
+        ret = len(statusLines) > 0 and re.search("[0-9]+", statusLines, re.MULTILINE) != None
+        debugTrace("isServerUp(): returning = %s" % ret)
+        return ret
+
+    def isMaster(self):
+        """
+        Check if the postgresql server is a master by asking the server if it is in recovery (meaning not a master)
+        """
+        ret = None
+        con = None
+        try:
+            pwd = getCdfPropValue("postgres", True)
+            # debugTrace("using pwd=%s" % pwd)
+            con = psycopg2.connect(database = "postgres", user="postgres", password=pwd, host= HOST_NAME)
+            str = dbGetFirstRowOneValue(con, "select pg_is_in_recovery()")
+            debugTrace("pg_is_in_recovery() <= %s" % str)
+            ret = not str
+
+        except psycopg2.DatabaseError as e:
+            errTrace('Database Error %s' % e)
+
+        except Exception as e:
+            traceback.print_exc()
+            errTrace(str(e))
+
+        finally:
+            if con is not None:
+                con.close()
+
+        debugTrace("isMaster(): returning = %s" % ret)
+        return ret
+
+    def hasRepmgr(self):
+        """
+        Check if the postgresql server is a master by asking the server if it is in recovery (meaning not a master)
+        """
+        ret = None
+        con = None
+        try:
+            pwd = getCdfPropValue("postgres", True)
+            # debugTrace("using pwd=%s" % pwd)
+            con = psycopg2.connect(database = "postgres", user="postgres", password=pwd, host= HOST_NAME)
+            str = dbGetFirstRowOneValue(con, "select * from pg_database where datname = 'repmgr'")
+            debugTrace("repmgr database check() <= %s" % str)
+            ret = str
+
+        except psycopg2.DatabaseError as e:
+            errTrace('Database Error %s' % e)
+
+        except Exception as e:
+            traceback.print_exc()
+            errTrace(str(e))
+
+        finally:
+            if con is not None:
+                con.close()
+
+        debugTrace("isMaster(): returning = %s" % ret)
+        return ret
+
+    def isValidPgHost(self, host):
+        """
+        Check a hostname against the list of nodes stored in the pgnodes CDF configuration file.
+        """
+        pgnodes = getCdfPropValue("pgnodes", "").split('|')
+        ret = host in pgnodes
+        debugTrace("isValidPgHost(): looking for host='%s' in pgnodes='%s' => %s" % (host, str(pgnodes), ret))
+        return ret
+
+    def checkAuth(self):
+        """
+        HTTP/1.1 401 Unauthorized
+        Date: Mon, 04 Feb 2014 16:50:53 GMT
+        WWW-Authenticate: Basic realm="WallyWorld"
+
+        Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==
+        """
+        pswd = getCdfPropValue("wgetpswd", True)
+        b64pswd = base64.b64encode(("pgaas:" + pswd).encode("ascii"))
+        basicPlusPswd = "Basic %s" % b64pswd.decode("ascii")
+
+        if self.headers['Authorization'] == None:
+            return False
+        elif self.headers['Authorization'] == basicPlusPswd:
+            return True
+        else:
+            return False
+
+    def pgStatus(self, *pgargs):
+        """ return a table(s), using the system database of postgres """
+        return self.pgStatusDBx("postgres", *pgargs)
+
+    def pgStatusDB(self, DB, *pgargs):
+        """ return a table(s), using the given database """
+        return self.pgStatusDBx(DB, *pgargs)
+
+    def pgStatusDBx(self, DB, *pgargs):
+        """ return a table(s), using the given database """
+        debugTrace("pgStatusDBx(DB=" + DB + ")")
+        con = None
+        ret = None
+        try:
+            con = psycopg2.connect(database = DB, user="postgres", password=getCdfPropValue("postgres", True), host= HOST_NAME)
+            ret = getTableHtmls(con, DB, pgargs)
+
+        except psycopg2.DatabaseError as e:
+            errTrace('Database Error %s' % e)
+
+        except Exception as e:
+            traceback.print_exc()
+            errTrace(str(e))
+
+        finally:
+            if con is not None:
+                con.close()
+
+        return ret
+
+    def do_HEAD(self):
+        """Respond to a HEAD request."""
+        self.doHEADandGET(False)
+
+    def do_GET(self):
+        """Respond to a GET request."""
+        self.doHEADandGET(True)
+
+    def doHEADandGET(self, sendMsg):
+        resp = 400
+        msg = ""
+        sendBinary = False
+        contentType = "text/plain"
+        global debugOn
+
+        if self.path == "/statusall":
+            self.path = "/all/status/pgstatus"
+        elif self.path == "/pgstatusall":
+            self.path = "/pgstatus"
+
+        if self.path == '/ro':
+            if os.path.isfile("/var/run/postgresql/force-ro-off"):
+                isrw = "FORCE-RO-OFF"
+            elif os.path.isfile("/var/run/postgresql/force-ro-on"):
+                isrw = "Secondary"
+            else:
+                isrw = readFile("/var/run/postgresql/isrw", "n/a")
+            debugTrace("/ro: isrw returns %s" % isrw)
+            if re.match("Secondary", isrw) or re.match("Master", isrw):
+                resp = 200
+                msg = "server is up"
+            else:
+                msg = "server is not up " + isrw
+                errTrace("/ro: isrw returns %s" % isrw)
+            
+        elif self.path == '/rw':
+            isrw = readFile("/var/run/postgresql/isrw", "n/a")
+            debugTrace("/rw: isrw returns %s" % isrw)
+            if re.match("Master", isrw):
+                resp = 200
+                msg = "master server is up"
+            elif re.match("Secondary", isrw):
+                msg = "non-master server is up"
+            else:
+                msg = "server is not up " + isrw
+                errTrace("/ro: isrw returns %s" % isrw)
+
+        elif self.path == '/isrw':
+            isrw = readFile("/var/run/postgresql/isrw", "n/a")
+            debugTrace("/rw: isrw returns %s" % isrw)
+            resp = 200
+            msg = isrw
+
+        elif not self.checkAuth():
+            resp = 401
+            msg = "not authenticated"
+
+        elif self.path == '/ismaster':
+            masterYes = self.isMaster()
+            msg = ""
+            if masterYes:
+                resp = 200
+                msg = "master server"
+            else:
+                msg = "non-master server"
+
+        elif self.path == '/issecondary':
+            masterYes = self.isMaster()
+            msg = ""
+            if not masterYes:
+                resp = 200
+                msg = "secondary server"
+            else:
+                msg = "non-secondary server"
+
+        elif self.path == '/getpubkey':
+            try:
+                resp = 200
+                msg = readFile("/home/postgres/.ssh/id_rsa.pub")
+            except:
+                traceback.print_exc()
+                resp = 404
+                msg = "key does not exist"
+
+        elif re.match("/getssh/", self.path):
+            # getssh/hostname - push ssh pub/private keys across
+            host = re.sub("^/getssh/", "", self.path)
+            debugTrace("#1: /getssh/ host='%s'" % host)
+            host = re.sub("[^a-zA-Z0-9_.-]", "", host)
+            debugTrace("#2: /getssh/ host='%s'" % host)
+            if self.isValidPgHost(host):
+                p = readPipe("scp -o StrictHostKeyChecking=no ~postgres/.ssh/id_rsa* " + host + ":.ssh/ 2>&1")
+                debugTrace("#3: /getssh/ to '%s' returns '%s'" % (host, p))
+                msg = "OK " + p
+                resp = 200
+            else:
+                msg = "NOT OK INVALID HOST"
+                resp = 404
+
+        elif re.match("/getcdf/", self.path):
+            # getcdf/hostname - push cdf.cfg file across
+            fi = "/opt/app/cdf/lib/cdf.cfg"
+            # make sure that the file exists and contains the encrypted postgres password
+            if re.search("postgres.x", readFile(fi, "n/a")) and re.search("repmgr.x", readFile(fi, "n/a")):
+                host = re.sub("^/getcdf/", "", self.path)
+                debugTrace("#1: /getcdf/ host='%s'" % host)
+                host = re.sub("[^a-zA-Z0-9_.-]", "", host)
+                debugTrace("#2: /getcdf/ host='%s'" % host)
+                if self.isValidPgHost(host):
+                    p = readPipe("scp -o StrictHostKeyChecking=no " + fi + " " + host + ":/opt/app/cdf/lib/cdf.cfg 2>&1")
+                    debugTrace("#3: /getcdf/ to '%s' returns '%s'" % (host, p))
+                    msg = "OK " + p
+                    resp = 200
+                else:
+                    msg = "NOT OK INVALID HOST"
+                    resp = 404
+            else:
+                msg = "NOT OK YET"
+                resp = 404
+
+        elif self.path == '/hasrepmgr':
+            repmgrYes = self.hasRepmgr()
+            msg = ""
+            if repmgrYes:
+                resp = 200
+                msg = "OK"
+            else:
+                msg = "NOT OK YET"
+
+        elif self.path == '/status':
+            resp = 200
+            contentType = "text/html"
+            isServerUp = self.isServerUp()
+            isRepmgrdUp = self.isRepmgrdUp()
+            isMaster = self.isMaster()
+            color = "green" if (isServerUp and isRepmgrdUp) else "yellow" if (isServerUp or isRepmgrdUp) else "red"
+            dashed = "solid" if isMaster else "dashed"
+            jump = jumpTable("status", "ps", "repmgr", "df", "uptime", "loadavg", "cpuinfo", "meminfo", "pgaas-failures", "pgaas-inst-report", "nslookup", "ip-addr-show", "who-br")
+
+            msg = """<table style='border: 10px %s %s' width='100%%'><tr><td>
+                <b>isServerUp</b> %s
+                <b>isRepmgrdUp</b> %s
+                <b>isMaster</b> %s
+                <b>isrw</b> %s %s\n<br/>
+                %s
+                <h2><a name='status-ps'>ps</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-repmgr'>repmgr cluster show</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-df'>df</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-uptime'>uptime</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2>/proc/uptime%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-loadavg'>loadavg</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-cpuinfo'>cpuinfo</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-meminfo'>meminfo</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-pgaas-failures'>pgaas-failures</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-pgaas-inst-report'>pgaas.inst.report</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-nslookup'>nslookup</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-ip-addr-show'>ip addr</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                <h2><a name='status-who-br'>who -br</a>%s</h2>\n<pre>\n%s\n</pre>\n
+                </td></tr></table>""" % (color, dashed, isServerUp, isRepmgrdUp, isMaster,
+                readFileHtml("/var/run/postgresql/isrw", "n/a"),
+                readPipeHtml("hostname -f"), jump,
+                topButton, readPipeHtml("ps -fu postgres"),
+                topButton, readPipeHtml("/opt/app/pgaas/bin/repmgrc cluster show"),
+                topButton, readPipeHtml("df -h"),
+                topButton, readPipeHtml("uptime", defStr="n/a"),
+                topButton, readFileHtml("/proc/uptime", defStr="n/a"),
+                topButton, readFileHtml("/proc/loadavg", defStr="n/a"),
+                topButton, readFileHtml("/proc/cpuinfo", defStr="n/a"),
+                topButton, readFileHtml("/proc/meminfo", defStr="n/a"),
+                topButton, readFileHtml("/tmp/pgaas-failures", defStr="n/a"),
+                topButton, readFileHtml("/tmp/pgaas.inst.report", defStr="n/a"),
+                topButton, readPipeHtml("nslookup $(hostname -f)", defStr="n/a"),
+                topButton, readPipeHtml("ip addr show", defStr="n/a"),
+                topButton, readPipeHtml("who -br", defStr="n/a"))
+
+        elif self.path == '/stoplight':
+            isServerUp = self.isServerUp()
+            isRepmgrdUp = self.isRepmgrdUp()
+            isMaster = self.isMaster()
+            color = "green" if (isServerUp and isRepmgrdUp) else "yellow" if (isServerUp or isRepmgrdUp) else "red"
+            masterSecondary = "master" if isMaster else "secondary"
+            sendBinary = True
+            contentType = "image/gif"
+            msg = readFileBinary("/opt/app/postgresql-prep/lib/stoplight-" + masterSecondary + "-" + color + ".gif", "")
+
+        elif re.match("/perdb-", self.path):
+            # /perdb-
+            rest = re.sub("^/perdb-", "", self.path)
+            debugTrace("#1: /perdb- others='%s'" % rest)
+            rest = re.sub("[^a-zA-Z0-9_./-]", "", rest)
+            debugTrace("#2: /perdb- rest='%s'" % rest)
+            pgothers = [ x for x in rest.split('-') if x in validPerDbTables ]
+            resp = 200
+            contentType = "text/html"
+            con = None
+            try:
+                pwd = getCdfPropValue("postgres", True)
+                con = psycopg2.connect(database = "postgres", user="postgres", password=pwd, host= HOST_NAME)
+                databases = dbGetFirstColumn(con, "select datname from pg_database")
+                debugTrace("after select datname from pg_database")
+                databases[:] = [DB for DB in databases if not re.match("template[0-9]", DB)]
+                msg = msg + jumpTable("db", *databases) + "<br/>"
+                for DB in databases:
+                    debugTrace("looking at DB=" + DB)
+                    msg = msg + "<h1><a name='db-" + DB + "'>" + DB + "</a>" + topButton + "</h1>\n"
+                    msg = msg + jumpTable(DB + "-table", *pgothers)
+                    msg = msg + self.pgStatusDB(DB, *pgothers)
+
+            except psycopg2.DatabaseError as e:
+                errTrace('Database Error %s' % e)
+                msg = "DB is down"
+
+            except Exception as e:
+                traceback.print_exc()
+                errTrace(str(e))
+
+            finally:
+                if con is not None:
+                    con.close()
+
+        elif self.path == '/pgstatus':
+            tables = [ "pg_stat_activity", "pg_stat_archiver", "pg_stat_bgwriter", "pg_stat_database", "pg_stat_database_conflicts", "pg_stat_user_tables", "pg_stat_user_indexes", "pg_statio_user_tables", "pg_statio_user_indexes", "pg_statio_user_sequences", "pg_roles", "pg_database", "pg_tables", "pg_namespace", "pg_roles", "pg_group" ]
+            header = jumpTable("postgres-table", *tables)
+            msg = self.pgStatus(*tables)
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+                msg = header + msg
+
+        elif self.path == '/pg_stat_activity':
+            msg = self.pgStatus("pg_stat_activity")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_stat_archiver':
+            msg = self.pgStatus("pg_stat_archiver")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_stat_bgwriter':
+            msg = self.pgStatus("pg_stat_bgwriter")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_stat_database':
+            msg = self.pgStatus("pg_stat_database")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_stat_database_conflicts':
+            msg = self.pgStatus("pg_stat_database_conflicts")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_stat_user_tables':
+            msg = self.pgStatus("pg_stat_user_tables")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_stat_user_indexes':
+            msg = self.pgStatus("pg_stat_user_indexes")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_statio_user_tables':
+            msg = self.pgStatus("pg_statio_user_tables")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_statio_user_indexes':
+            msg = self.pgStatus("pg_statio_user_indexes")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_statio_user_sequences':
+            msg = self.pgStatus("pg_statio_user_sequences")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_roles':
+            msg = self.pgStatus("pg_roles")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_database':
+            msg = self.pgStatus("pg_database")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_tables':
+            msg = self.pgStatus("pg_tables")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_namespace':
+            msg = self.pgStatus("pg_namespace")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif self.path == '/pg_group':
+            msg = self.pgStatus("pg_group")
+            if msg is not None:
+                contentType = "text/html"
+                resp = 200
+
+        elif re.match("/all/", self.path) or re.match("/small/", self.path):
+            if re.match("/small/", self.path):
+                height = 40
+            else:
+                height = 400
+            # /all/others
+            rest = re.sub("^/all/", "", self.path)
+            rest = re.sub("^/small/", "", self.path)
+            rest = re.sub("[^a-zA-Z0-9_./-]", "", rest)
+            debugTrace("/all/ rest='%s'" % rest)
+            others = rest.split('/')
+            try:
+                resp = 200
+                contentType = "text/html"
+                pgnodes = getCdfPropValue("pgnodes", "").split('|')
+                msg = msg + jumpTable("node", *pgnodes)
+                for node in pgnodes:
+                    hnode = html.escape(node)
+                    msg = msg + "<h2><a name='node-" + hnode + "'>" + hnode + "</a>" + topButton + "</h2>\n"
+                    msg = msg + jumpTable(hnode + "-other", *others)
+                    for other in others:
+                        msg = msg + "<h3><a name='" + hnode + "-other-" + other + "'>" + other + "</a>" + topButton + "</h3>\n"
+                        msg = msg + "<iframe src='http://" + hnode + ":" + str(PORT_NUMBER) + "/" + other + "'      frameborder='1' scrolling='yes' height='" + str(height) + "' width='1200'></iframe>\n"
+            except Exception as e:
+                traceback.print_exc()
+                errTrace(str(e))
+
+
+        elif self.path == '/debugon':
+            msg = "ON"
+            resp = 200
+            debugOn = True
+
+        elif self.path == '/debugoff':
+            msg = "OFF"
+            resp = 200
+            debugOn = False
+
+        elif self.path == '/log' or  self.path == '/log/':
+            msg = "<h2>%s</h2><pre>\n%s\n</pre>" % (self.path, addFilenameHrefs("/log/", readPipeHtml("ls -l /opt/app/log/postgresql/*/*")))
+            resp = 200
+            contentType = "text/html"
+
+        elif self.path == '/mlog' or  self.path == '/mlog/':
+            # /opt/app/dcae-controller-service-common-vm-manager/logs
+            msg = "<h2>%s</h2><pre>\n%s\n</pre>" % (self.path, addFilenameHrefs("/mlog/", readPipeHtml("ls -l /opt/app/dcae-controller-service-common-vm-manager/logs/*")))
+            resp = 200
+            contentType = "text/html"
+
+        elif self.path == '/tmp' or self.path == '/tmp/':
+            msg = "<h2>%s</h2><pre>\n%s\n</pre>" % (self.path, addFilenameHrefs("/tmp/", readPipeHtml("ls -l /tmp/*")))
+            resp = 200
+            contentType = "text/html"
+
+        elif re.match("/log/", self.path):
+            # /log/dir/path
+            rest = re.sub("^/log/", "", self.path)
+            debugTrace("#1: /log/ others='%s'" % rest)
+            rest = re.sub("[^a-zA-Z0-9_./-]", "", rest)
+            rest = re.sub("/[.][.]/", "", rest)
+            debugTrace("#2: /log/ rest='%s'" % rest)
+            msg = "<h2>%s</h2><pre>\n%s\n</pre>" % (rest, ifEmpty(readFileOrGzHtml("/opt/app/log/postgresql/" + rest, "n/a"), "<i>empty</i>"))
+            resp = 200
+            contentType = "text/html"
+
+        elif re.match("/mlog/", self.path):
+            # /log/dir/path
+            rest = re.sub("^/mlog/", "", self.path)
+            debugTrace("#1: /mlog/ others='%s'" % rest)
+            rest = re.sub("[^a-zA-Z0-9_./-]", "", rest)
+            rest = re.sub("/[.][.]/", "", rest)
+            rest = re.sub("^logs/", "", rest)
+            debugTrace("#2: /mlog/ rest='%s'" % rest)
+            msg = "<h2>%s</h2><pre>\n%s\n</pre>" % (rest, ifEmpty(readFileOrGzHtml("/opt/app/dcae-controller-service-common-vm-manager/logs/" + rest, "n/a"), "<i>empty</i>"))
+            resp = 200
+            contentType = "text/html"
+
+        elif re.match("/tmp/", self.path):
+            # /log/dir/path
+            rest = re.sub("^/tmp/", "", self.path)
+            debugTrace("#1: /tmp/ others='%s'" % rest)
+            rest = re.sub("[^a-zA-Z0-9_./-]", "", rest)
+            rest = re.sub("/[.][.]/", "", rest)
+            rest = re.sub("^tmp/", "", rest)
+            debugTrace("#2: /tmp/ rest='%s'" % rest)
+            msg = "<h2>%s</h2><pre>\n%s\n</pre>" % (rest, ifEmpty(readFileOrGzHtml("/tmp/" + rest, "n/a"), "<i>empty</i>"))
+            resp = 200
+            contentType = "text/html"
+
+        elif self.path == '/oldro':
+            serverYes = self.isServerUp()
+            if serverYes:
+                resp = 200
+                msg = "server is up"
+            else:
+                msg = "server is not up"
+
+        elif self.path == '/oldrw':
+            serverYes = self.isServerUp()
+            masterYes = self.isMaster()
+            msg = ""
+            if serverYes:
+                if masterYes:
+                    resp = 200
+                    msg = "master server is up"
+                elif masterYes is not None:
+                    msg = "non-master server is up"
+                else:
+                    msg = "master status is up, but not answering"
+            else:
+                if masterYes:
+                    msg = "status is down, but responded as master server"
+                elif masterYes is not None:
+                    msg = "status is down, but responded as non-master"
+                else:
+                    msg = "status is down, server is not up"
+
+        elif self.path == "/help":
+            resp = 200
+            contentType = "text/html"
+            msg = """<pre>
+                <a href='/statusall'>statusall</a> == all/status/pgstatus
+                <a href='/ro'>ro</a> == iDNS readonly
+                <a href='/rw'>rw</a> == iDNS readwrite
+                <a href='/isrw'>isrw</a> == /var/run/postgresql/isrw
+                <a href='/ismaster'>ismaster</a> == is master
+                <a href='/issecondary'>issecondary</a> == is secondary
+                <a href='/getpubkey'>getpubkey</a> == retrieve public key
+                <a href='/hasrepmgr'>hasrepmgr</a> == repmgr id and database are set up
+                <a href='/status'>status</a> == lots of info
+                <a href='/perdb-pg_tables-pg_indexes-pg_views'>perdb</a>-{<a href='/perdb-pg_tables'>pg_tables</a>-<a href='/perdb-pg_indexes'>pg_indexes</a>-<a href='/perdb-pg_views'>pg_views</a>} == info per DB
+                <a href='/pgstatus'>pgstatus</a> == lots of database info
+                <a href='/pg_stat_activity'>pg_stat_activity</a>, <a href='/pg_stat_archiver'>pg_stat_archiver</a>, <a href='/pg_stat_bgwriter'>pg_stat_bgwriter</a>,
+                <a href='/pg_stat_database'>pg_stat_database</a>, <a href='/pg_stat_database_conflicts'>pg_stat_database_conflicts</a>, <a href='/pg_stat_user_tables'>pg_stat_user_tables</a>,
+                <a href='/pg_stat_user_indexes'>pg_stat_user_indexes</a>, <a href='/pg_statio_user_tables'>pg_statio_user_tables</a>, <a href='/pg_statio_user_indexes'>pg_statio_user_indexes</a>,
+                <a href='/pg_statio_user_sequences'>pg_statio_user_sequences</a>,
+                <a href='/pg_roles'>pg_roles</a>, <a href='/pg_database'>pg_database</a>,
+                <a href='/pg_tables'>pg_tables</a>, <a href='/pg_namespace'>pg_namespace</a>,
+                <a href='/pg_group'>pg_group</a>,
+                <a href='/swmstatus'>swm proc_out files</a>
+                <a href='/log'>log</a> == ls -l logs
+                log/foo == log foo
+                <a href='/mlog'>mlog</a> == ls -l manager logs
+                mlog/foo == mlog foo
+                <a href='/tmp'>tmp</a> == ls -l /tmp
+                </pre>"""
+        else:
+            resp = 404
+            msg = "path does not exist"
+
+        # TODO == encode msg when binary
+        if sendBinary:
+            debugTrace("%s: Returning %d/%d/%s" % (self.path, resp, len(msg), "--binary--"))
+        else:
+            debugTrace("%s: Returning %d/%d/%s" % (self.path, resp, len(msg), msg))
+        traceMsg("- %s - \"%s %s %s\" %d %d" % (self.client_address[0], self.command, self.path, self.request_version, resp, len(msg)))
+        self.send_response(resp)
+        if resp == 401:
+            self.send_header('WWW-Authenticate', 'Basic realm="PGaaS"')
+        self.send_header("Content-type", contentType)
+        self.end_headers()
+        if sendMsg:
+            if msg is None:
+                msg = ""
+            if sendBinary:
+                self.wfile.write(msg)
+            else:
+                self.wfile.write((msg + "\n").encode("utf-8"))
+        sys.stderr.flush()
+
+"""
+database utility functions
+"""
+
+# def dbGetMap(con, cmd, args=[], skipTrace=False):
+# def dbGetOneRowMap(con, cmd, args=[], skipTrace=False):
+
+def dbGetFirstRowOneValue(con, cmd, args=[], skipTrace=False):
+    """
+    Do a select and return a single value from the first row
+    """
+    row = dbGetFirstRow(con, cmd, args, skipTrace)
+    debugTrace("row=" + str(row))
+    if row is not None and len(row) > 0:
+        return row[0]
+    return None
+
+def dbGetFirstRow(con, cmd, args=[], skipTrace=False):
+    """
+    Do a select and return the values from the first row
+    """
+    cursor = dbExecute(con, cmd, args, skipTrace)
+    return cursor.fetchone()
+
+def dbGetFirstColumn(con, cmd, args=[], skipTrace=False):
+    """
+    Do a select and return the first column's value from each row
+    """
+    ret = []
+    cursor = dbExecute(con, cmd, args, skipTrace)
+    for row in cursor:
+        for col in row:
+            ret.append(col)
+            break
+    return ret
+
+def dbGetFirstColumnAsMap(con, cmd, args=[], skipTrace=False, val=1):
+    """
+    Do a select and return the first column's value from each row
+    """
+    ret = {}
+    cursor = dbExecute(con, cmd, args, skipTrace)
+    for row in cursor:
+        for col in row:
+            ret[col] = val
+            break
+    return ret
+
+def dumpTable(con, tableName, max=-1):
+    """
+    If being extra verbose, print out the entire table
+    """
+    if verbose < 2:
+        return
+    traceOutput = sys.stderr if testOn else openLogFile("/opt/app/log/postgresql/idns/debug.log")
+    print("================ " + tableName + " ================", file=traceOutput)
+
+    cols = dbGetFirstColumn(con, "select column_name from information_schema.columns where table_name='" + tableName + "'", skipTrace=True)
+    print("num", end="|", file=traceOutput)
+    for col in cols:
+        print(col, end="|", file=traceOutput)
+    print("", file=traceOutput)
+
+    if max > -1:
+        cursor = dbExecute(con, "select * from " + tableName + " limit " + str(max), skipTrace=True)
+    else:
+        cursor = dbExecute(con, "select * from " + tableName, skipTrace=True)
+    i = 0
+    for row in cursor:
+        print("%d" % i, end="|", file=traceOutput)
+        i += 1
+        for col in row:
+            print("%s" % (col), end="|", file=traceOutput)
+        print("", file=traceOutput)
+    print("================================================", file=traceOutput)
+
+def getTableHtmls(con, DB, tableNames):
+    """
+    Retrieve a dump of all specified tables, in HTML format
+    """
+    ret = ""
+    for tn in tableNames:
+        ret = ret + getTableHtml(con, DB, tn)
+    return ret
+
+def getTableHtml(con, DB, tableName, max=-1):
+    """
+    Retrieve a dump of a given table, in HTML format
+    """
+    # errTrace("getting %s" % str(tableName))
+    ret = "<h2><a name='" + DB + "-table-" + tableName + "'>" + DB + "&nbsp;" + tableName + "</a>" + topButton + "</h2>\n"
+    ret = ret + "<table border='1'>\n"
+    # ret = ret + "<tr><th colspan='" + str(len(cols)+1) + "'>" + tableName + "</th></tr>\n"
+    cols = dbGetFirstColumn(con, "select column_name from information_schema.columns where table_name='" + tableName + "'", skipTrace=True)
+
+    ret = ret + "<tr><th>num</th>"
+    for col in cols:
+        ret = ret + "<th>" + str(col) + "</th>"
+    ret = ret + "</tr>\n"
+
+    if max > -1:
+        cursor = dbExecute(con, "select * from " + tableName + " limit " + str(max), skipTrace=True)
+    else:
+        cursor = dbExecute(con, "select * from " + tableName, skipTrace=True)
+    i = 0
+    for row in cursor:
+        ret = ret + "<tr><th>" + str(i) + "</th>"
+        i = i + 1
+        for col in row:
+            ret = ret + "<td>" + str(col) + "</td>"
+        ret = ret + "</tr>\n"
+    ret = ret + "</table>\n"
+    return ret
+
+def dbExecute(con, statement, args=[], skipTrace=False):
+    """
+    Create a cursor, instantiate the arguments into a statement, trace print the statement, and execute the statement.
+    Return the cursor
+    """
+    cursor = con.cursor()
+    stmt = cursor.mogrify(statement, args);
+    if not skipTrace:
+        debugTrace("executing:" + str(stmt))
+    cursor.execute(stmt)
+    if not skipTrace:
+        debugTrace("statusmessage=" + cursor.statusmessage + ", rowcount=" + str(cursor.rowcount))
+    return cursor
+
+if __name__ == '__main__':
+    server_class = http.server.HTTPServer
+    httpd = server_class(("0.0.0.0", PORT_NUMBER), MyHandler)
+    errTrace("Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER))
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    httpd.server_close()
+    errTrace("Server Stops - %s:%s" % (HOST_NAME, PORT_NUMBER))
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/makefile b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/makefile
new file mode 100644
index 0000000..2a94789
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/makefile
@@ -0,0 +1,22 @@
+all:
+
+NODES="uiopmno5qwpstg01.grant.example.com|uiopmno5qwpstg00.grant.example.com|uiopmno6qwpstg00.grant.example.com|uiopmno6qwpstg01.grant.example.com"
+
+test:
+	./gen-repmgr-info			-n $(NODES)
+	./gen-repmgr-info -S -L			-n $(NODES)
+	./gen-repmgr-info -s uiopmno6qwpstg	-n $(NODES)
+	./gen-repmgr-info -l uiopmno6qwpstg01.grant.example.com -n $(NODES)
+	./gen-repmgr-info -c uiopmno5qwpstg00.grant.example.com -n $(NODES)
+	./gen-repmgr-info -c uiopmno5qwpstg01.grant.example.com -n $(NODES)
+	./gen-repmgr-info -c uiopmno6qwpstg00.grant.example.com -n $(NODES)
+	./gen-repmgr-info -c uiopmno6qwpstg01.grant.example.com -n $(NODES)
+	./gen-repmgr-info -e uiopmno5qwpstg01.grant.example.com -n $(NODES)
+	./gen-repmgr-info -e uiopmno6qwpstg01.grant.example.com -n $(NODES)
+	./gen-repmgr-info -m			 -n $(NODES)
+	./gen-repmgr-info -C uiopmno5qwpstg00.grant.example.com -n $(NODES)
+	./gen-repmgr-info -C uiopmno5qwpstg01.grant.example.com -n $(NODES)
+	./gen-repmgr-info -C uiopmno6qwpstg00.grant.example.com -n $(NODES)
+	./gen-repmgr-info -C uiopmno6qwpstg01.grant.example.com -n $(NODES)
+	./gen-repmgr-info -p -n $(NODES)
+
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/pgwget b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/pgwget
new file mode 100644
index 0000000..d1d5f98
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/pgwget
@@ -0,0 +1,4 @@
+# this command is used to access the iDNS status server running on a PGaaS instance
+
+wgetpswd=`/opt/app/cdf/bin/getpropvalue -x -n wgetpswd`
+wget --http-user=pgaas --http-password=$wgetpswd "$@"
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/repmgrd-status-changes b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/repmgrd-status-changes
new file mode 100644
index 0000000..083b96f
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/bin/repmgrd-status-changes
@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this code except in compliance
+# with the License. You may obtain a copy of the License
+# at http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software  
+# distributed under the License is distributed on an "AS IS" BASIS,  
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  
+# implied. See the License for the specific language governing  
+# permissions and limitations under the License. 
+
+
+# %n - node ID
+# %e - event type
+# %s - success (1 or 0)
+# %t - timestamp
+# %d - details
+
+nodeID="$1"
+eventType="$2"
+success="$3"
+timestamp="$4"
+details="$5"
+
+LOG=/opt/app/log/postgresql/server/repmgrstatus.log
+PROMOTIONLOG=/var/run/postgresql/repmgr-promotion
+echo `date` "$@" >> $LOG
+
+# The following event types are available:
+#     master_register
+#     standby_register
+#     standby_unregister
+#     standby_clone
+#     standby_promote
+#     standby_follow
+#     standby_switchover
+#     standby_disconnect_manual
+#     witness_create
+#     witness_register
+#     witness_unregister
+#     repmgrd_start
+#     repmgrd_shutdown
+#     repmgrd_failover_promote
+#     repmgrd_failover_follow
+
+case "$eventType" in
+    standby_promote )
+        if [ "$success" -eq 1 ]
+	then echo $(date +%Y%m%d%H%M%S) "$@" >> $PROMOTIONLOG
+	fi
+esac
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/init/init-pgaas-idns.conf b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/init-pgaas-idns.conf
new file mode 100644
index 0000000..8ed39fb
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/init-pgaas-idns.conf
@@ -0,0 +1,21 @@
+# PGaaS - PostgreSQL as a Service
+#
+# The PGaaS iDNS server provides information on the system, primarily for the iDNS system
+
+description	"PGaaS iDNS server"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 10 5
+umask 022
+setuid postgres
+
+pre-start script
+    test -x /opt/app/postgresql-prep/bin/iDNS-responder.py || { stop; exit 0; }
+end script
+
+script
+    /opt/app/postgresql-prep/bin/iDNS-responder.py
+end script
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/init/init-pgaas-init.conf b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/init-pgaas-init.conf
new file mode 100644
index 0000000..4564129
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/init-pgaas-init.conf
@@ -0,0 +1,24 @@
+# PGaaS - PostgreSQL as a Service
+#
+# The PGaaS init process needs a directory to be created on reboot
+#
+# The best way to do this is to have a file in /usr/lib/tmpfiles.d/pgaas:
+#	d /var/run/postgresql 0755 postgres postgres -
+#
+# This is a workaround because systemd-tmpfiles is not present.
+
+description	"PGaaS init setup"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+umask 022
+
+pre-start script
+    mkdir /var/run/postgresql
+    chown postgres:postgres /var/run/postgresql
+end script
+
+script
+    :
+end script
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/init/logrotate b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/logrotate
new file mode 100644
index 0000000..aaa5e35
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/logrotate
@@ -0,0 +1,24 @@
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this code except in compliance
+# with the License. You may obtain a copy of the License
+# at http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software  
+# distributed under the License is distributed on an "AS IS" BASIS,  
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  
+# implied. See the License for the specific language governing  
+# permissions and limitations under the License. 
+
+
+# rotate PGaaS log files
+
+/opt/app/log/postgresql/init/*.log /opt/app/log/postgresql/idns/*.log /opt/app/log/postgresql/server/*.log {
+    missingok
+    compress
+    daily
+    rotate 60
+    create
+    dateext
+}
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/init/pglogs.cron b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/pglogs.cron
new file mode 100644
index 0000000..e905633
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/pglogs.cron
@@ -0,0 +1,4 @@
+30 * * * * ( date ; find /dbroot/pglogs -type f -mtime +3 -exec rm {} + ) >> /tmp/cleanpglogs.log 2>&1 
+50 1 * * * ( mv -f /tmp/cleanpglogs.log /tmp/cleanpglogs.log.old ) > /dev/null 2>&1
+* * * * * [ -x /opt/app/pgaas/bin/update_var_run_isrw ] && /opt/app/pgaas/bin/update_var_run_isrw
+* * * * * [ -x /opt/app/pgaas/bin/check_cluster ] && /opt/app/pgaas/bin/check_cluster >> /opt/app/log/postgresql/idns/cluster.log
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/init/systemd-pgaas-idns.service b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/systemd-pgaas-idns.service
new file mode 100644
index 0000000..bbc28ea
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/systemd-pgaas-idns.service
@@ -0,0 +1,12 @@
+# PGaaS - PostgreSQL as a Service                                                                                                                            
+#                                                                                                                                                            
+# The PGaaS iDNS server provides information on the system, primarily for the iDNS system                                                                    
+
+[Unit]
+Description=PGaaS iDNS server
+
+[Service]
+ExecStart= /opt/app/postgresql-prep/bin/iDNS-responder.py
+
+[Install]
+WantedBy=multi-user.target
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/init/tmpfiles-pgaas-init.conf b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/tmpfiles-pgaas-init.conf
new file mode 100644
index 0000000..9341a95
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/init/tmpfiles-pgaas-init.conf
@@ -0,0 +1,7 @@
+# PGaaS - PostgreSQL as a Service
+#
+# The PGaaS init process needs a directory to be created on reboot
+#
+
+d /var/run/postgresql 0755 postgres postgres -
+
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/green-flasher-12x10.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/green-flasher-12x10.gif
new file mode 100644
index 0000000..c6d33b5
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/green-flasher-12x10.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/grey-flasher-12x10.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/grey-flasher-12x10.gif
new file mode 100644
index 0000000..dfa81a2
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/grey-flasher-12x10.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/red-flasher-12x10.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/red-flasher-12x10.gif
new file mode 100644
index 0000000..ab16a81
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/red-flasher-12x10.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-G-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-G-12x25.gif
new file mode 100644
index 0000000..0ce2785
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-G-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-R-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-R-12x25.gif
new file mode 100644
index 0000000..0750594
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-R-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-Y-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-Y-12x25.gif
new file mode 100644
index 0000000..4f95bf0
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-Y-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-g-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-g-12x25.gif
new file mode 100644
index 0000000..0ce2785
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-g-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-g2-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-g2-12x25.gif
new file mode 100644
index 0000000..1be7fa1
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-g2-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-green.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-green.gif
new file mode 100644
index 0000000..1be7fa1
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-green.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-red.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-red.gif
new file mode 100644
index 0000000..daa6960
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-red.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-yellow.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-yellow.gif
new file mode 100644
index 0000000..1bd8731
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-master-yellow.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r-12x25.gif
new file mode 100644
index 0000000..0750594
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r2-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r2-12x25.gif
new file mode 100644
index 0000000..ff26986
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r2-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r3-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r3-12x25.gif
new file mode 100644
index 0000000..daa6960
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-r3-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-green.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-green.gif
new file mode 100644
index 0000000..c6d33b5
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-green.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-red.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-red.gif
new file mode 100644
index 0000000..ab16a81
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-red.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-yellow.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-yellow.gif
new file mode 100644
index 0000000..5e16750
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-secondary-yellow.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y-12x25.gif
new file mode 100644
index 0000000..4f95bf0
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y2-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y2-12x25.gif
new file mode 100644
index 0000000..886e548
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y2-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y3-12x25.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y3-12x25.gif
new file mode 100644
index 0000000..1bd8731
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/stoplight-y3-12x25.gif
diff --git a/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/yellow-flasher-12x10.gif b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/yellow-flasher-12x10.gif
new file mode 100644
index 0000000..5e16750
--- /dev/null
+++ b/postgresql-prep/src/stage/opt/app/postgresql-prep/lib/yellow-flasher-12x10.gif