Initial upload of F-GPS seed code 2/21

Includes:
Engine tools

Change-Id: I5611200a35db610c33a0bfe724c14fe9f492b41a
Issue-ID: OPTFRA-440
Signed-off-by: arthur.martella.1@att.com

9 files changed, 1120 insertions, 0 deletions

diff --git a/engine/src/tools/README.md b/engine/src/tools/README.md
new file mode 100644
index 0000000..9296f39
--- /dev/null
+++ b/engine/src/tools/README.md
@@ -0,0 +1,69 @@
+### Valet Tools for development, test, and production support
+
+|File| Description |
+|---|---|
+|crim.py|Commandline Rest Interface for Music<br>*read, add, delete from the music database*|
+|lock.py|Manual (Un)Locking Of Valet Regions<br>*from the regions (locking) table*|
+|ppdb.py|pretty print database<br>*try to make the database data readable*|
+|lib/common|collection of functions<br>- **set_argument** - *Get arg from file, cmdline, a pipe or prompt user*<br>- **list2string** - *join list and return as a string*<br>- **chop** - *like perl*|
+|lib/hosts.json|*contains all the currently known (by me) hosts for music and valet*|
+|lib/logger.py|*like the official logger but allows logger to point to file and or console*|
+|lib/tables.py|*Tables object, that handles each valet table as small subclass (could replace db_handler.py)*|
+|lib/song.py|*Song is music for scripts, a subclass of music.py, with script helpers*|
+
+#### Examples
+`$ crim.py -?`
+
+Show help message and exit (a lot more options than I am showing here...)
+
+`$ crim.py -names -read requests -read results`
+
+Show the contents of the requests and results tables in the default keyspace
+
+`$ crim.py -n -r q -r u`
+
+Same as above, but with using [watch](https://linux.die.net/man/1/watch "watch(1) - Linux man page") to execute the script repeatadly displaying the output
+Also this is an example of using shortcuts for arguments
+
+`$ watch crim.py -n -r q -r u`
+
+Show the contents of the regions tables (locking) in the all the known keyspaces
+
+`$ crim.py -K all -r regions`
+
+Delete the cw keyspace - this is used for testing to "clean" the database
+
+`$ crim.py -sD cw`
+
+Show the database stuff for the pn2 keyspace
+
+`$ crim.py -show -K pn`
+
+Show the config stuff for the pn2 keyspace
+
+`$ crim.py -ShowConfig -K pn`
+
+Show the database tables and definitions (hardcoded, not a query from the database)
+
+`$ crim.py -viewSchema`
+
+Show the requests record with id create-0000-0003 
+
+`$ crim.py -i create-0000-0003 -r q`
+
+Show the resources record in the pk2 keyspace with id "reg6:alan_stack_N003"
+
+`$ crim.py -r s -K pk2 -i "reg6:alan_stack_N003"`
+
+##### Testing Example
+
+Here we are going to copy a record from one environment to another
+
+Get a record from the request table, into a file; *Note:* Not the default config file...
+
+`$ crim.py -config ../test/solver.json -r q -K ist -id "create-abc-10099" > z`
+
+Put that record from the file into the request table of another keyspace 
+
+`$ crim.py -c ../test/solver.json -t q -K pk2 -a i -f z`
+
diff --git a/engine/src/tools/crim.py b/engine/src/tools/crim.py
new file mode 100644
index 0000000..a92ac6c
--- /dev/null
+++ b/engine/src/tools/crim.py
@@ -0,0 +1,216 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+#!/usr/bin/env python2.7
+
+"""
+Commandline Rest Interface Music  (CRIM)
+For help invoke with crim.py --help
+"""
+
+
+import argparse
+import json
+import os
+from pprint import pprint
+import re
+import sys
+import traceback
+
+from lib.song import Song
+from lib.tables import Tables
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))  # ../../..
+from valet.utils.logger import Logger
+
+
+# options <<<1
+def options():
+    toc = Tables.option_choices()
+
+    parser = argparse.ArgumentParser(description='\033[34mCommandline Rest-Interface to Music.\033[0m', add_help=False)
+    Song.add_args(parser)
+
+    group = parser.add_argument_group("Choose table\n\033[48;5;227m" + toc["alias"] + "\033[0m")
+
+    group.add_argument('-action', help='perform insert, update, delete, delete all, create table', choices=['i', 'u', 'd', 'D', 'c'])
+    group.add_argument('-id',  metavar='ID', action="append", help='id(s) in a table; (multiples allowed)')
+    group.add_argument('-file', help='json file required for -action create')
+    group.add_argument('-table', metavar='table', help='perform action on this table', default="request", choices=toc["choices"])
+
+    group = parser.add_argument_group("Add/Delete Schema")
+    group.add_argument("-sA", "-schemaAdd", metavar='keyspace', dest="schemaAdd", help="create keyspace")
+    group.add_argument("-sD", "-schemaDel", metavar='keyspace', dest="schemaDel", help="delete keyspace")
+
+    group = parser.add_argument_group("Query Tables")
+    group.add_argument('-read', metavar='all|table', action='append', help='read all or tables (multiples allowed)', choices=toc["choices"])
+    group.add_argument('-names', action='store_true', help='show names of tables on read')
+    group.add_argument('-Raw', action='store_true', help='dont strip out the music added fields')
+
+    group = parser.add_argument_group("Other Output")
+    group.add_argument("-?", "--help", action="help", help="show this help message and exit")
+    group.add_argument("-json", metavar='FILE', help="view json file")
+    group.add_argument('-show', action='store_true', help='show db stuff')
+    group.add_argument('-ShowConfig', action='store_true', help='show config stuff')
+    group.add_argument('-vt', '-viewTables', action='store_true', dest="viewTables", help='list tables (hardcoded)')
+    group.add_argument('-vs', '-viewSchema', action='store_true', dest="viewSchema", help='list table schema (hardcoded)')
+
+    return parser.parse_args()  # >>>1
+
+
+# setTable <<<1
+def _set_table(opts_table):
+    """ set table based on requested table. """
+
+    for sub_table in Tables.__subclasses__():
+        if opts_table in sub_table.table_alias():
+            return sub_table(music, logger)
+# >>>1
+
+
+question = lambda q: raw_input(q).lower().strip()[0] == "y"
+
+
+def clean_env(var):
+    if var in os.environ:
+        del os.environ[var]
+
+
+""" MAIN """
+if __name__ == "__main__":
+    clean_env('HTTP_PROXY')
+    clean_env('http_proxy')
+    opts = options()
+
+    # Get logger; Read config and strike up a song <<<1
+    logger = Logger().get_logger('console')
+    config = json.loads(open(opts.config).read())
+    music = Song(opts, config, logger)
+    # >>>1
+
+    if opts.viewTables:
+        for table in Tables.__subclasses__():
+            print (re.sub("[[\]',]", '', str(table.table_alias()))).split(" ")[0]
+        sys.exit(0)
+
+    if opts.viewSchema:
+        for table in Tables.__subclasses__():
+            sys.stdout.write(re.sub("[[\]',]", '', str(table.table_alias())) + ' ')
+            print json.dumps(table.schema, sort_keys=True, indent=2), "\n"
+        sys.exit(0)
+
+    """ Keyspace Create """  # <<<1
+    if opts.schemaAdd:
+        sys.exit(music.create_keyspace(opts.schemaAdd))
+
+    """ Keyspace Delete """  # <<<1
+    if opts.schemaDel:
+        if question("You sure you wanna delete keyspace '%s'? [y/n] " % opts.schemaDel):
+            sys.exit(music.drop_keyspace(opts.schemaDel))
+
+    # all the tables listed with '-read's <<<1
+    if opts.read:
+        if 'all' in opts.read:
+            if music.keyspace == "all":
+                sys.exit("read all tables for all keyspaces is not currently supported")
+
+            for table in Tables.__subclasses__():
+                table(music, logger).read(raw=opts.Raw, names=True)
+            sys.exit(0)
+
+        if music.keyspace == "all":
+            opts.names = True
+            for keyspace in Song.Keyspaces.keys():
+                music.keyspace = Song.Keyspaces[keyspace]
+                print "\n-----------------    %s : %s    -----------------" % (keyspace, music.keyspace)
+                # noinspection PyBroadException
+                try:
+                    for tName in opts.read:
+                        _set_table(tName).read(ids=opts.id, json_file=opts.file, names=opts.names, raw=opts.Raw)
+                except Exception as e:
+                    pass
+            sys.exit(0)
+
+        for tName in opts.read:
+            _set_table(tName).read(ids=opts.id, json_file=opts.file, names=opts.names, raw=opts.Raw)
+        sys.exit(0)
+
+    table = _set_table(opts.table)
+
+    # show all db stuff <<<1
+    if opts.show or opts.ShowConfig:
+        if opts.show:
+            print "music"
+            pprint(music.__dict__, indent=2)
+            print "\nrest"
+            pprint(music.rest.__dict__, indent=2)
+
+            if table is not None:
+                print "\n", table.table()
+                pprint(table.__dict__, indent=2)
+
+        if opts.ShowConfig:
+            print (json.dumps(config, indent=4))
+
+        sys.exit(0)
+    # >>>1
+
+    """ VIEW JSON FILE open, convert to json, convert to string and print it """  # <<<1
+    if opts.json:
+        # noinspection PyBroadException
+        try:
+            print (json.dumps(json.loads(open(opts.json).read()), indent=4))
+        except Exception as e:
+            print (traceback.format_exc())
+            sys.exit(2)
+        sys.exit(0)
+
+    """ Insert use json file to add record to database """  # <<<1
+    if opts.action == 'i':
+        table.create(opts.file)
+        sys.exit(0)
+    
+    """ CREATE Table """  # <<<1
+    if opts.action == 'c':
+        table.create_table()
+        sys.exit(0)
+    
+    """ UPDATE use json file to update db record """  # <<<1
+    if opts.action == 'u':
+        if not opts.file or not os.path.exists(opts.file):
+            print "--file filename (filename exists) is required for update"
+            sys.exit(1)
+
+        table.update(opts.file)
+        sys.exit(0)
+    
+    """ DELETE  use id to delete record from db -- requres ID """  # <<<1
+    if opts.action == 'd':
+        if not opts.id:
+            print "--id ID is required for delete"
+            sys.exit(1)
+
+        if opts.id:
+            table.delete(opts.id)
+            sys.exit(0)
+
+    """ DELETE ALL from table"""  # <<<1
+    if opts.action == 'D':
+        table.clean_table()
+        sys.exit(0)
+# >>>1
diff --git a/engine/src/tools/lib/__init__.py b/engine/src/tools/lib/__init__.py
new file mode 100644
index 0000000..bd50995
--- /dev/null
+++ b/engine/src/tools/lib/__init__.py
@@ -0,0 +1,18 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
diff --git a/engine/src/tools/lib/common.py b/engine/src/tools/lib/common.py
new file mode 100644
index 0000000..4973f4e
--- /dev/null
+++ b/engine/src/tools/lib/common.py
@@ -0,0 +1,73 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+
+import os
+import select
+import sys
+
+
+def set_argument(arg=None, prompt=None, multiline=False):
+    """Return argument from file, cmd line, read from a pipe or prompt user """
+
+    if arg:
+        if os.path.isfile(arg):
+            f = open(arg)
+            message = f.readlines()
+            f.close()
+        else:
+            message = arg
+    else:
+        if sys.stdin in select.select([sys.stdin], [], [], .5)[0]:
+            message = sys.stdin.readlines()
+        else:
+            print prompt,
+            if multiline:
+                sentinel = ''
+                message = list(iter(raw_input, sentinel))
+            else:
+                message = [raw_input()]
+
+    return message
+
+
+def list2string(message):
+    return ''.join(message)
+
+
+def chop(message):
+
+    if message.endswith('\n'):
+        message = message[:-1]
+
+    return message
+
+
+# MAIN
+if __name__ == "__main__":
+
+    msg = set_argument(sys.argv[0])
+    for row in msg:
+        print row,
+    print "\n", list2string(msg)
+
+    msg = set_argument(prompt="Message? ")
+    for row in msg:
+        print row,
+    print "\n", list2string(msg)
diff --git a/engine/src/tools/lib/hosts.json b/engine/src/tools/lib/hosts.json
new file mode 100644
index 0000000..f45096b
--- /dev/null
+++ b/engine/src/tools/lib/hosts.json
@@ -0,0 +1,100 @@
+{
+	"music": {
+		"dev": {
+			"hosts": {
+				"id": "openstack region a",
+				"a": [ "255.255.255.0", "255.255.255.1", "255.255.255.2" ],
+				"b": [ "255.255.255.3", "255.255.255.4", "255.255.255.5" ]
+			}
+		},
+
+		"ist": {
+			"hosts": {
+				"id": "ist",
+				"a": [ "255.255.255.6", "255.255.255.7", "255.255.255.8" ],
+				"b": [ "255.255.255.9", "255.255.255.10", "255.255.255.11" ]
+			}
+		},
+
+		"e2e": {
+			"hosts": {
+				"id": "e2e",
+				"a": [ "255.255.255.12", "255.255.255.13", "255.255.255.14" ],
+				"b": [ "255.255.255.15", "255.255.255.16", "255.255.255.17" ]
+			}
+		},
+
+		"music": {
+			"hosts": {
+				"id": "music dev",
+				"a": [ "255.255.255.18" ],
+				"b": [ "255.255.255.19" ]
+			}
+		}
+	},
+
+	"valet": {
+		"dev": [
+			{
+				"ip": "255.255.255.20", 
+				"fqdn": "dev1.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.21", 
+				"fqdn": "dev2.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.22", 
+				"fqdn": "dev3.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.23", 
+				"fqdn": "dev4.site.onap.org"
+			}
+		],
+		"ist": [
+			{
+				"ip": "255.255.255.24", 
+				"fqdn": "ist1.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.25", 
+				"fqdn": "ist2.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.26", 
+				"fqdn": "ist3.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.27", 
+				"fqdn": "ist4.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.28", 
+				"fqdn": "ist5.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.29", 
+				"fqdn": "ist6.site.onap.org"
+			}
+		],
+		"e2e": [
+			{
+				"ip": "255.255.255.30", 
+				"fqdn": "e2e1.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.31", 
+				"fqdn": "e2e2.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.32", 
+				"fqdn": "e2e3.site.onap.org"
+			},
+			{
+				"ip": "255.255.255.33", 
+				"fqdn": "e2e4.site.onap.org"
+			}
+		]
+	}
+}
diff --git a/engine/src/tools/lib/song.py b/engine/src/tools/lib/song.py
new file mode 100644
index 0000000..fdbba99
--- /dev/null
+++ b/engine/src/tools/lib/song.py
@@ -0,0 +1,153 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+"""Song is music for scripts
+
+    This is a subclass of music that scripts can use to add an option to override:
+        add_args - the commandline arguments that Song uses
+        keyspace - to allow the same script to run vs other databases
+        hosts table - to allow the same script to run vs other databases on other machines
+        connect - login may be different for other databases
+"""
+
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))  # ../../..
+from valet.utils.logger import Logger
+from valet.engine.db_connect.db_apis.music import Music
+
+
+def hosts(opts, config):
+    """override configs hosts"""
+
+    hosts_json = os.path.join(sys.path[-1], "tools", "lib", "hosts.json")
+    _hosts = json.loads(open(hosts_json).read())
+    config["music"]["hosts"] = _hosts["music"][opts.env or "dev"]["hosts"]["a"]
+
+    if opts.verbose:
+        print "hosts: " + str(config["music"]["hosts"])
+
+
+# noinspection PyBroadException
+class Song(Music):
+    """music but with script helpers"""
+
+    Keyspaces = {
+        "cw":  "valet_TestDB123",
+        "cmw": "valet_cmw",
+        "pk":  "valet_TestDB420",
+        "pk2": "valetdb2",
+        "pn":  "pn2",
+        "st":  "valet_saisree",
+        "ist": "valet_IST",
+        "gj":  "valet_TestDB2"
+    }
+    Keyspaces.update(dict((v, v) for k, v in Keyspaces.iteritems()))  # full name is valid too
+
+    def __init__(self, opts, config, logger):
+        if opts.env:
+            hosts(opts, config)
+
+        self.keyspace = config["db"]["keyspace"]
+        self.defaultKeyspace = True 
+
+        if opts.Keyspace:
+            if opts.Keyspace == "all":
+                self.keyspace = opts.Keyspace
+            else:
+                self.keyspace = Song.Keyspaces[opts.Keyspace]
+            self.defaultKeyspace = False
+
+        if opts.db:
+            self.keyspace = opts.db
+            self.defaultKeyspace = False
+       
+        # TODO cmw: move keyspace into music object, pass in like config["keyspace"] = self.keyspace
+
+        super(Song, self).__init__(config, logger)
+
+    @staticmethod
+    def add_args(parser):
+        """add common parser arguments"""
+        default_config = "/opt/config/solver.json"
+        if not os.path.isfile(default_config):
+            default_config = os.path.join(sys.path[-1], "config", "solver.json")
+
+        valid_keyspaces = Song.Keyspaces.keys()
+        valid_keyspaces.append("all")
+        valid_keyspaces_str = "{" + ",".join(valid_keyspaces) + "}"
+
+        valid_hosts = ["a1", "a2", "a3", "b3", "ab", "m"]
+        valid_env = ["dev", "ist", "e2e"]
+
+        song_args = parser.add_argument_group("Common Music Arguments")
+        song_args.add_argument('-env', metavar=valid_env, help='pick set of hosts -deprecated', choices=valid_env)
+        song_args.add_argument('-host', metavar=valid_hosts, help='pick set of hosts -deprecated', choices=valid_hosts)
+        ex = song_args.add_mutually_exclusive_group()
+        ex.add_argument('-Keyspace', metavar=valid_keyspaces_str, help='override configs keyspace with a users', choices=valid_keyspaces)
+        ex.add_argument('-db', metavar='keyspace_string', help='override keyspace with typed in value')
+        song_args.add_argument('-config', metavar='file', default=default_config, help="default: " + default_config)
+        song_args.add_argument('-verbose', action='store_true', help="verbose output")
+
+    def create_keyspace(self, keyspace):
+        """override creates a keyspace."""
+
+        data = {
+            'replicationInfo': {
+                "DC2": 3,
+                "DC1": 3,
+                "class": "NetworkTopologyStrategy"
+            },
+            'durabilityOfWrites': True,
+            'consistencyInfo': {
+                'type': 'eventual',
+            },
+        }
+
+        path = '/keyspaces/%s' % keyspace
+        try:
+            self.rest.request(method='post', path=path, data=data)
+            return 0
+        except Exception:
+            # "this exception should be handled here but it's done in music :("
+            return -1
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Extended Music DB.', add_help=False)
+    Song.add_args(parser)
+    parser.add_argument("-?", "--help", action="help", help="show this help message and exit")
+    opts = parser.parse_args()
+
+    logger = Logger().get_logger('console')
+
+    config = json.loads(open(opts.config).read())
+    music = Song(opts, config, logger)
+
+    print json.dumps(config.get("music"))
+    print (music.keyspace)
+
+
+# MAIN
+if __name__ == "__main__":
+    main()
+    sys.exit(0)
diff --git a/engine/src/tools/lib/tables.py b/engine/src/tools/lib/tables.py
new file mode 100644
index 0000000..7ad4836
--- /dev/null
+++ b/engine/src/tools/lib/tables.py
@@ -0,0 +1,308 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+
+import json
+import os
+import sys
+import time
+from datetime import datetime
+from textwrap import TextWrapper
+
+import pytz
+
+
+class Tables(object):
+    """parent class for all tables."""
+
+    schema = None
+    alias = None
+
+    def __init__(self, music, logger):
+        """Initializer. Accepts target host list, port, and path."""
+
+        self.music = music
+        self.logger = logger
+        self.key = None
+        self.keyspace = music.keyspace
+        self.tz = pytz.timezone('America/New_York')
+
+    @classmethod
+    def table_alias(cls):
+        s = cls.__name__
+        s = s[0].lower() + s[1:]
+        return [s] + cls.alias
+
+    @staticmethod
+    def option_choices():
+        choices = []
+        aliases = None
+        for table in Tables.__subclasses__():
+            choices = choices + table.table_alias()
+            for alias in choices:
+                if aliases:
+                    aliases = aliases + ' ' + alias
+                else:
+                    aliases = alias
+
+        choices.append('all')
+
+        return {
+            "alias":  TextWrapper(subsequent_indent="    ", initial_indent="    ", width=79).fill(aliases),
+            "choices": choices
+        }
+
+    def table(self):
+        """Return tables name (same as class name but all lc)."""
+
+        s = type(self).__name__
+        return s[0].lower() + s[1:]
+
+    def utc2local(self, utc):
+        """Change utc time to local time for readability."""
+
+        return " (" + datetime.fromtimestamp(utc, self.tz).strftime('%Y-%m-%d %I:%M:%S %Z%z') + ")"
+
+    def get_rows(self, ids=None, raw=False):
+        """ get_rows read table, or rows by id and return rows array """
+
+        # TODO if ids == None and hasattr(self, 'ids'): ids = self.ids
+        key = None if (ids is None) else self.key
+        rows = []
+
+        for row_id in ids or [None]:
+            if raw:
+                rows.append(json.dumps(self.music.read_row(self.keyspace, self.table(), key, row_id), sort_keys=True, indent=4))
+                continue
+
+            result = self.music.read_row(self.keyspace, self.table(), key, row_id)["result"]
+
+            # strip "Row n"
+            for _, data in sorted(result.iteritems()):
+                rows.append(data)
+
+        if raw:
+            return rows
+
+        if len(rows) == 1:
+            rows = rows[0]  # one row? not a list
+
+        return rows
+
+    def read(self, ids=None, json_file=None, raw=False, rows=None, names=None):
+        """ read rows (array or single dict) to stdout or to a file """
+
+        if rows is None:
+            rows = self.get_rows(ids, raw)
+
+        if raw:
+            if names:
+                print "\n" + self.table()
+            for row in rows:
+                print row
+            return
+
+        if isinstance(rows, list):
+            for row in rows:
+                if not ("timestamp" in row or "expire_time" in row):
+                    break
+                for key in (["timestamp", "expire_time"]):
+                    if (not (key in row)) or (row[key] is None):
+                        continue
+                    try:
+                        row[key] = row[key] + self.utc2local(float(row[key])/1000)
+                    except ValueError:
+                        row[key] = "Error: "+ row[key]
+                        
+        else:
+            row = rows
+            for key in (["timestamp", "expire_time"]):
+                if (not (key in row)) or (row[key] is None):
+                    continue
+                try:
+                    row[key] = row[key] + self.utc2local(float(row[key])/1000)
+                except ValueError:
+                    row[key] = "Error: "+ row[key]
+
+        if json_file is None:
+            if names:
+                print "\n" + self.table()
+            print json.dumps(rows, sort_keys=True, indent=4)
+            return
+
+        fh = open(json_file, "w")
+        fh.write(json.dumps(rows, sort_keys=True, indent=4))
+        fh.close() 
+
+    def create(self, json_file=None):
+        """ add records from a file to db """
+
+        if json_file and os.path.exists(json_file):
+                inf = open(json_file)
+                f = json_file
+        else:
+                inf = sys.stdin
+                f = "stdin"
+
+        self.logger.info("Create " + self.table() + " from: " + f)
+        self.insert(json.loads(inf.read()))
+
+    def insert(self, data):
+        """ add records """
+
+        self.logger.debug(data)
+
+        if isinstance(data, list):
+            for row in data:
+                if "timestamp" in row:
+                    row['timestamp'] = int(round(time.time() * 1000))
+                self.music.create_row(self.keyspace, self.table(), row) 
+        else:
+            row = data
+            if "timestamp" in row:
+                row['timestamp'] = int(round(time.time() * 1000))
+            self.music.create_row(self.keyspace, self.table(), row)
+
+    def create_table(self):
+        """ create table """
+
+        self.logger.info(self.schema)
+        self.music.create_table(self.keyspace, self.table(), self.schema)
+
+    def update(self, json_file):
+        """Update a row. Not atomic."""
+        
+        self.logger.info("Update " + self.table() + " from: " + json_file)
+        data = json.loads(open(json_file).read())
+        self.logger.debug(data)
+
+        if isinstance(data, list):
+            for row in data:
+                self.music.update_row_eventually(self.keyspace, self.table(), row)
+        else:
+            self.music.update_row_eventually(self.keyspace, self.table(), data)
+
+    def delete(self, ids):
+        """ delete records db based on id """
+
+        for row_id in ids:
+            self.logger.info("Delete from" + self.table() + " id: " + row_id)
+            self.music.delete_row_eventually(self.keyspace, self.table(), self.key, row_id)
+
+    def clean_table(self):
+        """ delete all records in table """
+
+        ids = []
+        rows = self.get_rows()
+
+        if isinstance(rows, list):
+            for row in rows:
+                ids.append(row[self.key])
+        else:
+            row = rows
+            ids.append(row[self.key])
+
+        for row_id in ids:
+            self.music.delete_row_eventually(self.keyspace, self.table(), self.key, row_id)
+
+#  Subclasses of Tables:
+
+
+class Requests(Tables):
+    alias = ["req", "q"]
+    key = "request_id"
+    schema = json.loads('{ "request_id": "text", "timestamp": "text", "request": "text", "PRIMARY KEY": "(request_id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Requests.key
+
+
+class Results(Tables):
+    alias = ["resu", "u"]
+    key = "request_id"
+    schema = json.loads('{ "request_id": "text", "status": "text", "timestamp": "text", "result": "text", "PRIMARY KEY": "(request_id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Results.key
+
+class Group_rules(Tables):
+    alias = ["rule", "gr"]
+    key = "id"
+    schema = json.loads('{ "id": "text", "app_scope": "text", "type": "text", "level": "text", "members": "text", "description": "text", "groups": "text", "status": "text", "timestamp": "text", "PRIMARY KEY": "(id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Group_rules.key
+
+
+class Stacks(Tables):
+    alias = ["stack", "s"]
+    key = "id"
+    schema = json.loads('{ "id": "text", "last_status": "text", "datacenter": "text", "stack_name": "text", "uuid": "text", "tenant_id": "text", "metadata": "text", "servers": "text", "prior_servers": "text", "state": "text", "prior_State": "text", "timestamp": "text", "PRIMARY KEY": "(id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Stacks.key
+
+
+class Stack_id_map(Tables):
+    alias = ["map", "m"]
+    key = "request_id"
+    schema = json.loads('{ "request_id": "text", "stack_id": "text", "timestamp": "text", "PRIMARY KEY": "(request_id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Stack_id_map.key
+
+
+class Resources(Tables):
+    alias = ["reso", "o"]
+    key = "id"
+    schema = json.loads('{ "id": "text", "url": "text", "resource": "text", "timestamp": "text", "requests": "text", "PRIMARY KEY": "(id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Resources.key
+
+
+class Regions(Tables):
+    alias = ["reg", "i", "lock"]
+    key = "region_id"
+    schema = json.loads('{ "region_id ": "text", "timestamp": "text", "last_updated ": "text", "keystone_url": "text", "locked_by": "text", "locked_time ": "text", "expire_time": "text", "PRIMARY KEY": "(region_id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Regions.key
+
+
+class Groups(Tables):
+    alias = ["group", "g"]
+    key = "id"
+    schema = json.loads('{ "id ": "text", "uuid": "text", "type ": "text", "level": "text", "factory": "text", "rule_id ": "text", "metadata ": "text", "server_list": "text", "member_hosts": "text", "status": "text", "PRIMARY KEY": "(id)" }')
+
+    def __init__(self, music, logger):
+        Tables.__init__(self, music, logger)
+        self.key = Groups.key
+
+
+if __name__ == "__main__":
+
+    print Tables.option_choices()
diff --git a/engine/src/tools/lock.py b/engine/src/tools/lock.py
new file mode 100644
index 0000000..a77eb8a
--- /dev/null
+++ b/engine/src/tools/lock.py
@@ -0,0 +1,92 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+#!/usr/bin/env python2.7
+
+
+import argparse
+import json
+import os
+import sys
+
+import lib.tables as tables
+from lib.song import Song
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from valet.engine.db_connect.db_handler import DBHandler
+from valet.engine.db_connect.locks import Locks, later
+from valet.utils.logger import Logger
+
+
+def options():
+    parser = argparse.ArgumentParser(description='\033[34mManual Locking And Unlocking Of Valet Regions.\033[0m', add_help=False)
+    Song.add_args(parser)
+
+    g1 = parser.add_argument_group('Control Music Locks')
+    ex = g1.add_mutually_exclusive_group()
+    ex.add_argument('-unlock', metavar="region", help='unlock this region')
+    ex.add_argument('-lock', metavar="region", help='lock this region')
+
+    g2 = parser.add_argument_group('Update Locks In Region Table')
+    ex = g2.add_mutually_exclusive_group()
+    ex.add_argument('-delete', metavar="region", help='delete region from table')
+    ex.add_argument('-add', metavar="region", help='update/add region to table')
+    g2.add_argument('-timeout', metavar="seconds", help='seconds till region expires (for -add)')
+
+    group = parser.add_argument_group("Change The Output")
+    group.add_argument("-?", "--help", action="help", help="show this help message and exit")
+    group.add_argument('-show', action='store_true', help='print out regions (locking) table')
+
+    return parser.parse_args()
+
+
+# MAIN
+if __name__ == "__main__":
+    opts = options()
+
+    logger = Logger().get_logger('console')
+    config = json.loads(open(opts.config).read())
+    music_config = config.get("music")
+    music_config["keyspace"] = config.get("db")["keyspace"]
+    music = Song(opts, config, logger)
+    dbh = DBHandler(music, config.get("db"), logger)
+
+    if opts.add:
+        timeout = opts.timeout if opts.timeout else config["engine"]["timeout"]
+        dbh.add_region(opts.add, later(seconds=int(timeout)))
+        logger.debug("added region to table")
+
+    if opts.lock:
+        if Locks(dbh, 0).got_lock(opts.lock) == "ok":
+            logger.debug("added region lock")
+        else:
+            logger.debug("failed to add region lock")
+
+    if opts.unlock:
+        Locks.unlock(dbh, opts.unlock)
+        logger.debug("deleted region lock '%s'" % opts.unlock)
+
+    if opts.delete:
+        dbh.delete_region(opts.delete)
+        logger.debug("deleted region from table")
+
+    if opts.show:
+        music.keyspace = dbh.keyspace
+        tables.Regions(music, logger).read(names=True)
+
+    sys.exit(0)
diff --git a/engine/src/tools/ppdb.py b/engine/src/tools/ppdb.py
new file mode 100644
index 0000000..c89f0f2
--- /dev/null
+++ b/engine/src/tools/ppdb.py
@@ -0,0 +1,91 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+#!/usr/bin/env python2.7
+
+
+import sys
+import os
+import json
+import argparse
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='parse results.')
+    parser.add_argument('-f', '--file', help='json file required for create/view')
+    parser.add_argument('-verbose', action='store_true', help='more output than you want')
+    opts = parser.parse_args()
+
+    if opts.file and os.path.exists(opts.file):
+            inf = open(opts.file)
+    else:
+            inf = sys.stdin
+
+    results = json.loads(inf.read())
+    if opts.verbose:
+        print (json.dumps(results, sort_keys=True, indent=4))
+        print "---------------------------------------------"
+
+    if "request" in results.keys():
+        key = "request"
+        result = json.loads(results[key])
+        print (json.dumps(result, sort_keys=True, indent=4))
+        sys.exit(0)
+
+    if "result" in results.keys():
+        result = results["result"]
+
+        if not isinstance(result, list):
+            sys.stdout.write("result ")
+            sys.stdout.flush()
+            result = json.loads(result)
+            print (json.dumps(result, sort_keys=True, indent=4))
+            sys.exit(0)
+
+        for _, row in result.iteritems():
+            rr = json.loads(row["result"])
+
+            # for k, d in row.iteritems():
+            #     print ("%s)  %s"% (k, d))
+
+            sys.stdout.write("result ")
+            sys.stdout.flush()
+            print json.dumps(rr, indent=4)
+            # for f in rr:
+            #     for line in (json.dumps(f, sort_keys=True, indent=4)).splitlines():
+            #         print "\t%s"%line
+            # print "}"
+
+        sys.exit(0)
+
+    if "resource" in results.keys():
+        key = "resource"
+        result = json.loads(results[key])
+        print (json.dumps(result, sort_keys=True, indent=4))
+
+        if not isinstance(result, list):
+            sys.stdout.write("resource ")
+            sys.stdout.flush()
+            result = json.loads(result)
+            print (json.dumps(result, sort_keys=True, indent=4))
+            sys.exit(0)
+
+        print (json.dumps(result, sort_keys=True, indent=4))
+        sys.exit(0)
+
+    print (results.keys())