diff options
Diffstat (limited to 'engine/src/tools')
| -rw-r--r-- | engine/src/tools/README.md | 69 | ||||
| -rw-r--r-- | engine/src/tools/crim.py | 216 | ||||
| -rw-r--r-- | engine/src/tools/lib/__init__.py | 18 | ||||
| -rw-r--r-- | engine/src/tools/lib/common.py | 73 | ||||
| -rw-r--r-- | engine/src/tools/lib/hosts.json | 100 | ||||
| -rw-r--r-- | engine/src/tools/lib/song.py | 153 | ||||
| -rw-r--r-- | engine/src/tools/lib/tables.py | 308 | ||||
| -rw-r--r-- | engine/src/tools/lock.py | 92 | ||||
| -rw-r--r-- | engine/src/tools/ppdb.py | 91 |
9 files changed, 1120 insertions, 0 deletions
diff --git a/engine/src/tools/README.md b/engine/src/tools/README.md new file mode 100644 index 0000000..9296f39 --- /dev/null +++ b/engine/src/tools/README.md @@ -0,0 +1,69 @@ +### Valet Tools for development, test, and production support + +|File| Description | +|---|---| +|crim.py|Commandline Rest Interface for Music<br>*read, add, delete from the music database*| +|lock.py|Manual (Un)Locking Of Valet Regions<br>*from the regions (locking) table*| +|ppdb.py|pretty print database<br>*try to make the database data readable*| +|lib/common|collection of functions<br>- **set_argument** - *Get arg from file, cmdline, a pipe or prompt user*<br>- **list2string** - *join list and return as a string*<br>- **chop** - *like perl*| +|lib/hosts.json|*contains all the currently known (by me) hosts for music and valet*| +|lib/logger.py|*like the official logger but allows logger to point to file and or console*| +|lib/tables.py|*Tables object, that handles each valet table as small subclass (could replace db_handler.py)*| +|lib/song.py|*Song is music for scripts, a subclass of music.py, with script helpers*| + +#### Examples +`$ crim.py -?` + +Show help message and exit (a lot more options than I am showing here...) + +`$ crim.py -names -read requests -read results` + +Show the contents of the requests and results tables in the default keyspace + +`$ crim.py -n -r q -r u` + +Same as above, but with using [watch](https://linux.die.net/man/1/watch "watch(1) - Linux man page") to execute the script repeatadly displaying the output +Also this is an example of using shortcuts for arguments + +`$ watch crim.py -n -r q -r u` + +Show the contents of the regions tables (locking) in the all the known keyspaces + +`$ crim.py -K all -r regions` + +Delete the cw keyspace - this is used for testing to "clean" the database + +`$ crim.py -sD cw` + +Show the database stuff for the pn2 keyspace + +`$ crim.py -show -K pn` + +Show the config stuff for the pn2 keyspace + +`$ crim.py -ShowConfig -K pn` + +Show the database tables and definitions (hardcoded, not a query from the database) + +`$ crim.py -viewSchema` + +Show the requests record with id create-0000-0003 + +`$ crim.py -i create-0000-0003 -r q` + +Show the resources record in the pk2 keyspace with id "reg6:alan_stack_N003" + +`$ crim.py -r s -K pk2 -i "reg6:alan_stack_N003"` + +##### Testing Example + +Here we are going to copy a record from one environment to another + +Get a record from the request table, into a file; *Note:* Not the default config file... + +`$ crim.py -config ../test/solver.json -r q -K ist -id "create-abc-10099" > z` + +Put that record from the file into the request table of another keyspace + +`$ crim.py -c ../test/solver.json -t q -K pk2 -a i -f z` + diff --git a/engine/src/tools/crim.py b/engine/src/tools/crim.py new file mode 100644 index 0000000..a92ac6c --- /dev/null +++ b/engine/src/tools/crim.py @@ -0,0 +1,216 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +#!/usr/bin/env python2.7 + +""" +Commandline Rest Interface Music (CRIM) +For help invoke with crim.py --help +""" + + +import argparse +import json +import os +from pprint import pprint +import re +import sys +import traceback + +from lib.song import Song +from lib.tables import Tables + +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) # ../../.. +from valet.utils.logger import Logger + + +# options <<<1 +def options(): + toc = Tables.option_choices() + + parser = argparse.ArgumentParser(description='\033[34mCommandline Rest-Interface to Music.\033[0m', add_help=False) + Song.add_args(parser) + + group = parser.add_argument_group("Choose table\n\033[48;5;227m" + toc["alias"] + "\033[0m") + + group.add_argument('-action', help='perform insert, update, delete, delete all, create table', choices=['i', 'u', 'd', 'D', 'c']) + group.add_argument('-id', metavar='ID', action="append", help='id(s) in a table; (multiples allowed)') + group.add_argument('-file', help='json file required for -action create') + group.add_argument('-table', metavar='table', help='perform action on this table', default="request", choices=toc["choices"]) + + group = parser.add_argument_group("Add/Delete Schema") + group.add_argument("-sA", "-schemaAdd", metavar='keyspace', dest="schemaAdd", help="create keyspace") + group.add_argument("-sD", "-schemaDel", metavar='keyspace', dest="schemaDel", help="delete keyspace") + + group = parser.add_argument_group("Query Tables") + group.add_argument('-read', metavar='all|table', action='append', help='read all or tables (multiples allowed)', choices=toc["choices"]) + group.add_argument('-names', action='store_true', help='show names of tables on read') + group.add_argument('-Raw', action='store_true', help='dont strip out the music added fields') + + group = parser.add_argument_group("Other Output") + group.add_argument("-?", "--help", action="help", help="show this help message and exit") + group.add_argument("-json", metavar='FILE', help="view json file") + group.add_argument('-show', action='store_true', help='show db stuff') + group.add_argument('-ShowConfig', action='store_true', help='show config stuff') + group.add_argument('-vt', '-viewTables', action='store_true', dest="viewTables", help='list tables (hardcoded)') + group.add_argument('-vs', '-viewSchema', action='store_true', dest="viewSchema", help='list table schema (hardcoded)') + + return parser.parse_args() # >>>1 + + +# setTable <<<1 +def _set_table(opts_table): + """ set table based on requested table. """ + + for sub_table in Tables.__subclasses__(): + if opts_table in sub_table.table_alias(): + return sub_table(music, logger) +# >>>1 + + +question = lambda q: raw_input(q).lower().strip()[0] == "y" + + +def clean_env(var): + if var in os.environ: + del os.environ[var] + + +""" MAIN """ +if __name__ == "__main__": + clean_env('HTTP_PROXY') + clean_env('http_proxy') + opts = options() + + # Get logger; Read config and strike up a song <<<1 + logger = Logger().get_logger('console') + config = json.loads(open(opts.config).read()) + music = Song(opts, config, logger) + # >>>1 + + if opts.viewTables: + for table in Tables.__subclasses__(): + print (re.sub("[[\]',]", '', str(table.table_alias()))).split(" ")[0] + sys.exit(0) + + if opts.viewSchema: + for table in Tables.__subclasses__(): + sys.stdout.write(re.sub("[[\]',]", '', str(table.table_alias())) + ' ') + print json.dumps(table.schema, sort_keys=True, indent=2), "\n" + sys.exit(0) + + """ Keyspace Create """ # <<<1 + if opts.schemaAdd: + sys.exit(music.create_keyspace(opts.schemaAdd)) + + """ Keyspace Delete """ # <<<1 + if opts.schemaDel: + if question("You sure you wanna delete keyspace '%s'? [y/n] " % opts.schemaDel): + sys.exit(music.drop_keyspace(opts.schemaDel)) + + # all the tables listed with '-read's <<<1 + if opts.read: + if 'all' in opts.read: + if music.keyspace == "all": + sys.exit("read all tables for all keyspaces is not currently supported") + + for table in Tables.__subclasses__(): + table(music, logger).read(raw=opts.Raw, names=True) + sys.exit(0) + + if music.keyspace == "all": + opts.names = True + for keyspace in Song.Keyspaces.keys(): + music.keyspace = Song.Keyspaces[keyspace] + print "\n----------------- %s : %s -----------------" % (keyspace, music.keyspace) + # noinspection PyBroadException + try: + for tName in opts.read: + _set_table(tName).read(ids=opts.id, json_file=opts.file, names=opts.names, raw=opts.Raw) + except Exception as e: + pass + sys.exit(0) + + for tName in opts.read: + _set_table(tName).read(ids=opts.id, json_file=opts.file, names=opts.names, raw=opts.Raw) + sys.exit(0) + + table = _set_table(opts.table) + + # show all db stuff <<<1 + if opts.show or opts.ShowConfig: + if opts.show: + print "music" + pprint(music.__dict__, indent=2) + print "\nrest" + pprint(music.rest.__dict__, indent=2) + + if table is not None: + print "\n", table.table() + pprint(table.__dict__, indent=2) + + if opts.ShowConfig: + print (json.dumps(config, indent=4)) + + sys.exit(0) + # >>>1 + + """ VIEW JSON FILE open, convert to json, convert to string and print it """ # <<<1 + if opts.json: + # noinspection PyBroadException + try: + print (json.dumps(json.loads(open(opts.json).read()), indent=4)) + except Exception as e: + print (traceback.format_exc()) + sys.exit(2) + sys.exit(0) + + """ Insert use json file to add record to database """ # <<<1 + if opts.action == 'i': + table.create(opts.file) + sys.exit(0) + + """ CREATE Table """ # <<<1 + if opts.action == 'c': + table.create_table() + sys.exit(0) + + """ UPDATE use json file to update db record """ # <<<1 + if opts.action == 'u': + if not opts.file or not os.path.exists(opts.file): + print "--file filename (filename exists) is required for update" + sys.exit(1) + + table.update(opts.file) + sys.exit(0) + + """ DELETE use id to delete record from db -- requres ID """ # <<<1 + if opts.action == 'd': + if not opts.id: + print "--id ID is required for delete" + sys.exit(1) + + if opts.id: + table.delete(opts.id) + sys.exit(0) + + """ DELETE ALL from table""" # <<<1 + if opts.action == 'D': + table.clean_table() + sys.exit(0) +# >>>1 diff --git a/engine/src/tools/lib/__init__.py b/engine/src/tools/lib/__init__.py new file mode 100644 index 0000000..bd50995 --- /dev/null +++ b/engine/src/tools/lib/__init__.py @@ -0,0 +1,18 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/engine/src/tools/lib/common.py b/engine/src/tools/lib/common.py new file mode 100644 index 0000000..4973f4e --- /dev/null +++ b/engine/src/tools/lib/common.py @@ -0,0 +1,73 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + + +import os +import select +import sys + + +def set_argument(arg=None, prompt=None, multiline=False): + """Return argument from file, cmd line, read from a pipe or prompt user """ + + if arg: + if os.path.isfile(arg): + f = open(arg) + message = f.readlines() + f.close() + else: + message = arg + else: + if sys.stdin in select.select([sys.stdin], [], [], .5)[0]: + message = sys.stdin.readlines() + else: + print prompt, + if multiline: + sentinel = '' + message = list(iter(raw_input, sentinel)) + else: + message = [raw_input()] + + return message + + +def list2string(message): + return ''.join(message) + + +def chop(message): + + if message.endswith('\n'): + message = message[:-1] + + return message + + +# MAIN +if __name__ == "__main__": + + msg = set_argument(sys.argv[0]) + for row in msg: + print row, + print "\n", list2string(msg) + + msg = set_argument(prompt="Message? ") + for row in msg: + print row, + print "\n", list2string(msg) diff --git a/engine/src/tools/lib/hosts.json b/engine/src/tools/lib/hosts.json new file mode 100644 index 0000000..f45096b --- /dev/null +++ b/engine/src/tools/lib/hosts.json @@ -0,0 +1,100 @@ +{ + "music": { + "dev": { + "hosts": { + "id": "openstack region a", + "a": [ "255.255.255.0", "255.255.255.1", "255.255.255.2" ], + "b": [ "255.255.255.3", "255.255.255.4", "255.255.255.5" ] + } + }, + + "ist": { + "hosts": { + "id": "ist", + "a": [ "255.255.255.6", "255.255.255.7", "255.255.255.8" ], + "b": [ "255.255.255.9", "255.255.255.10", "255.255.255.11" ] + } + }, + + "e2e": { + "hosts": { + "id": "e2e", + "a": [ "255.255.255.12", "255.255.255.13", "255.255.255.14" ], + "b": [ "255.255.255.15", "255.255.255.16", "255.255.255.17" ] + } + }, + + "music": { + "hosts": { + "id": "music dev", + "a": [ "255.255.255.18" ], + "b": [ "255.255.255.19" ] + } + } + }, + + "valet": { + "dev": [ + { + "ip": "255.255.255.20", + "fqdn": "dev1.site.onap.org" + }, + { + "ip": "255.255.255.21", + "fqdn": "dev2.site.onap.org" + }, + { + "ip": "255.255.255.22", + "fqdn": "dev3.site.onap.org" + }, + { + "ip": "255.255.255.23", + "fqdn": "dev4.site.onap.org" + } + ], + "ist": [ + { + "ip": "255.255.255.24", + "fqdn": "ist1.site.onap.org" + }, + { + "ip": "255.255.255.25", + "fqdn": "ist2.site.onap.org" + }, + { + "ip": "255.255.255.26", + "fqdn": "ist3.site.onap.org" + }, + { + "ip": "255.255.255.27", + "fqdn": "ist4.site.onap.org" + }, + { + "ip": "255.255.255.28", + "fqdn": "ist5.site.onap.org" + }, + { + "ip": "255.255.255.29", + "fqdn": "ist6.site.onap.org" + } + ], + "e2e": [ + { + "ip": "255.255.255.30", + "fqdn": "e2e1.site.onap.org" + }, + { + "ip": "255.255.255.31", + "fqdn": "e2e2.site.onap.org" + }, + { + "ip": "255.255.255.32", + "fqdn": "e2e3.site.onap.org" + }, + { + "ip": "255.255.255.33", + "fqdn": "e2e4.site.onap.org" + } + ] + } +} diff --git a/engine/src/tools/lib/song.py b/engine/src/tools/lib/song.py new file mode 100644 index 0000000..fdbba99 --- /dev/null +++ b/engine/src/tools/lib/song.py @@ -0,0 +1,153 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +"""Song is music for scripts + + This is a subclass of music that scripts can use to add an option to override: + add_args - the commandline arguments that Song uses + keyspace - to allow the same script to run vs other databases + hosts table - to allow the same script to run vs other databases on other machines + connect - login may be different for other databases +""" + + +import argparse +import json +import os +import sys + +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) # ../../.. +from valet.utils.logger import Logger +from valet.engine.db_connect.db_apis.music import Music + + +def hosts(opts, config): + """override configs hosts""" + + hosts_json = os.path.join(sys.path[-1], "tools", "lib", "hosts.json") + _hosts = json.loads(open(hosts_json).read()) + config["music"]["hosts"] = _hosts["music"][opts.env or "dev"]["hosts"]["a"] + + if opts.verbose: + print "hosts: " + str(config["music"]["hosts"]) + + +# noinspection PyBroadException +class Song(Music): + """music but with script helpers""" + + Keyspaces = { + "cw": "valet_TestDB123", + "cmw": "valet_cmw", + "pk": "valet_TestDB420", + "pk2": "valetdb2", + "pn": "pn2", + "st": "valet_saisree", + "ist": "valet_IST", + "gj": "valet_TestDB2" + } + Keyspaces.update(dict((v, v) for k, v in Keyspaces.iteritems())) # full name is valid too + + def __init__(self, opts, config, logger): + if opts.env: + hosts(opts, config) + + self.keyspace = config["db"]["keyspace"] + self.defaultKeyspace = True + + if opts.Keyspace: + if opts.Keyspace == "all": + self.keyspace = opts.Keyspace + else: + self.keyspace = Song.Keyspaces[opts.Keyspace] + self.defaultKeyspace = False + + if opts.db: + self.keyspace = opts.db + self.defaultKeyspace = False + + # TODO cmw: move keyspace into music object, pass in like config["keyspace"] = self.keyspace + + super(Song, self).__init__(config, logger) + + @staticmethod + def add_args(parser): + """add common parser arguments""" + default_config = "/opt/config/solver.json" + if not os.path.isfile(default_config): + default_config = os.path.join(sys.path[-1], "config", "solver.json") + + valid_keyspaces = Song.Keyspaces.keys() + valid_keyspaces.append("all") + valid_keyspaces_str = "{" + ",".join(valid_keyspaces) + "}" + + valid_hosts = ["a1", "a2", "a3", "b3", "ab", "m"] + valid_env = ["dev", "ist", "e2e"] + + song_args = parser.add_argument_group("Common Music Arguments") + song_args.add_argument('-env', metavar=valid_env, help='pick set of hosts -deprecated', choices=valid_env) + song_args.add_argument('-host', metavar=valid_hosts, help='pick set of hosts -deprecated', choices=valid_hosts) + ex = song_args.add_mutually_exclusive_group() + ex.add_argument('-Keyspace', metavar=valid_keyspaces_str, help='override configs keyspace with a users', choices=valid_keyspaces) + ex.add_argument('-db', metavar='keyspace_string', help='override keyspace with typed in value') + song_args.add_argument('-config', metavar='file', default=default_config, help="default: " + default_config) + song_args.add_argument('-verbose', action='store_true', help="verbose output") + + def create_keyspace(self, keyspace): + """override creates a keyspace.""" + + data = { + 'replicationInfo': { + "DC2": 3, + "DC1": 3, + "class": "NetworkTopologyStrategy" + }, + 'durabilityOfWrites': True, + 'consistencyInfo': { + 'type': 'eventual', + }, + } + + path = '/keyspaces/%s' % keyspace + try: + self.rest.request(method='post', path=path, data=data) + return 0 + except Exception: + # "this exception should be handled here but it's done in music :(" + return -1 + + +def main(): + parser = argparse.ArgumentParser(description='Extended Music DB.', add_help=False) + Song.add_args(parser) + parser.add_argument("-?", "--help", action="help", help="show this help message and exit") + opts = parser.parse_args() + + logger = Logger().get_logger('console') + + config = json.loads(open(opts.config).read()) + music = Song(opts, config, logger) + + print json.dumps(config.get("music")) + print (music.keyspace) + + +# MAIN +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/engine/src/tools/lib/tables.py b/engine/src/tools/lib/tables.py new file mode 100644 index 0000000..7ad4836 --- /dev/null +++ b/engine/src/tools/lib/tables.py @@ -0,0 +1,308 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + + +import json +import os +import sys +import time +from datetime import datetime +from textwrap import TextWrapper + +import pytz + + +class Tables(object): + """parent class for all tables.""" + + schema = None + alias = None + + def __init__(self, music, logger): + """Initializer. Accepts target host list, port, and path.""" + + self.music = music + self.logger = logger + self.key = None + self.keyspace = music.keyspace + self.tz = pytz.timezone('America/New_York') + + @classmethod + def table_alias(cls): + s = cls.__name__ + s = s[0].lower() + s[1:] + return [s] + cls.alias + + @staticmethod + def option_choices(): + choices = [] + aliases = None + for table in Tables.__subclasses__(): + choices = choices + table.table_alias() + for alias in choices: + if aliases: + aliases = aliases + ' ' + alias + else: + aliases = alias + + choices.append('all') + + return { + "alias": TextWrapper(subsequent_indent=" ", initial_indent=" ", width=79).fill(aliases), + "choices": choices + } + + def table(self): + """Return tables name (same as class name but all lc).""" + + s = type(self).__name__ + return s[0].lower() + s[1:] + + def utc2local(self, utc): + """Change utc time to local time for readability.""" + + return " (" + datetime.fromtimestamp(utc, self.tz).strftime('%Y-%m-%d %I:%M:%S %Z%z') + ")" + + def get_rows(self, ids=None, raw=False): + """ get_rows read table, or rows by id and return rows array """ + + # TODO if ids == None and hasattr(self, 'ids'): ids = self.ids + key = None if (ids is None) else self.key + rows = [] + + for row_id in ids or [None]: + if raw: + rows.append(json.dumps(self.music.read_row(self.keyspace, self.table(), key, row_id), sort_keys=True, indent=4)) + continue + + result = self.music.read_row(self.keyspace, self.table(), key, row_id)["result"] + + # strip "Row n" + for _, data in sorted(result.iteritems()): + rows.append(data) + + if raw: + return rows + + if len(rows) == 1: + rows = rows[0] # one row? not a list + + return rows + + def read(self, ids=None, json_file=None, raw=False, rows=None, names=None): + """ read rows (array or single dict) to stdout or to a file """ + + if rows is None: + rows = self.get_rows(ids, raw) + + if raw: + if names: + print "\n" + self.table() + for row in rows: + print row + return + + if isinstance(rows, list): + for row in rows: + if not ("timestamp" in row or "expire_time" in row): + break + for key in (["timestamp", "expire_time"]): + if (not (key in row)) or (row[key] is None): + continue + try: + row[key] = row[key] + self.utc2local(float(row[key])/1000) + except ValueError: + row[key] = "Error: "+ row[key] + + else: + row = rows + for key in (["timestamp", "expire_time"]): + if (not (key in row)) or (row[key] is None): + continue + try: + row[key] = row[key] + self.utc2local(float(row[key])/1000) + except ValueError: + row[key] = "Error: "+ row[key] + + if json_file is None: + if names: + print "\n" + self.table() + print json.dumps(rows, sort_keys=True, indent=4) + return + + fh = open(json_file, "w") + fh.write(json.dumps(rows, sort_keys=True, indent=4)) + fh.close() + + def create(self, json_file=None): + """ add records from a file to db """ + + if json_file and os.path.exists(json_file): + inf = open(json_file) + f = json_file + else: + inf = sys.stdin + f = "stdin" + + self.logger.info("Create " + self.table() + " from: " + f) + self.insert(json.loads(inf.read())) + + def insert(self, data): + """ add records """ + + self.logger.debug(data) + + if isinstance(data, list): + for row in data: + if "timestamp" in row: + row['timestamp'] = int(round(time.time() * 1000)) + self.music.create_row(self.keyspace, self.table(), row) + else: + row = data + if "timestamp" in row: + row['timestamp'] = int(round(time.time() * 1000)) + self.music.create_row(self.keyspace, self.table(), row) + + def create_table(self): + """ create table """ + + self.logger.info(self.schema) + self.music.create_table(self.keyspace, self.table(), self.schema) + + def update(self, json_file): + """Update a row. Not atomic.""" + + self.logger.info("Update " + self.table() + " from: " + json_file) + data = json.loads(open(json_file).read()) + self.logger.debug(data) + + if isinstance(data, list): + for row in data: + self.music.update_row_eventually(self.keyspace, self.table(), row) + else: + self.music.update_row_eventually(self.keyspace, self.table(), data) + + def delete(self, ids): + """ delete records db based on id """ + + for row_id in ids: + self.logger.info("Delete from" + self.table() + " id: " + row_id) + self.music.delete_row_eventually(self.keyspace, self.table(), self.key, row_id) + + def clean_table(self): + """ delete all records in table """ + + ids = [] + rows = self.get_rows() + + if isinstance(rows, list): + for row in rows: + ids.append(row[self.key]) + else: + row = rows + ids.append(row[self.key]) + + for row_id in ids: + self.music.delete_row_eventually(self.keyspace, self.table(), self.key, row_id) + +# Subclasses of Tables: + + +class Requests(Tables): + alias = ["req", "q"] + key = "request_id" + schema = json.loads('{ "request_id": "text", "timestamp": "text", "request": "text", "PRIMARY KEY": "(request_id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Requests.key + + +class Results(Tables): + alias = ["resu", "u"] + key = "request_id" + schema = json.loads('{ "request_id": "text", "status": "text", "timestamp": "text", "result": "text", "PRIMARY KEY": "(request_id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Results.key + +class Group_rules(Tables): + alias = ["rule", "gr"] + key = "id" + schema = json.loads('{ "id": "text", "app_scope": "text", "type": "text", "level": "text", "members": "text", "description": "text", "groups": "text", "status": "text", "timestamp": "text", "PRIMARY KEY": "(id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Group_rules.key + + +class Stacks(Tables): + alias = ["stack", "s"] + key = "id" + schema = json.loads('{ "id": "text", "last_status": "text", "datacenter": "text", "stack_name": "text", "uuid": "text", "tenant_id": "text", "metadata": "text", "servers": "text", "prior_servers": "text", "state": "text", "prior_State": "text", "timestamp": "text", "PRIMARY KEY": "(id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Stacks.key + + +class Stack_id_map(Tables): + alias = ["map", "m"] + key = "request_id" + schema = json.loads('{ "request_id": "text", "stack_id": "text", "timestamp": "text", "PRIMARY KEY": "(request_id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Stack_id_map.key + + +class Resources(Tables): + alias = ["reso", "o"] + key = "id" + schema = json.loads('{ "id": "text", "url": "text", "resource": "text", "timestamp": "text", "requests": "text", "PRIMARY KEY": "(id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Resources.key + + +class Regions(Tables): + alias = ["reg", "i", "lock"] + key = "region_id" + schema = json.loads('{ "region_id ": "text", "timestamp": "text", "last_updated ": "text", "keystone_url": "text", "locked_by": "text", "locked_time ": "text", "expire_time": "text", "PRIMARY KEY": "(region_id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Regions.key + + +class Groups(Tables): + alias = ["group", "g"] + key = "id" + schema = json.loads('{ "id ": "text", "uuid": "text", "type ": "text", "level": "text", "factory": "text", "rule_id ": "text", "metadata ": "text", "server_list": "text", "member_hosts": "text", "status": "text", "PRIMARY KEY": "(id)" }') + + def __init__(self, music, logger): + Tables.__init__(self, music, logger) + self.key = Groups.key + + +if __name__ == "__main__": + + print Tables.option_choices() diff --git a/engine/src/tools/lock.py b/engine/src/tools/lock.py new file mode 100644 index 0000000..a77eb8a --- /dev/null +++ b/engine/src/tools/lock.py @@ -0,0 +1,92 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +#!/usr/bin/env python2.7 + + +import argparse +import json +import os +import sys + +import lib.tables as tables +from lib.song import Song + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from valet.engine.db_connect.db_handler import DBHandler +from valet.engine.db_connect.locks import Locks, later +from valet.utils.logger import Logger + + +def options(): + parser = argparse.ArgumentParser(description='\033[34mManual Locking And Unlocking Of Valet Regions.\033[0m', add_help=False) + Song.add_args(parser) + + g1 = parser.add_argument_group('Control Music Locks') + ex = g1.add_mutually_exclusive_group() + ex.add_argument('-unlock', metavar="region", help='unlock this region') + ex.add_argument('-lock', metavar="region", help='lock this region') + + g2 = parser.add_argument_group('Update Locks In Region Table') + ex = g2.add_mutually_exclusive_group() + ex.add_argument('-delete', metavar="region", help='delete region from table') + ex.add_argument('-add', metavar="region", help='update/add region to table') + g2.add_argument('-timeout', metavar="seconds", help='seconds till region expires (for -add)') + + group = parser.add_argument_group("Change The Output") + group.add_argument("-?", "--help", action="help", help="show this help message and exit") + group.add_argument('-show', action='store_true', help='print out regions (locking) table') + + return parser.parse_args() + + +# MAIN +if __name__ == "__main__": + opts = options() + + logger = Logger().get_logger('console') + config = json.loads(open(opts.config).read()) + music_config = config.get("music") + music_config["keyspace"] = config.get("db")["keyspace"] + music = Song(opts, config, logger) + dbh = DBHandler(music, config.get("db"), logger) + + if opts.add: + timeout = opts.timeout if opts.timeout else config["engine"]["timeout"] + dbh.add_region(opts.add, later(seconds=int(timeout))) + logger.debug("added region to table") + + if opts.lock: + if Locks(dbh, 0).got_lock(opts.lock) == "ok": + logger.debug("added region lock") + else: + logger.debug("failed to add region lock") + + if opts.unlock: + Locks.unlock(dbh, opts.unlock) + logger.debug("deleted region lock '%s'" % opts.unlock) + + if opts.delete: + dbh.delete_region(opts.delete) + logger.debug("deleted region from table") + + if opts.show: + music.keyspace = dbh.keyspace + tables.Regions(music, logger).read(names=True) + + sys.exit(0) diff --git a/engine/src/tools/ppdb.py b/engine/src/tools/ppdb.py new file mode 100644 index 0000000..c89f0f2 --- /dev/null +++ b/engine/src/tools/ppdb.py @@ -0,0 +1,91 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +#!/usr/bin/env python2.7 + + +import sys +import os +import json +import argparse + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='parse results.') + parser.add_argument('-f', '--file', help='json file required for create/view') + parser.add_argument('-verbose', action='store_true', help='more output than you want') + opts = parser.parse_args() + + if opts.file and os.path.exists(opts.file): + inf = open(opts.file) + else: + inf = sys.stdin + + results = json.loads(inf.read()) + if opts.verbose: + print (json.dumps(results, sort_keys=True, indent=4)) + print "---------------------------------------------" + + if "request" in results.keys(): + key = "request" + result = json.loads(results[key]) + print (json.dumps(result, sort_keys=True, indent=4)) + sys.exit(0) + + if "result" in results.keys(): + result = results["result"] + + if not isinstance(result, list): + sys.stdout.write("result ") + sys.stdout.flush() + result = json.loads(result) + print (json.dumps(result, sort_keys=True, indent=4)) + sys.exit(0) + + for _, row in result.iteritems(): + rr = json.loads(row["result"]) + + # for k, d in row.iteritems(): + # print ("%s) %s"% (k, d)) + + sys.stdout.write("result ") + sys.stdout.flush() + print json.dumps(rr, indent=4) + # for f in rr: + # for line in (json.dumps(f, sort_keys=True, indent=4)).splitlines(): + # print "\t%s"%line + # print "}" + + sys.exit(0) + + if "resource" in results.keys(): + key = "resource" + result = json.loads(results[key]) + print (json.dumps(result, sort_keys=True, indent=4)) + + if not isinstance(result, list): + sys.stdout.write("resource ") + sys.stdout.flush() + result = json.loads(result) + print (json.dumps(result, sort_keys=True, indent=4)) + sys.exit(0) + + print (json.dumps(result, sort_keys=True, indent=4)) + sys.exit(0) + + print (results.keys()) |