diff options
Diffstat (limited to 'asdctool/src/main/resources/scripts/python')
5 files changed, 576 insertions, 0 deletions
diff --git a/asdctool/src/main/resources/scripts/python/duplicates.py b/asdctool/src/main/resources/scripts/python/duplicates.py new file mode 100644 index 0000000000..be60b05909 --- /dev/null +++ b/asdctool/src/main/resources/scripts/python/duplicates.py @@ -0,0 +1,47 @@ +import json +import sys + +dict = {} +dupliacteUid = {} +#debugFlag = True +debugFlag = False + +def debug(str1, str2=""): + 'print only if debug enabled' + if (debugFlag == True): print str1, str2 + +print 'Number of arguments:', len(sys.argv), 'arguments.' + + +with open(sys.argv[1]) as json_file: + json_data = json.load(json_file) + for x in json_data['vertices']: + uid = None + nodeLabel=x.get('nodeLabel') + debug(nodeLabel) + if ( nodeLabel == 'user' ): + uid = x['userId'] + elif ( nodeLabel == 'tag' ): + uid = x['name'] + elif ( nodeLabel == None ): + pass + elif ( nodeLabel == 'lockNode' ): + uid = x.get('uid') + else: uid = x['uid'] + + debug(nodeLabel, uid) + + existId = dict.get(uid) + if (existId == None): + dict[uid] = x.get('_id') + else: + dupliacteUid[uid] = existId + + print dupliacteUid + +# with open('data.txt', 'w') as outfile: +# json.dump(json_data, outfile) + + + +# print x['uid'] diff --git a/asdctool/src/main/resources/scripts/python/duplicatesAndRemove.py b/asdctool/src/main/resources/scripts/python/duplicatesAndRemove.py new file mode 100644 index 0000000000..a4bd35dd2b --- /dev/null +++ b/asdctool/src/main/resources/scripts/python/duplicatesAndRemove.py @@ -0,0 +1,136 @@ +import json +import sys, getopt +from collections import OrderedDict + +dict = {} +dupliacteUid = {} +#debugFlag = True +debugFlag = False + +def join_strings(lst): + concat = "" + for string in lst: + if (string != None): + if (type(string) == int): + string = str(string) + concat += (string + " ") + return concat + +def debug(desc, *args): + 'print only if debug enabled' + if (debugFlag == True): + print desc, join_strings(args) + +def log(desc, arg): + 'print log info' + print desc, arg + +def getUid(vertex): + uid = None + nodeLabel=vertex.get('nodeLabel') + debug(nodeLabel) + if ( nodeLabel == 'user' ): + uid = vertex['userId'] + elif ( nodeLabel == 'tag' ): + uid = vertex['name'] + elif ( nodeLabel == None ): + pass + elif ( nodeLabel == 'lockNode' ): + uid = vertex.get('uid') + else: uid = vertex['uid'] + + debug(nodeLabel, uid) + + return uid + +def generateFile(inputFile, outputFile): + + with open(inputFile) as json_file: + dupliacteUid = {} + json_data = json.load(json_file) + for x in json_data['vertices']: + uid = getUid(x) + + existId = dict.get(uid) + if (existId == None): + dict[uid] = x.get('_id') + else: + dupliacteUid[uid] = existId + + log("duplicate ids", dupliacteUid) + + json_data_vertices = json_data['vertices'] + log("number of vertices is", len(json_data_vertices)) + + ids = {} + deleteIndexes = [] + + for i in xrange(len(json_data_vertices)): + #print "****** ", i, " *************" + #print json_data_vertices[i] + id = json_data_vertices[i]["_id"] + uid = getUid(json_data_vertices[i]) + isDuplicateId = dupliacteUid.get(uid) + if (isDuplicateId != None): + debug("uid to id pair", uid if uid != None else 'None', id) + value = ids.get(uid) + if (value == None): + list = [id,] + ids[uid] = list + else: + value.append(id) + deleteIndexes.append(id) + + log("ids", ids) + log("deleteIndexes", deleteIndexes) + log("deleteIndexes size", len(deleteIndexes)) + + filter_vertex = [ x for x in json_data_vertices if x.get('_id') not in deleteIndexes ] + json_data['vertices'] = filter_vertex + + log("number of vertexes after filter", len(filter_vertex)) + + json_data_edges = json_data['edges'] + + log("number of edges", len(json_data_edges)) + + filter_edge = [ x for x in json_data_edges if x['_outV'] not in (deleteIndexes) and x['_inV'] not in (deleteIndexes) ] + json_data['edges'] = filter_edge + + log("number of edges after filter", len(json_data['edges'])) + + json_data = OrderedDict(sorted(json_data.items(), key=lambda t: t[0], reverse=True)) + + with open(outputFile, 'w') as outfile: + #json.dump(json_data, outfile) + json.dump(json_data, outfile) + log("output file is", outputFile); + +def main(argv): + print 'Number of arguments:', len(sys.argv), 'arguments.' + inputfile = None + outputfile = '' + try: + opts, args = getopt.getopt(argv,"h:i:o:",["ifile=","ofile="]) + except getopt.GetoptError: + print sys.argv[0], '-i <inputfile>' + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print sys.argv[0], '-i <inputfile>' + sys.exit(3) + elif opt in ("-i", "--ifile"): + inputfile = arg + + if ( inputfile == None ): + print sys.argv[0] ,'-i <inputfile>' + sys.exit(3) + + print 'Input file is "', inputfile + generateFile(inputfile, inputfile + '.noduplicates') + + +if __name__ == "__main__": + main(sys.argv[1:]) + +# print x['uid'] diff --git a/asdctool/src/main/resources/scripts/python/graphSize.py b/asdctool/src/main/resources/scripts/python/graphSize.py new file mode 100644 index 0000000000..6c36ec6844 --- /dev/null +++ b/asdctool/src/main/resources/scripts/python/graphSize.py @@ -0,0 +1,56 @@ +import json +import sys, getopt + +dict = {} +dupliacteUid = {} +#debugFlag = True +debugFlag = False + +def debug(desc, *args): + 'print only if debug enabled' + if (debugFlag == True): + print desc, join_strings(args) + +def log(desc, arg): + 'print log info' + print desc, arg + +def graphSize(inputFile): + + with open(inputFile) as json_file: + json_data = json.load(json_file) + + json_data_vertices = json_data['vertices'] + log("number of vertices is", len(json_data_vertices)) + + json_data_edges = json_data['edges'] + log("number of edges is", len(json_data_edges)) + +def main(argv): + print 'Number of arguments:', len(sys.argv), 'arguments.' + inputfile = None + outputfile = '' + try: + opts, args = getopt.getopt(argv,"h:i:o:",["ifile=","ofile="]) + except getopt.GetoptError: + print sys.argv[0], '-i <inputfile>' + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print sys.argv[0], '-i <inputfile>' + sys.exit(3) + elif opt in ("-i", "--ifile"): + inputfile = arg + + if ( inputfile == None ): + print sys.argv[0], '-i <inputfile>' + sys.exit(3) + + print 'Input file is ', inputfile + graphSize(inputfile) + + +if __name__ == "__main__": + main(sys.argv[1:]) + +# print x['uid'] diff --git a/asdctool/src/main/resources/scripts/python/user/exportUsers.py b/asdctool/src/main/resources/scripts/python/user/exportUsers.py new file mode 100644 index 0000000000..e32a3b0a21 --- /dev/null +++ b/asdctool/src/main/resources/scripts/python/user/exportUsers.py @@ -0,0 +1,122 @@ +import pycurl +import sys, getopt +from StringIO import StringIO +import json + + +################################################################################################################################################ +# # +# Export all active users to file - for 1602+ # +# # +# activation : # +# python exportUsers.py [-i <be host> | --ip=<be host>] [-p <be port> | --port=<be port> ] [-f <output file> | --ofile=<output file> ] # +# # +# shortest activation (be host = localhost, be port = 8080): # # +# python exportUsers.py [-f <output file> | --ofile=<output file> ] # +# # +################################################################################################################################################ + +ALL_USERS_SUFFIX = '/sdc2/rest/v1/user/users' + +def errorAndExit(errorCode, errorDesc): + if ( errorCode > 0 ): + print("status=" + str(errorCode) + ". " + errorDesc) + else: + print("status=" + str(errorCode)) + sys.exit(errorCode) + +def getUsers(beHost, bePort, adminUser): + + try: + buffer = StringIO() + c = pycurl.Curl() + + url = 'http://' + beHost + ':' + bePort + ALL_USERS_SUFFIX + print(url) + c.setopt(c.URL, url) + c.setopt(c.WRITEFUNCTION, buffer.write) + #c.setopt(c.WRITEFUNCTION, lambda x: None) + adminHeader = 'USER_ID: ' + adminUser + c.setopt(pycurl.HTTPHEADER, ['Content-Type: application/json', 'Accept: application/json', adminHeader]) + res = c.perform() + #print(res) + + #print('Status: %d' % c.getinfo(c.RESPONSE_CODE)) + + c.close() + + body = buffer.getvalue() + + #print(body) + + return (body, None) + + except Exception as inst: + print inst + #print type(inst) # the exception instance + #print inst.args # arguments stored in .args + #print inst # __str__ allows args to be printed directly + #x, y = inst.args + #print 'x =', x + + return (None, inst) + + +def usage(): + print sys.argv[0], '[-i <be host> | --ip=<be host>] [-p <be port> | --port=<be port> ] [-f <output file> | --ofile=<output file> ]' + +def main(argv): + print 'Number of arguments:', len(sys.argv), 'arguments.' + + adminHeader = 'jh0003' + beHost = 'localhost' + bePort = '8080' + outputfile = None + + try: + opts, args = getopt.getopt(argv,"i:p:f:h:",["ip=","port=","ofile="]) + except getopt.GetoptError: + usage() + errorAndExit(2, 'Invalid input') + + for opt, arg in opts: + #print opt, arg + if opt == '-h': + usage() + sys.exit(3) + elif opt in ("-i", "--ip"): + beHost = arg + elif opt in ("-p", "--port"): + bePort = arg + elif opt in ("-f", "--ofile"): + outputfile = arg + + print 'be host =',beHost,', be port =', bePort,', output file =',outputfile + + if ( outputfile == None ): + usage() + sys.exit(3) + + users = getUsers(beHost, bePort, adminHeader) + error = users[1] + body = users[0] + + if ( error != None ): + errorAndExit(5, str(error)) + + #print body + + io = StringIO(body) + usersAsJson = json.load(io) + + writeFile = open(outputfile, 'w') + + json.dump(usersAsJson, writeFile) + + writeFile.close() + + print("-------------------------------------------") + errorAndExit(0, None) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/asdctool/src/main/resources/scripts/python/user/importUsers.py b/asdctool/src/main/resources/scripts/python/user/importUsers.py new file mode 100644 index 0000000000..669cbbe6f2 --- /dev/null +++ b/asdctool/src/main/resources/scripts/python/user/importUsers.py @@ -0,0 +1,215 @@ +import pycurl +import sys, getopt +from StringIO import StringIO +import json +import copy + +################################################################################################################################################ +# # +# Import all users from a given file # +# # +# activation : # +# python importUsers.py [-i <be host> | --ip=<be host>] [-p <be port> | --port=<be port> ] [-f <input file> | --ifile=<input file> ] # +# # +# shortest activation (be host = localhost, be port = 8080): # # +# python importUsers.py [-f <input file> | --ifile=<input file> ] # +# # +################################################################################################################################################ + + +def importUsers(beHost, bePort, users, adminUser): + + result = [] + + for user in users: + + #print("Going to add user " + user['userId']) + + getRes = getUser(beHost, bePort, user) + userId = getRes[0] + error = getRes[1] + #print error + if ( error != None and error == 404 ): + res = createUser(beHost, bePort, user ,adminUser) + result.append(res) + else: + if ( error == 200 ): + curResult = (userId, 409) + result.append(curResult) + else: + result.append(getRes) + + return result + + +def convertUsersToCreationObject(users): + + cloneUsers = copy.deepcopy(users) + for user in cloneUsers: + #print user + if (user.get('fullName') != None): + del user['fullName'] + #user['userId'] = user['userId'] + '1' + #print user + + return cloneUsers + +def getUser(beHost, bePort, user): + + userId = user['userId'] + try: + buffer = StringIO() + c = pycurl.Curl() + + #print type(userId) + url = 'http://' + beHost + ':' + bePort + '/sdc2/rest/v1/user/' + str(userId) + c.setopt(c.URL, url) + + #adminHeader = 'USER_ID: ' + adminUser + c.setopt(pycurl.HTTPHEADER, ['Content-Type: application/json', 'Accept: application/json']) + c.setopt(c.WRITEFUNCTION, lambda x: None) + res = c.perform() + + #print("Before get response code") + httpRes = c.getinfo(c.RESPONSE_CODE) + #print("After get response code") + responseCode = c.getinfo(c.RESPONSE_CODE) + + #print('Status: ' + str(responseCode)) + + c.close() + + return (userId, httpRes) + + except Exception as inst: + print(inst) + return (userId, None) + + + +def createUser(beHost, bePort, user, adminUser): + + userId = user['userId'] + try: + buffer = StringIO() + c = pycurl.Curl() + + url = 'http://' + beHost + ':' + bePort + '/sdc2/rest/v1/user' + c.setopt(c.URL, url) + c.setopt(c.POST, 1) + + adminHeader = 'USER_ID: ' + adminUser + c.setopt(pycurl.HTTPHEADER, ['Content-Type: application/json', 'Accept: application/json', adminHeader]) + + data = json.dumps(user) + c.setopt(c.POSTFIELDS, data) + + c.setopt(c.WRITEFUNCTION, lambda x: None) + #print("before perform") + res = c.perform() + #print(res) + + #print("Before get response code") + httpRes = c.getinfo(c.RESPONSE_CODE) + #print("After get response code") + responseCode = c.getinfo(c.RESPONSE_CODE) + + #print('Status: ' + str(responseCode)) + + c.close() + + return (userId, httpRes) + + except Exception as inst: + print(inst) + return (userId, None) + + +def errorAndExit(errorCode, errorDesc): + if ( errorCode > 0 ): + print("status=" + str(errorCode) + ". " + errorDesc) + else: + print("status=" + str(errorCode)) + sys.exit(errorCode) + +def usage(): + print sys.argv[0], '[-i <be host> | --ip=<be host>] [-p <be port> | --port=<be port> ] [-f <input file> | --ifile=<input file> ]' + +def main(argv): + print 'Number of arguments:', len(sys.argv), 'arguments.' + + beHost = 'localhost' + bePort = '8080' + inputfile = None + + adminUser = 'jh0003' + + try: + opts, args = getopt.getopt(argv,"i:p:f:h:",["ip=","port=","ifile="]) + except getopt.GetoptError: + usage() + errorAndExit(2, 'Invalid input') + + for opt, arg in opts: + #print opt, arg + if opt == '-h': + usage() + sys.exit(3) + elif opt in ("-i", "--ip"): + beHost = arg + elif opt in ("-p", "--port"): + bePort = arg + elif opt in ("-f", "--ifile"): + inputfile = arg + + print 'be host =',beHost,', be port =', bePort,', users file =',inputfile + + if ( inputfile == None ): + usage() + sys.exit(3) + + print 'Input file is ', inputfile + + usersFile = open(inputfile) + + json_data = json.load(usersFile) + + #print json_data + + cloneUsers = convertUsersToCreationObject(json_data) + + activeUsers = filter(lambda x: x['status'] == 'ACTIVE', cloneUsers) + + #print activeUsers + + resultTable = importUsers(beHost, bePort, activeUsers, adminUser) + + g = lambda x: x[1] != 201 and x[1] != 409 + + result = filter(g, resultTable) + + if ( len(result) > 0 ): + #print("ERROR: Failed to load the users " + ', '.join(map(lambda x: x[0],result))) + errorAndExit(3, "Failed to load the users " + ', '.join(map(lambda x: x[0],result))) + + g = lambda x: x[1] == 409 + result = filter(g, resultTable) + + print("-------------------------------------------") + print("Existing users: " + ', '.join(map(lambda x: x[0],result))) + + result = filter(lambda x: x[1] == 201, resultTable) + if ( len(result) == 0 ): + print("-------------------------------------------") + print("No NEW user was loaded. All users are already exist") + print("-------------------------------------------") + else: + print("-------------------------------------------") + print("Loaded users: " + ', '.join(map(lambda x: x[0],result))) + print("-------------------------------------------") + + errorAndExit(0, None) + + +if __name__ == "__main__": + main(sys.argv[1:]) |