diff options
Diffstat (limited to 'ansible-server/src')
9 files changed, 460 insertions, 354 deletions
diff --git a/ansible-server/src/main/Dockerfile b/ansible-server/src/main/Dockerfile index 40de69e6..d44f5c39 100644 --- a/ansible-server/src/main/Dockerfile +++ b/ansible-server/src/main/Dockerfile @@ -29,7 +29,7 @@ WORKDIR /opt/ansible-server RUN mkdir /opt/onap ; ln -s /opt/ansible-server /opt/onap/ccsdk RUN echo > /var/log/ansible-server.log -RUN chown ansible:ansible /var/log/ansible-server.log +RUN chown -R ansible:ansible /var/log RUN chown ansible:ansible /opt/ansible-server USER ansible:ansible diff --git a/ansible-server/src/main/ansible-server/AnsibleModule.py b/ansible-server/src/main/ansible-server/AnsibleModule.py index 2da79129..180373f4 100755 --- a/ansible-server/src/main/ansible-server/AnsibleModule.py +++ b/ansible-server/src/main/ansible-server/AnsibleModule.py @@ -19,7 +19,6 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* ECOMP is a trademark and service mark of AT&T Intellectual Property. * ============LICENSE_END========================================================= */ ''' @@ -52,11 +51,11 @@ def ansibleSysCall (inventory_path, playbook_path, nodelist, mandatory, str_parameters += '"' if len(str_parameters) > 0: - cmd = 'export HOME=/root; env; cd ' + playbookdir + ';' +'timeout -s KILL -t ' + str(timeout) + \ + cmd = 'export HOME=/root; env; cd ' + playbookdir + ';' +'timeout --signal=KILL ' + str(timeout) + \ ' ansible-playbook -v --timeout ' + str(timeout) + ' --extra-vars ' + str_parameters + ' -i ' + \ inventory_path + ' ' + playbook_path + ' | tee log.file' else: - cmd = 'export HOME=/root; env; cd ' + playbookdir + ';' +'timeout -s KILL -t ' + str(timeout) + \ + cmd = 'export HOME=/root; env; cd ' + playbookdir + ';' +'timeout --signal=KILL ' + str(timeout) + \ ' ansible-playbook -v --timeout ' + str(timeout) + ' -i ' + inventory_path + ' ' + playbook_path +' | tee log.file' cherrypy.log("CMD: " + cmd) diff --git a/ansible-server/src/main/ansible-server/AnsibleSql.py b/ansible-server/src/main/ansible-server/AnsibleSql.py index ab58a96c..c910777a 100755 --- a/ansible-server/src/main/ansible-server/AnsibleSql.py +++ b/ansible-server/src/main/ansible-server/AnsibleSql.py @@ -19,7 +19,6 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* ECOMP is a trademark and service mark of AT&T Intellectual Property. * ============LICENSE_END========================================================= */ ''' @@ -29,7 +28,7 @@ from os import listdir from os.path import isfile, join class mySql(): - + def __init__(self, myhost, myuser, mypasswd, mydb): self.con = True self.error = '' diff --git a/ansible-server/src/main/ansible-server/BuildHostFile.py b/ansible-server/src/main/ansible-server/BuildHostFile.py index 20bbc904..a49f94aa 100755 --- a/ansible-server/src/main/ansible-server/BuildHostFile.py +++ b/ansible-server/src/main/ansible-server/BuildHostFile.py @@ -19,7 +19,6 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* ECOMP is a trademark and service mark of AT&T Intellectual Property. * ============LICENSE_END========================================================= */ ''' diff --git a/ansible-server/src/main/ansible-server/LoadAnsibleMySql.py b/ansible-server/src/main/ansible-server/LoadAnsibleMySql.py index 0a1c78a6..d3c0ecef 100755 --- a/ansible-server/src/main/ansible-server/LoadAnsibleMySql.py +++ b/ansible-server/src/main/ansible-server/LoadAnsibleMySql.py @@ -19,7 +19,6 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* ECOMP is a trademark and service mark of AT&T Intellectual Property. * ============LICENSE_END========================================================= */ ''' diff --git a/ansible-server/src/main/ansible-server/README b/ansible-server/src/main/ansible-server/README index 9aff2c01..555dadda 100644 --- a/ansible-server/src/main/ansible-server/README +++ b/ansible-server/src/main/ansible-server/README @@ -19,7 +19,6 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* ECOMP is a trademark and service mark of AT&T Intellectual Property. * ============LICENSE_END========================================================= */ ''' diff --git a/ansible-server/src/main/ansible-server/RestServer.py b/ansible-server/src/main/ansible-server/RestServer.py index d5f85d1a..65f73986 100644 --- a/ansible-server/src/main/ansible-server/RestServer.py +++ b/ansible-server/src/main/ansible-server/RestServer.py @@ -1,10 +1,9 @@ ''' -#!/usr/bin/python /*- * ============LICENSE_START======================================================= * ONAP : APPC * ================================================================================ -* Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +* Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Copyright (C) 2017 Amdocs * ============================================================================= @@ -19,8 +18,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -* -* ECOMP is a trademark and service mark of AT&T Intellectual Property. +* * ============LICENSE_END========================================================= */ ''' @@ -29,6 +27,8 @@ import time, datetime, json, os, sys, subprocess, re import uuid import tarfile import shutil +import glob +import crypt import requests @@ -39,9 +39,7 @@ from cherrypy.lib import auth_basic from multiprocessing import Process, Manager from AnsibleModule import ansibleSysCall - -import AnsibleSql -from AnsibleSql import readPlaybook, readCredentials +from BuildHostFile import buildHostsSysCall from os import listdir from os.path import isfile, join @@ -49,6 +47,12 @@ from os.path import isfile, join TestRecord = Manager().dict() ActiveProcess = {} +def validate_password(realm, username, password): + comp = crypt.crypt(password, salt) + if username in userpassdict and userpassdict[username] == comp: + return True + return False + def sys_call (cmd): p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = p.stdout.readlines() @@ -76,7 +80,6 @@ def callback (Id, Result, Output, Log, returncode): # Need to update the whole data structure for key=Id otherwise Manager is not updated TestRecord[Id] = {'PlaybookName': TestRecord[Id]['PlaybookName'], - 'LCM': TestRecord[Id]['LCM'], 'Version': TestRecord[Id]['Version'], 'NodeList': TestRecord[Id]['NodeList'], 'HostGroupList': TestRecord[Id]['HostGroupList'], @@ -105,6 +108,9 @@ def callback (Id, Result, Output, Log, returncode): "Duration": TestRecord[Id]["Duration"], "Results": TestRecord[Id]['Result']['Results']} + cherrypy.log("CALLBACK: TestRecord[Id]['Output']['Output']:", str(TestRecord[Id]['Output']['Output'])) + cherrypy.log("CALLBACK: Results:", str(data["Results"])) + if not TestRecord[Id]['Output']['Output'] == {}: for key in data["Results"]: if key in TestRecord[Id]['Output']['Output']: @@ -129,27 +135,36 @@ def RunAnsible_Playbook (callback, Id, Inventory, Playbook, NodeList, TestRecord TestRecord[Id]['Mandatory'], TestRecord[Id]['EnvParameters'], TestRecord[Id]['LocalParameters'], - TestRecord[Id]['LCM'], - TestRecord[Id]['Timeout']) + TestRecord[Id]['Timeout'], + Path) - print " returncode:", returncode - print " retval: ", retval - print " log: ", log + cherrypy.log("Return code:" + str(returncode)) + cherrypy.log("Return val:" + str(retval)) Log = ''.join(log) - Output = {'Output': {}} + #Output = {'Output': {}} + Output = {} onlyfiles = [f for f in listdir(Path) if isfile(join(Path, f))] + cherrypy.log("Checking for results.txt files: ") for file in onlyfiles: if "results.txt" in file: +# if file.endswith("results.txt"): + cherrypy.log("results file: " + file) f = open(Path + "/" + file, "r") - key = file.split("_")[0] - Output['Output'][key] = f.read() + resultsData = f.read() # Not to pass vnf instance name + OutputP = json.loads(resultsData) + Output['Output'] = OutputP + cherrypy.log("Output = " + str(Output['Output'])) + #Output['Output'][key] = f.read() # To pass vnf instance name f.close() + if Output == {}: + Output = {'Output': {}} + Result = {'Results': {}} if 'could not be found' in Log: Result['Results'] = {"StatusCode": 101, @@ -292,62 +307,140 @@ class TestManager (object): time_now = datetime.datetime.utcnow() # Erase old test results (2x timeout) + # Do cleanup too of ActiveProcess list and old Records - PAP if TestRecord: for key in TestRecord.copy(): + cherrypy.log( "LOOKING AT ALL TestRecords: " + str(key)) + if key in ActiveProcess: + if not ActiveProcess[key].is_alive(): # Just to cleanup defunct processes + cherrypy.log( "Not ActiveProcess for ID: " + str(key)) delta_time = (time_now - TestRecord[key]['Time']).seconds if delta_time > 2*TestRecord[key]['Timeout']: - print "Deleted history for test", key - if os.path.exists(TestRecord[key]['Path']): - shutil.rmtree (TestRecord[key]['Path']) + cherrypy.log( "DELETED HISTORY for ID: " + str(key)) + if key in ActiveProcess: + if not ActiveProcess[key].is_alive(): + ActiveProcess.pop (key) + cherrypy.log( "DELETED ActiveProcess for ID: " + str(key)) + #if os.path.exists(TestRecord[key]['Path']): + # don't remove run dirrectory + #shutil.rmtree (TestRecord[key]['Path']) del TestRecord[key] - print "***> in RestServer.Dispatch:", cherrypy.request.method + cherrypy.log("RestServer.Dispatch: " + cherrypy.request.method) - HomeDir = os.path.dirname(os.path.realpath("~/")) if 'POST' in cherrypy.request.method: input_json = cherrypy.request.json - print " Payload: ", input_json + cherrypy.log("Payload: " + str(input_json)) - if 'Id' in input_json and 'PlaybookName' in input_json: + if 'Id' in input_json and 'PlaybookName' in input_json and 'EnvParameters' in input_json: if True: if not input_json['Id'] in TestRecord: - + # check if Id exists in previous run dirctory + # if so retun error + s_cmd = 'ls ' + ansible_temp + '/*_' + input_json['Id'] + #if subprocess.check_output([s_cmd, ]): Id = input_json['Id'] + if glob.glob( ansible_temp + '/*_' + input_json['Id']): + cherrypy.log("Old directory found for ID: " + Id) + return {"StatusCode": 101, "StatusMessage": "TEST ID FILE ALREADY DEFINED"} + PlaybookName = input_json['PlaybookName'] + # if required it should be passed as an argument + EnvParameters = input_json['EnvParameters'] + + # The lines below are to test multiple EnvParameters being passed + #for i in EnvParameters: + # cherrypy.log("EnvParameter object: " + i) + # cherrypy.log(" EnvParameter Value: " + EnvParameters[ i ]) + + # Now get things out of EnvParameters + VNF_instance = None + VNF_instance = EnvParameters.get('vnf_instance') + # Get Version if present version = None if 'Version' in input_json: version = input_json['Version'] - AnsibleInvFail = True - AnsiblePlaybookFail = True + # GetInventoryNames + HaveNodeList = False + HaveInventoryNames = False + inventory_names = None + if 'InventoryNames' in input_json: + inventory_names = input_json['InventoryNames'] + HaveInventoryNames = True - MySqlConFail = True - MySqlCause = '' + #AnsibleInvFail = True + AnsiblePlaybookFail = True LocalNodeList = None str_uuid = str (uuid.uuid4()) - LCM = PlaybookName.split(".")[0].split('_')[-1] - PlaybookDir = HomeDir + "/" + ansible_temp + "/" + \ - PlaybookName.split(".")[0] + "_" + str_uuid - AnsibleInv = LCM + "_" + "inventory" + + VnfType= PlaybookName.split("/")[0] + if auth: + cherrypy.log( "Request USER : " + cherrypy.request.login) + cherrypy.log( "Request Decode: ID " + Id) + cherrypy.log( "Request Decode: VnfType " + VnfType) + cherrypy.log( "Request Decode: EnvParameters " + json.dumps(EnvParameters)) + + # Verify VNF_instance was passed in EnvParameters + if VNF_instance != None: + cherrypy.log( "Request Decode: VnfInstance " + VNF_instance) + else: + cherrypy.log( "StatusCode: 107, StatusMessage: VNF_instance NOT PROVIDED" ) + return {"StatusCode": 107, + "StatusMessage": "VNF_instance NOT PROVIDED"} + + if inventory_names != None: + cherrypy.log( "Request Decode: Inventory Names " + inventory_names) + else: + cherrypy.log( "Request Decode: Inventory Names " + "Not provided") + + cherrypy.log( "Request Decode: PlaybookName " + PlaybookName) + PlayBookFunction = PlaybookName.rsplit("/",2)[1] + PlayBookFile = PlayBookFunction + "/site.yml" + cherrypy.log( "Request Decode: PlaybookFunction " + PlayBookFunction) + cherrypy.log( "Request Decode: Playbook file " + PlayBookFile) + + BaseDir = ansible_path + "/" + PlaybookName.rsplit("/",1)[0] + CopyDir = ansible_path + "/" + PlaybookName.rsplit("/",2)[0] + cherrypy.log( "Request Decode: Basedir " + BaseDir) + cherrypy.log( "Request Decode: Copydir " + CopyDir) + + + PlaybookDir = ansible_temp + "/" + \ + VNF_instance + "_" + str_uuid + "_" + str(Id) + + # AnsibleInv is the directory where the host file to be run exsists + AnsibleInv = ansible_path + "/" + VnfType + "/latest/ansible/inventory/" + VNF_instance ArchiveFlag = False - print " LCM: ", LCM - print " PlaybookDir: ", ansible_temp + PlaybookDir.split(ansible_temp)[1] - print " AnsibleInv: ", AnsibleInv - print " ansible_temp: ", ansible_temp + # Create base run directory if it doesn't exist + if not os.path.exists(ansible_temp): + cherrypy.log( "Creating Base Run Directory: " + ansible_temp) + os.makedirs(ansible_temp) - if not os.path.exists(HomeDir + "/" + ansible_temp): - os.makedirs(HomeDir + "/" + ansible_temp) + if not os.path.exists( CopyDir ): + cherrypy.log("Playbook Not Found") + return {"StatusCode": 101, + "StatusMessage": "PLAYBOOK NOT FOUND"} + + # copy static playbook dir to run dir + cherrypy.log("Copying from " + CopyDir + " to " + PlaybookDir) + shutil.copytree(CopyDir, PlaybookDir) + cmd="/usr/bin/find " + PlaybookDir + " -exec /usr/bin/touch {} \;" + cmd="/usr/bin/find " + PlaybookDir + " -exec chmod +rx {} \;" + sys_call(cmd) + cherrypy.log(cmd) - os.mkdir(PlaybookDir) + cherrypy.log( "PlaybookDir: " + PlaybookDir) + cherrypy.log( "AnsibleInv: " + AnsibleInv) # Process inventory file for target @@ -358,111 +451,56 @@ class TestManager (object): if 'NodeList' in input_json: NodeList = input_json['NodeList'] - print " NodeList: ", NodeList + cherrypy.log("NodeList: " + str(NodeList)); + # if NodeList empty if NodeList == []: - # By default set to local host - AnsibleInvFail = False - - LocalNodeList = "host" - LocalCredentials = "localhost ansible_connection=local" - f = open(PlaybookDir + "/" + AnsibleInv, "w") - f.write("[" + LocalNodeList + "]\n") - f.write(LocalCredentials) - f.close() + cherrypy.log( "*** NodeList - Empty ***") else: + HaveNodeList = True + + ############################################################################### + ##### Host file processing ########################### + ##### 1. Use file delivered with playbook ########################### + ##### 2. If HostNames + NodeList generate and use ########################### + ############################################################################### + + #Verify inventory directory exists + path = PlaybookDir + "/inventory/" + if not os.path.isdir(path): + cherrypy.log ("Inventory directory %s does not exist - create it" % path) + try: + os.mkdir(path) + except OSError: + cherrypy.log ("Creation of the directory %s failed" % path) + else: + cherrypy.log ("Successfully created the directory %s " % path) - if from_files: - - # Get credentials from file - - data_inventory_orig = {} - data_inventory_target = {} - curr_group = None - - print "***>", ansible_path + "/" + ansible_inv - f = open(ansible_path + "/" + ansible_inv, "r") - for line in f: - line = line.rstrip() - - if len(line)> 0: - if '#' not in line: - if "[" in line and "]" in line: - data_inventory_orig[line] = [] - curr_group = line - else: - data_inventory_orig[curr_group].append(line) - f.close() - - for node in NodeList: - Fail = True - if "[" + node + "]" in data_inventory_orig: - if not "[" + node + "]" in data_inventory_target: - - print "RESET", "[" + node + "]" - data_inventory_target["[" + node + "]"] = [] - else: - print "OK", "[" + node + "]" - Fail = False - for cred in data_inventory_orig["[" + node + "]"]: - data_inventory_target["[" + node + "]"].append(cred) - - else: - for key in data_inventory_orig: - if node+" " in " ".join(data_inventory_orig[key]): - if not key in data_inventory_target: - data_inventory_target[key] = [] - for cred in data_inventory_orig[key]: - if node+" " in cred: - data_inventory_target[key].append(cred) - Fail = False - - if Fail: - data_inventory_target["["+node+"]"] = \ - [node + " ansible_connection=ssh ansible_ssh_user=na ansible_ssh_private_key_file=na"] - - AnsibleInvFail = False - - f = open(PlaybookDir + "/" + AnsibleInv, "w") - for key in data_inventory_target: - f.write(key + "\n") - for rec in data_inventory_target[key]: - hostgrouplist.append(key.replace("[", '').replace("]", '')) - hostnamelist.append(rec.split(' ')[0]) - f.write(rec + "\n") - f.close() + #location of host file - Default + HostFile = PlaybookDir + "/inventory/" + VNF_instance + "hosts" + cherrypy.log("HostFile: " + HostFile) - else: + # if NodeList and InventoryNames need to build host file + if HaveInventoryNames & HaveNodeList: + cherrypy.log("Build host file from NodeList") + ret = buildHostsSysCall (input_json, PlaybookDir, inventory_names) + if (ret < 0): + cherrypy.log("Returning Error: Not running Playbook") + return {"StatusCode": 105, + "StatusMessage": "NodeList: Missing vnfc-type field"} - # Get credentials from mySQL - - sqlintf = AnsibleSql.mySql (host, user, passwd, - db) - if sqlintf.con: - MySqlConFail = False - errorCode, diag = readCredentials (sqlintf, - NodeList) - - print errorCode, diag - if len (diag) > 0: - f = open(PlaybookDir + "/" + AnsibleInv, - "w") - AnsibleInvFail = False - # [hostgroup, hostname, credentials] - for i in range(len(diag)): - f.write('[' + diag[i][0] + ']' + "\n") - f.write(diag[i][1]+ " " + diag[i][2] + "\n\n") - hostgrouplist.append(diag[i][0]) - hostnamelist.append(diag[i][1]) - f.close() - else: - MySqlConFailCause = sqlintf.error - sqlintf.Close() + # Having been built now copy new file to correct file + shutil.copy(PlaybookDir + "/host_file.txt", HostFile) + cherrypy.log("Copying Generated host file to: " + HostFile) timeout = timeout_seconds if 'Timeout' in input_json: timeout = int (input_json['Timeout']) + cherrypy.log("Timeout from API: " + str(timeout)) + + else: + cherrypy.log("Timeout not passed from API using default: " + str(timeout)) EnvParam = {} if 'EnvParameters' in input_json: @@ -480,8 +518,30 @@ class TestManager (object): if 'CallBack' in input_json: callback_flag = input_json['CallBack'] - TestRecord[Id] = {'PlaybookName': PlaybookName, - 'LCM': LCM, + # if AnsibleServer is not set to 'na' don't send AnsibleServer in PENDING responce. + if AnsibleServer != 'na': + TestRecord[Id] = {'PlaybookName': PlaybookName, + 'Version': version, + 'NodeList': NodeList, + 'HostGroupList': hostgrouplist, + 'HostNameList': hostnamelist, + 'Time': time_now, + 'Duration': timeout, + 'Timeout': timeout, + 'EnvParameters': EnvParam, + 'LocalParameters': LocalParam, + 'FileParameters': FileParam, + 'CallBack': callback_flag, + 'Result': {"StatusCode": 100, + "StatusMessage": 'PENDING', + "AnsibleServer": str(AnsibleServer), + "ExpectedDuration": str(timeout) + "sec"}, + 'Log': '', + 'Output': {}, + 'Path': PlaybookDir, + 'Mandatory': None} + else: + TestRecord[Id] = {'PlaybookName': PlaybookName, 'Version': version, 'NodeList': NodeList, 'HostGroupList': hostgrouplist, @@ -501,6 +561,7 @@ class TestManager (object): 'Path': PlaybookDir, 'Mandatory': None} + cherrypy.log("Test_Record: " + str(TestRecord[Id])) # Write files if not TestRecord[Id]['FileParameters'] == {}: @@ -513,115 +574,19 @@ class TestManager (object): # Process playbook + if os.path.exists( ansible_path + '/' + PlaybookName): + AnsiblePlaybookFail = False - if from_files: - - # Get playbooks from files - - MySqlConFail = False - - version = None - target_PlaybookName = None - - if '@' in PlaybookName: - version = PlaybookName.split("@")[1] - version = version.replace('.yml','') - version = version.replace('.tar.gz','') - - onlyfiles = [f for f in listdir(ansible_path) - if isfile(join(ansible_path, f))] - - version_max = '0.00' - version_target = '' - - for file in onlyfiles: - if LCM in file: - temp_version = file.split("@")[1] - temp_version = temp_version.replace('.yml','') - temp_version = temp_version.replace('.tar.gz','') - if version_max < temp_version: - version_max = temp_version - - if not version == None: - if version in PlaybookName: - version_target = version - target_PlaybookName = file - - if target_PlaybookName == None: - for file in onlyfiles: - if LCM in file and version_max in file: - target_PlaybookName = file - version_target = version_max - - if target_PlaybookName: - AnsiblePlaybookFail = False - readversion = version_target - src = ansible_path + "/" + target_PlaybookName - if ".tar.gz" in target_PlaybookName: - dest = PlaybookDir + "/" + LCM + ".tar.gz" - shutil.copy2(src, dest) - retcode = subprocess.call(['tar', '-xvzf', - dest, "-C", PlaybookDir]) - ArchiveFlag = True - else: - dest = PlaybookDir + "/" + LCM + ".yml" - shutil.copy2(src, dest) - - else: - # Get playbooks from mySQL - - sqlintf = AnsibleSql.mySql (host, user, passwd, db) - if sqlintf.con: - MySqlConFail = False - - name, readversion, AnsiblePlaybookFail, diag = \ - readPlaybook (sqlintf, PlaybookName.split(".")[0], - version) - - if not AnsiblePlaybookFail: - - f = open(PlaybookDir + "/" + LCM + diag[1], "w") - f.write(diag[0]) - f.close() - - if ".tar.gz" in diag[1]: - retcode = subprocess.call(['tar', '-xvzf', - PlaybookDir + "/" + LCM + diag[1], "-C", PlaybookDir]) - f.close() - ArchiveFlag = True - else: - MySqlConFailCause = sqlintf.error - sqlintf.Close() - - if MySqlConFail: - if os.path.exists(PlaybookDir): - shutil.rmtree (PlaybookDir) - del TestRecord[Id] - return {"StatusCode": 101, - "StatusMessage": "CANNOT CONNECT TO MYSQL: " \ - + MySqlConFailCause} - elif AnsiblePlaybookFail: - if os.path.exists(PlaybookDir): - shutil.rmtree (PlaybookDir) + if AnsiblePlaybookFail: + #if os.path.exists(PlaybookDir): + #shutil.rmtree (PlaybookDir) del TestRecord[Id] return {"StatusCode": 101, "StatusMessage": "PLAYBOOK NOT FOUND"} - elif AnsibleInvFail: - if os.path.exists(PlaybookDir): - shutil.rmtree (PlaybookDir) - del TestRecord[Id] - return {"StatusCode": 101, - "StatusMessage": "NODE LIST CREDENTIALS NOT FOUND"} else: # Test EnvParameters - playbook_path = None - if ArchiveFlag: - for dName, sdName, fList in os.walk(PlaybookDir): - if LCM+".yml" in fList: - playbook_path = dName - else: - playbook_path = PlaybookDir + playbook_path = PlaybookDir # Store local vars if not os.path.exists(playbook_path + "/vars"): @@ -629,6 +594,19 @@ class TestManager (object): if not os.path.isfile(playbook_path + "/vars/defaults.yml"): os.mknod(playbook_path + "/vars/defaults.yml") + ################################################### + # PAP + #write local parameters passed into defaults.yml + # PAP + f = open(playbook_path + "/vars/defaults.yml","a") + #for id, record in TestRecord.items(): + print TestRecord[Id]['LocalParameters'] + local_parms = TestRecord[Id]['LocalParameters'] + for key, value in local_parms.items(): + f.write(key +"=" + value + "\n"); + f.close() + ################################################### + for key in TestRecord[Id]['LocalParameters']: host_index = [] for i in range(len(TestRecord[Id]['HostNameList'])): @@ -649,110 +627,118 @@ class TestManager (object): "\n") f.close() - # Get mandatory parameters from playbook - Mandatory = [] - with open(playbook_path + "/" + LCM + ".yml") as origin_file: - for line in origin_file: - if "Mandatory" in line: - temp = line.split(":")[1].strip().replace(' ', '') - if len(temp) > 0: - Mandatory = temp.split(",") - - TestRecord[Id] = {'PlaybookName': TestRecord[Id]['PlaybookName'], - 'LCM': TestRecord[Id]['LCM'], - 'Version': readversion, - 'NodeList': TestRecord[Id]['NodeList'], - 'HostGroupList': TestRecord[Id]['HostGroupList'], - 'HostNameList': TestRecord[Id]['HostNameList'], - 'Time': TestRecord[Id]['Time'], - 'Timeout': TestRecord[Id]['Timeout'], - 'Duration': TestRecord[Id]['Duration'], - 'EnvParameters': TestRecord[Id]['EnvParameters'], - 'LocalParameters': TestRecord[Id]['LocalParameters'], - 'FileParameters': TestRecord[Id]['FileParameters'], - 'CallBack': TestRecord[Id]['CallBack'], - 'Result': TestRecord[Id]['Result'], - 'Log': TestRecord[Id]['Log'], - 'Output': TestRecord[Id]['Output'], - 'Path': TestRecord[Id]['Path'], - 'Mandatory': Mandatory} - - TestKey = False - - if Mandatory: - for val in Mandatory: - if EnvParam: - if val in EnvParam: - TestKey = True - else: - if LocalParam: - for key in TestRecord[Id]['NodeList']: - if key in LocalParam: - if val in LocalParam[key]: - TestKey = True - else: - if LocalParam: - for key in TestRecord[Id]['NodeList']: - if key in LocalParam: - if val in LocalParam[key]: - TestKey = True - - if not TestKey: - if os.path.exists(PlaybookDir): - shutil.rmtree (PlaybookDir) - del TestRecord[Id] - return {"StatusCode": 101, - "StatusMessage": "MISSING MANDATORY PARAMETER: " + \ - " ".join(str(x) for x in Mandatory)} + # write some info out to files before running + if auth: + f = open(playbook_path + "/User.txt", "a") + f.write(cherrypy.request.login) + f.close() + f = open(playbook_path + "/PlaybookName.txt", "a") + f.write(PlaybookName) + f.close() + f = open(playbook_path + "/PlaybookExDir.txt", "a") + f.write(PlaybookDir + "/" + PlayBookFunction) + f.close() + f = open(playbook_path + "/JsonRequest.txt", "w") + #f.write(str(input_json)) + #print( json.dumps(input_json, indent=4, sort_keys=True)) + f.write( json.dumps(input_json, indent=4, sort_keys=True)) + f.close() + + + # Check that HostFile exists + if not os.path.isfile(HostFile): + cherrypy.log("Inventory file Not Found: " + HostFile) + return {"StatusCode": 101, + "StatusMessage": "PLAYBOOK INVENTORY FILE NOT FOUND"} # Cannot use thread because ansible module uses # signals which are only supported in main thread. # So use multiprocess with shared object + # args = (callback, Id, PlaybookDir + "/" + AnsibleInv, p = Process(target = RunAnsible_Playbook, - args = (callback, Id, PlaybookDir + "/" + AnsibleInv, - playbook_path + "/" + LCM + ".yml", - NodeList, TestRecord, PlaybookDir, + args = (callback, Id, HostFile, + PlaybookDir + '/' + PlayBookFile, + NodeList, TestRecord, PlaybookDir + "/" + PlayBookFunction, ArchiveFlag)) p.start() ActiveProcess[Id] = p return TestRecord[Id]['Result'] else: + cherrypy.log("TEST ID ALREADY DEFINED") return {"StatusCode": 101, "StatusMessage": "TEST ID ALREADY DEFINED"} else: return {"StatusCode": 500, "StatusMessage": "REQUEST MUST INCLUDE: NODELIST"} else: - return {"StatusCode": 500, "StatusMessage": "JSON OBJECT MUST INCLUDE: ID, PLAYBOOKNAME"} + return {"StatusCode": 500, "StatusMessage": "JSON OBJECT MUST INCLUDE: ID, PLAYBOOKNAME, EnvParameters"} elif 'GET' in cherrypy.request.method: + # Lets pause for a second just incase the resquest was just kicked off + time.sleep(1) + input_data = parse_query_string(cherrypy.request.query_string) - print "***> in RestServer.GET" - print " Payload: ", input_data, input_data['Type'] + # Verify we have a Type passed in GET request + if not ( 'Type' in input_data): + return {"StatusCode": 500, "StatusMessage": "RESULTS TYPE UNDEFINED"} + + if auth: + cherrypy.log( "Request USER: " + cherrypy.request.login) + cherrypy.log("Payload: " + str(input_data) + " Type " + input_data['Type']) + + if 'LogRest' in input_data['Type']: + sys.stdout.close() + sys.stdout = open("/var/log/RestServer.log", "w") + + # Just a debug to dump any records + if 'GetStatus' in input_data['Type']: + cherrypy.log( "******** Dump Records **********") + if TestRecord.items(): + for id, record in TestRecord.items(): + cherrypy.log( " Id: " + id) + cherrypy.log( "Record: " + str(record)) + else: + cherrypy.log(" No Records to dump") if 'Id' in input_data and 'Type' in input_data: - if not ('GetResult' in input_data['Type'] or 'GetOutput' in input_data['Type'] or 'GetLog' in input_data['Type']): + if not ('GetResult' in input_data['Type'] or 'GetOutputLog' in input_data['Type'] or'GetTheOutput' in input_data['Type'] or 'GetLog' in input_data['Type']): return {"StatusCode": 500, "StatusMessage": "RESULTS TYPE UNDEFINED"} if input_data['Id'] in TestRecord: if 'GetResult' in input_data['Type']: - print "Result:", TestRecord[input_data['Id']]['Result'] + cherrypy.log( " ** GetResult for: " + str (input_data['Id'])) if 'StatusMessage' in TestRecord[input_data['Id']]['Result'] and getresults_block: - print "*** Request blocked", input_data['Id'] + #check if playbook is still running while ActiveProcess[input_data['Id']].is_alive(): - time.sleep(5) + cherrypy.log( "*** Playbook running returning PENDING for " + str(input_data['Id'])) + ## + ## If still running return PENDING response + ## + if AnsibleServer != 'na': + return {"StatusCode": 100, + "StatusMessage": 'PENDING', + "AnsibleServer": str(AnsibleServer)} + else: + return {"StatusCode": 100, + "StatusMessage": 'PENDING'} + #time.sleep(5) - print "*** Request released ", input_data['Id'] + #cherrypy.log( "*** Request released " + input_data['Id']) - print TestRecord[input_data['Id']]['Result'] + cherrypy.log(str( TestRecord[input_data['Id']]['Result'])) + cherrypy.log("Output: " + str( TestRecord[input_data['Id']]['Output'])) + cherrypy.log("StatusCode: " + str( TestRecord[input_data['Id']]['Result']['StatusCode'])) + cherrypy.log("StatusMessage: " + str( TestRecord[input_data['Id']]['Result']['StatusMessage'])) + + #out_obj gets returned to GET request if TestRecord[input_data['Id']]['Result']['StatusCode'] == 500: out_obj = TestRecord[input_data['Id']]['Result']['Results'] else: @@ -761,60 +747,164 @@ class TestManager (object): "PlaybookName": TestRecord[input_data['Id']]["PlaybookName"], "Version": TestRecord[input_data['Id']]["Version"], "Duration": TestRecord[input_data['Id']]["Duration"], + "Output": TestRecord[input_data['Id']]["Output"]["Output"], "Results": TestRecord[input_data['Id']]['Result']['Results']} if not TestRecord[input_data['Id']]['Output']['Output'] == {}: + cherrypy.log("TestRecord has Output:" + str(TestRecord[input_data['Id']]['Output']['Output'])) + # PAP for key in out_obj["Results"]: + cherrypy.log("Output key: " + str(key)) if key in TestRecord[input_data['Id']]['Output']['Output']: out_obj["Results"][key]["Output"] = TestRecord[input_data['Id']]['Output']['Output'][key] + cherrypy.log("***** GET RETURNING RESULTS Back ****") + cherrypy.log(str(out_obj)) return out_obj - elif 'GetOutput' in input_data['Type']: + elif 'GetStatus' in input_data['Type']: + print " Dump Records" + for id, record in TestRecord,items(): + print " id: " + id + print " Record:" + str(reecord) + + elif 'GetTheOutput' in input_data['Type']: if TestRecord[input_data['Id']]['Output'] == {} and \ getresults_block: - print "*** Request blocked", input_data['Id'] + cherrypy.log( "*** Request blocked " + input_data['Id']) while TestRecord[input_data['Id']]['Output'] == {} \ or 'StatusMessage' in TestRecord[input_data['Id']]['Result']: time.sleep(5) - print "*** Request released ", input_data['Id'] + cherrypy.log( "*** Request released " + input_data['Id']) - print "Output:", TestRecord[input_data['Id']]['Output'] + cherrypy.log( "Output: " + str(TestRecord[input_data['Id']]['Output'])) return {"Output": TestRecord[input_data['Id']]['Output']['Output']} + + elif 'GetOutputLog' in input_data['Type']: + cherrypy.log("GetOutputLog: processing.") + if glob.glob( ansible_temp + '/*_' + input_data['Id']): + id = input_data['Id'] + cherrypy.log("Old directory found for ID: " + id) + run_dir = glob.glob( ansible_temp + '/*_' + input_data['Id']) + for dir in run_dir: + rdir=dir + if os.path.exists (rdir + "/PlaybookExDir.txt"): + cherrypy.log("Found PlaybookExDir.txt file") + f = open( rdir + '/PlaybookExDir.txt', 'r') + playbookexdir = f.readline() + rdir = playbookexdir + f.close() + cherrypy.log("Id: " + id) + cherrypy.log("RunDir: " + rdir) + if os.path.exists( rdir + "/output.log"): + cherrypy.log("Found output.log file") + f = open( rdir + '/output.log', 'r') + output_log = f.readline() + f.close() + return output_log + else: + cherrypy.log("Globglob failed:") + return + else: # GetLog if TestRecord[input_data['Id']]['Log'] == '' and \ getresults_block: - print "*** Request blocked", input_data['Id'] + cherrypy.log( "*** Request blocked " + input_data['Id']) while TestRecord[input_data['Id']]['Log'] == '' \ or 'StatusMessage' in TestRecord[input_data['Id']]['Result']: time.sleep(5) - print "*** Request released ", input_data['Id'] + cherrypy.log( "*** Request released " + input_data['Id']) - print "Log:", TestRecord[input_data['Id']]['Log'] + cherrypy.log( "Log:" + str(TestRecord[input_data['Id']]['Log'])) return {"Log": TestRecord[input_data['Id']]['Log']} else: - return {"StatusCode": 500, "StatusMessage": "TEST ID UNDEFINED"} + # Not in memory check for a file + if glob.glob( ansible_temp + '/*_' + input_data['Id']): + id = input_data['Id'] + cherrypy.log("Old directory found for ID: " + id) + run_dir = glob.glob( ansible_temp + '/*_' + input_data['Id']) + for dir in run_dir: + rdir=dir + if os.path.exists (rdir + "/PlaybookExDir.txt"): + cherrypy.log("Found PlaybookExDir.txt file") + f = open( rdir + '/PlaybookExDir.txt', 'r') + playbookexdir = f.readline() + rdir = playbookexdir + f.close() + cherrypy.log("Id: " + id) + cherrypy.log("RunDir: " + rdir) + if 'GetLog' in input_data['Type']: + if os.path.exists( rdir + "/output.log"): + cherrypy.log("Found output.log file") + f = open( rdir + '/output.log', 'r') + output_log = f.readline() + f.close() + return output_log + elif 'GetOutputLog' in input_data['Type']: + if os.path.exists( rdir + "/output.log"): + cherrypy.log("Found output.log file") + f = open( rdir + '/output.log', 'r') + output_log = f.readline() + f.close() + return output_log + elif 'GetResult' in input_data['Type']: + if os.path.exists (rdir + "/PlaybookName.txt"): + cherrypy.log("Found PlaybookName.txt file") + f = open( rdir + '/PlaybookName.txt', 'r') + playbooknametxt = f.readline() + f.close() + else: + playbooknametxt = "NA" + + # Add code to get other items not just output.log from files + if os.path.exists( rdir + "/log.file"): + cherrypy.log("Found log.file") + out_results = "NA:" + f = open( rdir + '/log.file', 'r') + + line = f.readline() + while line : + if "fatal" in line: + out_results = out_results + line + elif "RECAP" in line: + out_results = out_results + line + recap_line = f.readline() + while recap_line : + out_results = out_results + recap_line + recap_line = f.readline() + line = f.readline() + f.close() + out_obj = {"StatusCode": 200, + "StatusMessage": "FINISHED", + "PlaybookName": playbooknametxt, + "Version": "Version", + "Duration": 200, + "Results": out_results} + return out_obj + else: + return {"StatusCode": 500, "StatusMessage": "PLAYBOOK FAILED "} + + + return {"StatusCode": 500, "StatusMessage": "TEST ID UNDEFINED"} else: return {"StatusCode": 500, "StatusMessage": "MALFORMED REQUEST"} elif 'DELETE' in cherrypy.request.method: input_data = parse_query_string(cherrypy.request.query_string) - print "***> in RestServer.DELETE" - print " Payload: ", input_data + cherrypy.log( "***> in RestServer.DELETE") + cherrypy.log("Payload: " + str(input_data)) if input_data['Id'] in TestRecord: if not 'PENDING' in TestRecord[input_data['Id']]['Result']: - print " Path:", TestRecord[input_data['Id']]['Path'] - if os.path.exists(TestRecord[input_data['Id']]['Path']): - shutil.rmtree (TestRecord[input_data['Id']]['Path']) + cherrypy.log(" Path: " + str(TestRecord[input_data['Id']]['Path'])) TestRecord.pop (input_data['Id']) if input_data['Id'] in ActiveProcess: ActiveProcess.pop (input_data['Id']) @@ -837,21 +927,17 @@ if __name__ == '__main__': sys.exit(0) ip = 'na' + AnsibleServer = 'na' port = 'na' tls = False auth = False pub = 'na' - id = 'na' priv = 'na' - psswd = 'na' timeout_seconds = 'na' ansible_path = 'na' - ansible_inv = 'na' ansible_temp = 'na' host = 'na' - user = 'na' - passwd = 'na' - db = 'na' + users= 'na' getresults_block = False from_files = False @@ -860,8 +946,12 @@ if __name__ == '__main__': if '#' not in line: if 'ip:' in line: ip = line.split(':')[1].strip() + elif 'AnsibleServer:' in line: + AnsibleServer = line.split(':')[1].strip() elif 'port:' in line: port = line.split(':')[1].strip() + elif 'ksalt:' in line: + salt = line.split(':')[1].strip() elif 'tls:' in line: tls = 'YES' in line.split(':')[1].strip().upper() elif 'auth:' in line: @@ -870,29 +960,18 @@ if __name__ == '__main__': priv = line.split(':')[1].strip() if tls and 'pub:' in line: pub = line.split(':')[1].strip() - if auth and 'id:' in line: - id = line.split(':')[1].strip() - if auth and 'psswd:' in line: - psswd = line.split(':')[1].strip() + if tls and 'inter_cert:' in line: + intermediate = line.split(':')[1].strip() if 'timeout_seconds' in line: timeout_seconds = int (line.split(':')[1].strip()) if 'ansible_path' in line: ansible_path = line.split(':')[1].strip() - if 'ansible_inv' in line: - ansible_inv = line.split(':')[1].strip() - if not os.path.exists(ansible_path + "/" + ansible_inv): - print '[INFO] The ansible_inv file does not exist' - sys.exit(0) if 'ansible_temp' in line: ansible_temp = line.split(':')[1].strip() if 'host' in line: host = line.split(':')[1].strip() - if 'user' in line: - user = line.split(':')[1].strip() - if 'passwd' in line: - passwd = line.split(':')[1].strip() - if 'db' in line: - db = line.split(':')[1].strip() + if 'users' in line: + users = line.split(':')[1].strip() if 'getresults_block' in line: getresults_block = 'YES' in line.split(':')[1].strip().upper() if 'from_files' in line: @@ -903,6 +982,8 @@ if __name__ == '__main__': global_conf = { 'global': { + 'log.screen': True, + 'response.timeout': 5400, 'server.socket_host': ip, 'server.socket_port': int(port), 'server.protocol_version': 'HTTP/1.1' @@ -923,26 +1004,58 @@ if __name__ == '__main__': print '[INFO] The private key does not exist' sys.exit(0) + if not os.path.exists(intermediate): + print '[INFO] The intermediate certificate does not exist' + sys.exit(0) + + cherrypy.server.ssl_certificate = pub + cherrypy.server.ssl_certificate_chain = intermediate cherrypy.server.ssl_private_key = priv if auth: - userpassdict = {id: psswd} - checkpassword = cherrypy.lib.auth_basic.checkpassword_dict(userpassdict) + # Read in and build user dictionary + if not os.path.exists(users): + print '[INFO] The users file does not exist: ' + users + sys.exit(0) + userpassdict = {} + user_file = open(users, 'r') + for line in user_file.readlines(): + if '#' not in line: + id = line.split(':')[0].strip() + pw = line.split(':')[1].strip() + userpassdict[id] = pw + #print str(userpassdict) app_conf = {'/': {'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'earth', - 'tools.auth_basic.checkpassword': checkpassword, + 'tools.auth_basic.checkpassword': validate_password, } } - cherrypy.tree.mount(TestManager(), '/', app_conf) + application = cherrypy.tree.mount(TestManager(), '/', app_conf) else: - cherrypy.tree.mount(TestManager(), '/') + application = cherrypy.tree.mount(TestManager(), '/') + cherrypy.config.update({ + 'log.access_file': "/var/log/RestServer.access" + }) + accessLogName = "/var/log/RestServer.access" + applicationLogName = "/var/log/RestServer.log" cherrypy.config.update(global_conf) + log = application.log + log.error_file = "" + log.access_file = "" + from logging import handlers + applicationLogFileHandler = handlers.RotatingFileHandler(applicationLogName, 'a', 1000000, 5000) + accessLogFileHandler = handlers.RotatingFileHandler(accessLogName, 'a', 1000000, 5000) + import logging + applicationLogFileHandler.setLevel(logging.DEBUG) + log.error_log.addHandler(applicationLogFileHandler) + log.access_log.addHandler(accessLogFileHandler) + # Start server cherrypy.engine.start() diff --git a/ansible-server/src/main/ansible-server/RestServer_config b/ansible-server/src/main/ansible-server/RestServer_config index 380bd79f..6824dd64 100644 --- a/ansible-server/src/main/ansible-server/RestServer_config +++ b/ansible-server/src/main/ansible-server/RestServer_config @@ -18,7 +18,6 @@ # * See the License for the specific language governing permissions and # * limitations under the License. # * -# * ECOMP is a trademark and service mark of AT&T Intellectual Property. # * ============LICENSE_END========================================================= # */ diff --git a/ansible-server/src/main/ansible-server/UsersRestServer.py b/ansible-server/src/main/ansible-server/UsersRestServer.py index 9da6fb91..b8992978 100755 --- a/ansible-server/src/main/ansible-server/UsersRestServer.py +++ b/ansible-server/src/main/ansible-server/UsersRestServer.py @@ -19,7 +19,6 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* ECOMP is a trademark and service mark of AT&T Intellectual Property. * ============LICENSE_END========================================================= */ ''' |