path: root/ansible-server/src/main/ansible-server/RestServer.py

'''
/*-
* ============LICENSE_START=======================================================
* ONAP : APPC
* ================================================================================
* Copyright (C) 2017-2019 AT&T Intellectual Property.  All rights reserved.
* ================================================================================
* Copyright (C) 2017 Amdocs
* =============================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* 
* ============LICENSE_END=========================================================
*/
'''

import time, datetime, json, os, sys, subprocess
import uuid
import shutil
import glob
import crypt

import requests

import cherrypy
from cherrypy.lib.httputil import parse_query_string

from multiprocessing import Process, Manager

from AnsibleModule import ansibleSysCall
from BuildHostFile import buildHostsSysCall
from BuildPlaybookParams import buildInventorySysCall, getPlaybookFile

from os import listdir
from os.path import isfile, join

TestRecord = Manager().dict()
ActiveProcess = {}


def validate_password(realm, username, password):
    comp = crypt.crypt(password, salt)
    if username in userpassdict and userpassdict[username] == comp:
        return True
    return False


def sys_call(cmd):
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    output = p.stdout.readlines()
    retval = p.wait()
    if len(output) > 0:
        for i in range(len(output)):
            output[i] = output[i].strip()
    return retval, output


def callback(Id, Result, Output, Log, returncode):
    cherrypy.log("***> in RestServer.callback")

    if Id in TestRecord:
        time_now = datetime.datetime.utcnow()
        delta_time = (time_now - TestRecord[Id]['Time']).total_seconds()
        Result['PlaybookName'] = TestRecord[Id]['PlaybookName']
        Result['Version'] = TestRecord[Id]['Version']
        if returncode == 137:
            Result['StatusCode'] = 500
            Result['StatusMessage'] = "TERMINATED"
        else:
            Result['StatusCode'] = 200
            Result['StatusMessage'] = "FINISHED"

        # Need to update the whole data structure for key=Id otherwise Manager is not updated
        TestRecord[Id] = {
            'PlaybookName': TestRecord[Id]['PlaybookName'],
            'Version': TestRecord[Id]['Version'],
            'NodeList': TestRecord[Id]['NodeList'],
            'HostGroupList': TestRecord[Id]['HostGroupList'],
            'HostNameList': TestRecord[Id]['HostNameList'],
            'Time': TestRecord[Id]['Time'],
            'Timeout': TestRecord[Id]['Timeout'],
            'Duration': str(delta_time),
            'EnvParameters': TestRecord[Id]['EnvParameters'],
            'LocalParameters': TestRecord[Id]['LocalParameters'],
            'FileParameters': TestRecord[Id]['FileParameters'],
            'CallBack': TestRecord[Id]['CallBack'],
            'Result': Result,
            'Log': Log,
            'Output': Output,
            'Path': TestRecord[Id]['Path'],
            'Mandatory': TestRecord[Id]['Path']
        }

        if TestRecord[Id]['CallBack'] is not None:

            # Posting results to callback server
            data = {
                "StatusCode": 200,
                "StatusMessage": "FINISHED",
                "PlaybookName": TestRecord[Id]["PlaybookName"],
                "Version": TestRecord[Id]["Version"],
                "Duration": TestRecord[Id]["Duration"],
                "Results": TestRecord[Id]['Result']['Results']
            }

            cherrypy.log("CALLBACK: TestRecord[Id]['Output']['Output']:", str(TestRecord[Id]['Output']['Output']))
            cherrypy.log("CALLBACK: Results:", str(data["Results"]))

            if not TestRecord[Id]['Output']['Output'] == {}:
                for key in data["Results"]:
                    if key in TestRecord[Id]['Output']['Output']:
                        data["Results"][key]["Output"] = TestRecord[Id]['Output']['Output'][key]

            cherrypy.log("     Posting to", TestRecord[Id]['CallBack'])

            s = requests.Session()
            r = s.post(TestRecord[Id]['CallBack'], data=json.dumps(data),
                       headers={'content-type': 'application/json'})
            cherrypy.log("     Response", r.status_code, r.text)


def RunAnsible_Playbook(callback, Id, Inventory, Playbook, NodeList, TestRecord, Path, ArchiveFlag, pnf_flag=False):
    cherrypy.log("***> in RestServer.RunAnsible_Playbook")

    # Run test in playbook for given target
    retval, log, returncode = ansibleSysCall(Inventory, Playbook, NodeList,
                                             TestRecord[Id]['Mandatory'],
                                             TestRecord[Id]['EnvParameters'],
                                             TestRecord[Id]['LocalParameters'],
                                             TestRecord[Id]['Timeout'],
                                             Path)

    cherrypy.log("Return code:" + str(returncode))
    cherrypy.log("Return value:" + str(retval))

    Log = ''.join(log)
    if pnf_flag:
        Output = {'Output': {}}
    else:
        Output = {}

    onlyfiles = [f for f in listdir(Path)
                 if isfile(join(Path, f))]

    cherrypy.log("Checking for results.txt files: ")
    for file in onlyfiles:
        if "results.txt" in file:
            # if file.endswith("results.txt"):
            cherrypy.log("results file: " + file)
            f = open(Path + "/" + file, "r")
            if pnf_flag:
                key = file.split("_")[0]
                Output['Output'][key] = f.read()
            else:
                resultsData = f.read()  # Not to pass vnf instance name
                OutputP = json.loads(resultsData)
                Output['Output'] = OutputP
            cherrypy.log("Output = " + str(Output['Output']))
            # Output['Output'][key] = f.read() # To pass vnf instance name
            f.close()

    if Output == {}:
        Output = {'Output': {}}

    Result = {'Results': {}}
    if 'could not be found' in Log:
        Result['Results'] = {"StatusCode": 101, "StatusMessage": "PLAYBOOK NOT FOUND"}

    if returncode == 137:
        Result['Results'] = {"StatusCode": 500, "StatusMessage": "TERMINATED"}
    elif TestRecord[Id]['NodeList'] == []:
        host_index = None
        if 'TargetNode' in TestRecord[Id]['EnvParameters']:
            targetlist = TestRecord[Id]['EnvParameters']['TargetNode'].split(' ')
        else:
            targetlist = ["localhost"]

        for key in retval:
            for i in range(len(targetlist)):
                if key in targetlist[i]:
                    host_index = i

            if int(retval[key][0]) > 0 and int(retval[key][2]) == 0 and int(retval[key][3]) == 0:
                if host_index:
                    Result['Results'][targetlist[host_index]] = \
                        {"GroupName": 'na', "StatusCode": 200, "StatusMessage": "SUCCESS"}
                else:
                    Result['Results'][key] = {"GroupName": 'na', "StatusCode": 200, "StatusMessage": "SUCCESS"}
            elif int(retval[key][2]) > 0:
                if host_index:
                    Result['Results'][targetlist[host_index]] = \
                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "NOT REACHABLE"}
                else:
                    Result['Results'][key] = \
                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "NOT REACHABLE"}
            elif int(retval[key][3]) > 0:
                if host_index:
                    Result['Results'][targetlist[host_index]] = \
                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "FAILURE"}
                else:
                    Result['Results'][key] = \
                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "FAILURE"}
    else:
        for key in retval:
            if len(TestRecord[Id]['HostNameList']) > 0:
                host_index = []
                for i in range(len(TestRecord[Id]['HostNameList'])):
                    if key in TestRecord[Id]['HostNameList'][i]:
                        host_index.append(i)

                if int(retval[key][0]) > 0 and int(retval[key][2]) == 0 and \
                        int(retval[key][3]) == 0:
                    if len(host_index) > 0:
                        Result['Results'][TestRecord[Id]['HostNameList'][host_index[0]]] = \
                            {
                                "GroupName": TestRecord[Id]['HostGroupList'][host_index[0]],
                                "StatusCode": 200, "StatusMessage": "SUCCESS"
                            }

                        for i in range(1, len(host_index)):
                            Result['Results'][TestRecord[Id]['HostNameList'][host_index[i]]]["GroupName"] += \
                                "," + TestRecord[Id]['HostGroupList'][host_index[i]]
                    else:
                        Result['Results'][key] = {"GroupName": key, "StatusCode": 200, "StatusMessage": "SUCCESS"}

                elif int(retval[key][2]) > 0:
                    if len(host_index) > 0:
                        Result['Results'][TestRecord[Id]['HostNameList'][host_index[0]]] = \
                            {
                                "GroupName": TestRecord[Id]['HostGroupList'][host_index[0]],
                                "StatusCode": 400, "StatusMessage": "NOT REACHABLE"
                            }

                        for i in range(1, len(host_index)):
                            Result['Results'][TestRecord[Id]['HostNameList'][host_index[i]]]["GroupName"] += \
                                "," + TestRecord[Id]['HostGroupList'][host_index[i]]
                    else:
                        Result['Results'][key] = \
                            {"GroupName": key, "StatusCode": 200, "StatusMessage": "NOT REACHABLE"}
                elif int(retval[key][3]) > 0:
                    if len(host_index) > 0:
                        Result['Results'][TestRecord[Id]['HostNameList'][host_index[0]]] = \
                            {
                                "GroupName": TestRecord[Id]['HostGroupList'][host_index[0]],
                                "StatusCode": 400, "StatusMessage": "FAILURE"
                            }

                        for i in range(1, len(host_index)):
                            Result['Results'][TestRecord[Id]['HostNameList'][host_index[i]]]["GroupName"] += \
                                "," + TestRecord[Id]['HostGroupList'][host_index[i]]
                    else:
                        Result['Results'][key] = \
                            {"GroupName": key, "StatusCode": 200, "StatusMessage": "FAILURE"}
            else:
                host_index = None
                for i in range(len(TestRecord[Id]['NodeList'])):
                    if key in TestRecord[Id]['NodeList'][i]:
                        host_index = i

                if int(retval[key][0]) > 0 and int(retval[key][2]) == 0 and \
                        int(retval[key][3]) == 0:
                    Result['Results'][TestRecord[Id]['NodeList'][host_index]] = \
                        {"GroupName": 'na', "StatusCode": 200, "StatusMessage": "SUCCESS"}
                elif int(retval[key][2]) > 0:
                    Result['Results'][TestRecord[Id]['NodeList'][host_index]] = \
                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "NOT REACHABLE"}
                elif int(retval[key][3]) > 0:
                    Result['Results'][TestRecord[Id]['NodeList'][host_index]] = \
                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "FAILURE"}

    callback(Id, Result, Output, Log, returncode)


def store_local_vars(playbook_path, Id):
    if not os.path.exists(playbook_path + "/vars"):
        os.mkdir(playbook_path + "/vars")

    if not os.path.isfile(playbook_path + "/vars/defaults.yml"):
        os.mknod(playbook_path + "/vars/defaults.yml")

    # ##################################################
    # PAP
    # write local parameters passed into defaults.yml
    # PAP
    local_parms = TestRecord[Id]['LocalParameters']
    cherrypy.log("LocalParameters: " + str(local_parms))

    f = open(playbook_path + "/vars/defaults.yml", "a")
    for key, value in list(local_parms.items()):
        f.write(key + "=" + value + "\n")
    f.close()
    # ##################################################

    for key in TestRecord[Id]['LocalParameters']:
        host_index = []
        for i in range(len(TestRecord[Id]['HostNameList'])):
            if key in TestRecord[Id]['HostNameList'][i]:
                host_index.append(i)
        if len(host_index) == 0:
            for i in range(len(TestRecord[Id]['HostGroupList'])):
                if key in TestRecord[Id]['HostGroupList'][i]:
                    host_index.append(i)
        if len(host_index) > 0:
            for i in range(len(host_index)):
                f = open(playbook_path + "/vars/" + TestRecord[Id]['HostNameList'][host_index[i]] + ".yml", "a")
                for param in TestRecord[Id]['LocalParameters'][key]:
                    f.write(param + ": " + str(TestRecord[Id]['LocalParameters'][key][param]) + "\n")
                f.close()


def process_pnf_playbook(input_json, Id, EnvParameters, time_now):
    cherrypy.log("Processing playbook for PNF...")

    PlaybookName = input_json['PlaybookName']
    version = input_json.get('Version', None)

    if AUTH:
        cherrypy.log("Request USER  :                  " + cherrypy.request.login)
    cherrypy.log("Request Decode: ID               " + Id)
    cherrypy.log("Request Decode: EnvParameters    " + json.dumps(EnvParameters))
    cherrypy.log("Request Decode: PlaybookName     " + PlaybookName)

    str_uuid = str(uuid.uuid4())

    HomeDir = os.path.dirname(os.path.realpath("~/"))

    PlaybookType = PlaybookName.split(".")[0].split('_')[-1]
    PlaybookDir = HomeDir + '/' + ANSIBLE_TEMP + "/" + PlaybookName.split(".")[0] + "_" + str_uuid
    AnsibleInv = PlaybookType + "_" + "inventory"
    ArchiveFlag = False

    cherrypy.log("Request Decode: PlaybookType     " + PlaybookType)
    cherrypy.log("Request Decode: PlaybookDir      " + PlaybookDir)
    cherrypy.log("Request Decode: AnsibleInv       " + AnsibleInv)

    NodeList = input_json.get('NodeList', [])
    cherrypy.log("Request Decode: NodeList: " + str(NodeList))

    # Create base run directory if it doesn't exist
    if not os.path.exists(ANSIBLE_TEMP):
        cherrypy.log("Creating Base Run Directory: " + ANSIBLE_TEMP)
        os.makedirs(ANSIBLE_TEMP)

    os.mkdir(PlaybookDir)

    # Process inventory file for target
    hostgrouplist = []
    hostnamelist = []

    buildInventorySysCall(ANSIBLE_PATH, ANSIBLE_INV, NodeList, PlaybookDir, AnsibleInv, hostgrouplist, hostnamelist)

    version_target = getPlaybookFile(ANSIBLE_PATH, PlaybookName, PlaybookType, PlaybookDir)
    if not version_target:
        return {"StatusCode": 101, "StatusMessage": "PLAYBOOK NOT FOUND"}

    if version is None:
        version = version_target

    if 'Timeout' in input_json:
        timeout = int(input_json['Timeout'])
        cherrypy.log("Timeout from API: " + str(timeout))
    else:
        timeout = timeout_seconds
        cherrypy.log("Timeout not passed from API using default: " + str(timeout))

    EnvParam = input_json.get('EnvParameters', {})
    LocalParam = input_json.get('LocalParameters', {})
    FileParam = input_json.get('FileParameters', {})
    callback_flag = input_json.get('CallBack', None)

    # if AnsibleServer is not set to 'na' don't send AnsibleServer in PENDING response.
    TestRecord[Id] = {
        'PlaybookName': PlaybookName,
        'Version': version,
        'NodeList': NodeList,
        'HostGroupList': hostgrouplist,
        'HostNameList': hostnamelist,
        'Time': time_now,
        'Duration': timeout,
        'Timeout': timeout,
        'EnvParameters': EnvParam,
        'LocalParameters': LocalParam,
        'FileParameters': FileParam,
        'CallBack': callback_flag,
        'Result': {
            "StatusCode": 100,
            "StatusMessage": 'PENDING',
            "ExpectedDuration": str(timeout) + "sec"
        },
        'Log': '',
        'Output': {},
        'Path': PlaybookDir,
        'Mandatory': None
    }
    if AnsibleServer != 'na':
        TestRecord[Id]['Result']["AnsibleServer"] = str(AnsibleServer),

    cherrypy.log("Test_Record: " + str(TestRecord[Id]))

    # Write files
    if TestRecord[Id]['FileParameters']:
        for key in TestRecord[Id]['FileParameters']:
            filename = key
            filecontent = TestRecord[Id]['FileParameters'][key]
            f = open(PlaybookDir + "/" + filename, "w")
            f.write(filecontent)
            f.close()

    playbook_path = PlaybookDir

    # Store local vars
    store_local_vars(playbook_path, Id)

    # write some info out to files before running
    if AUTH:
        f = open(playbook_path + "/User.txt", "a")
        f.write(cherrypy.request.login)
        f.close()

    f = open(playbook_path + "/PlaybookName.txt", "a")
    f.write(PlaybookName)
    f.close()

    f = open(playbook_path + "/JsonRequest.txt", "w")
    f.write(json.dumps(input_json, indent=4, sort_keys=True))
    f.close()

    # Cannot use thread because ansible module uses signals which are only supported in main thread.
    # So use multiprocess with shared object
    p = Process(target=RunAnsible_Playbook,
                args=(callback, Id, PlaybookDir + '/' + AnsibleInv, PlaybookDir + '/' + PlaybookType + '.yml',
                      NodeList, TestRecord, PlaybookDir, ArchiveFlag, True))
    p.start()
    ActiveProcess[Id] = p
    return TestRecord[Id]['Result']


def process_vnf_playbook(input_json, Id, EnvParameters, time_now):
    cherrypy.log("Processing playbook for VNF...")

    PlaybookName = input_json['PlaybookName']
    VNF_instance = EnvParameters.get('vnf_instance')
    version = input_json.get('Version', None)

    # GetInventoryNames
    HaveNodeList = False
    HaveInventoryNames = False
    inventory_names = None
    if 'InventoryNames' in input_json:
        inventory_names = input_json['InventoryNames']
        HaveInventoryNames = True

    AnsiblePlaybookFail = True

    str_uuid = str(uuid.uuid4())

    # VnfType = PlaybookName.split("/")[0]

    if AUTH:
        cherrypy.log("Request USER  :                  " + cherrypy.request.login)
    cherrypy.log("Request Decode: ID               " + Id)
    # cherrypy.log("Request Decode: VnfType          " + VnfType)
    cherrypy.log("Request Decode: EnvParameters    " + json.dumps(EnvParameters))

    # Verify VNF_instance was passed in EnvParameters
    if VNF_instance is not None:
        cherrypy.log("Request Decode: VnfInstance      " + VNF_instance)
    else:
        cherrypy.log("StatusCode: 107, StatusMessage: VNF_instance NOT PROVIDED")
        return {"StatusCode": 107, "StatusMessage": "VNF_instance NOT PROVIDED"}

    if inventory_names is not None:
        cherrypy.log("Request Decode: Inventory Names  " + inventory_names)
    else:
        cherrypy.log("Request Decode: Inventory Names  " + "Not provided")

    cherrypy.log("Request Decode: PlaybookName     " + PlaybookName)

    PlayBookFunction = PlaybookName.rsplit("/", 2)[1]
    PlayBookFile = PlayBookFunction + "/site.yml"

    cherrypy.log("Request Decode: PlaybookFunction " + PlayBookFunction)
    cherrypy.log("Request Decode: PlaybookFile     " + PlayBookFile)

    BaseDir = ANSIBLE_PATH + "/" + PlaybookName.rsplit("/", 1)[0]
    CopyDir = ANSIBLE_PATH + "/" + PlaybookName.rsplit("/", 2)[0]
    cherrypy.log("Request Decode: Basedir          " + BaseDir)
    cherrypy.log("Request Decode: Copydir          " + CopyDir)

    PlaybookDir = ANSIBLE_TEMP + "/" + VNF_instance + "_" + str_uuid + "_" + str(Id)
    cherrypy.log("Request Decode: PlaybookDir      " + PlaybookDir)

    # AnsibleInv is the directory where the host file to be run exists
    # AnsibleInv = ANSIBLE_PATH + "/" + VnfType + "/latest/ansible/inventory/" + VNF_instance
    ArchiveFlag = False

    # Create base run directory if it doesn't exist
    if not os.path.exists(ANSIBLE_TEMP):
        cherrypy.log("Creating Base Run Directory: " + ANSIBLE_TEMP)
        os.makedirs(ANSIBLE_TEMP)

    if not os.path.exists(CopyDir):
        cherrypy.log("Playbook Not Found")
        return {"StatusCode": 101, "StatusMessage": "PLAYBOOK NOT FOUND"}

    # copy static playbook dir to run dir
    cherrypy.log("Copying from " + CopyDir + " to " + PlaybookDir)
    shutil.copytree(CopyDir, PlaybookDir)
    # cmd="/usr/bin/find " + PlaybookDir + " -exec /usr/bin/touch {} \;"
    cmd = "/usr/bin/find " + PlaybookDir + " -exec chmod +rx  {} \;"
    sys_call(cmd)
    cherrypy.log(cmd)

    cherrypy.log("PlaybookDir:    " + PlaybookDir)
    # cherrypy.log("AnsibleInv:     " + AnsibleInv)

    # Process inventory file for target
    hostgrouplist = []
    hostnamelist = []

    NodeList = input_json.get('NodeList', [])

    cherrypy.log("NodeList: " + str(NodeList))

    # if NodeList empty
    if not NodeList:
        cherrypy.log("*** NodeList - Empty ***")
    else:
        HaveNodeList = True

    # ##############################################################################
    # #### Host file processing                          ###########################
    # #### 1. Use file delivered with playbook           ###########################
    # #### 2. If HostNames + NodeList generate and use   ###########################
    # ##############################################################################

    # Verify inventory directory exists
    path = PlaybookDir + "/inventory/"
    if not os.path.isdir(path):
        cherrypy.log("Inventory directory %s does not exist - create it" % path)
        try:
            os.mkdir(path)
        except OSError:
            cherrypy.log("Creation of the directory %s failed" % path)
        else:
            cherrypy.log("Successfully created the directory %s " % path)

    # location of host file - Default
    HostFile = PlaybookDir + "/inventory/" + VNF_instance + "hosts"
    cherrypy.log("HostFile: " + HostFile)

    # if NodeList and InventoryNames need to build host file
    if HaveInventoryNames and HaveNodeList:
        cherrypy.log("Build host file from NodeList")
        ret = buildHostsSysCall(input_json, PlaybookDir, inventory_names)
        if ret < 0:
            cherrypy.log("Returning Error: Not running Playbook")
            return {"StatusCode": 105,
                    "StatusMessage": "NodeList: Missing vnfc-type field"}

        # Having been built now copy new file to correct file
        shutil.copy(PlaybookDir + "/host_file.txt", HostFile)
        cherrypy.log("Copying Generated host file to: " + HostFile)

    if 'Timeout' in input_json:
        timeout = int(input_json['Timeout'])
        cherrypy.log("Timeout from API: " + str(timeout))
    else:
        timeout = timeout_seconds
        cherrypy.log("Timeout not passed from API using default: " + str(timeout))

    EnvParam = input_json.get('EnvParameters', {})
    LocalParam = input_json.get('LocalParameters', {})
    FileParam = input_json.get('FileParameters', {})
    callback_flag = input_json.get('CallBack', None)

    # if AnsibleServer is not set to 'na' don't send AnsibleServer in PENDING response.
    TestRecord[Id] = {
        'PlaybookName': PlaybookName,
        'Version': version,
        'NodeList': NodeList,
        'HostGroupList': hostgrouplist,
        'HostNameList': hostnamelist,
        'Time': time_now,
        'Duration': timeout,
        'Timeout': timeout,
        'EnvParameters': EnvParam,
        'LocalParameters': LocalParam,
        'FileParameters': FileParam,
        'CallBack': callback_flag,
        'Result': {
            "StatusCode": 100,
            "StatusMessage": 'PENDING',
            "ExpectedDuration": str(timeout) + "sec"
        },
        'Log': '',
        'Output': {},
        'Path': PlaybookDir,
        'Mandatory': None
    }
    if AnsibleServer != 'na':
        TestRecord[Id]['Result']["AnsibleServer"] = str(AnsibleServer),

    cherrypy.log("Test_Record: " + str(TestRecord[Id]))

    # Write files
    if TestRecord[Id]['FileParameters']:
        for key in TestRecord[Id]['FileParameters']:
            filename = key
            filecontent = TestRecord[Id]['FileParameters'][key]
            f = open(PlaybookDir + "/" + filename, "w")
            f.write(filecontent)
            f.close()

    # Process playbook
    if os.path.exists(ANSIBLE_PATH + '/' + PlaybookName):
        AnsiblePlaybookFail = False

    if AnsiblePlaybookFail:
        # if os.path.exists(PlaybookDir):
        # shutil.rmtree (PlaybookDir)
        cherrypy.log("AnsiblePlaybookFail")
        del TestRecord[Id]
        return {"StatusCode": 101, "StatusMessage": "PLAYBOOK NOT FOUND"}
    else:
        # Test EnvParameters
        playbook_path = PlaybookDir

        # Store local vars
        store_local_vars(playbook_path, Id)

        # write some info out to files before running
        if AUTH:
            f = open(playbook_path + "/User.txt", "a")
            f.write(cherrypy.request.login)
            f.close()

        f = open(playbook_path + "/PlaybookName.txt", "a")
        f.write(PlaybookName)
        f.close()

        f = open(playbook_path + "/PlaybookExDir.txt", "a")
        f.write(PlaybookDir + "/" + PlayBookFunction)
        f.close()

        f = open(playbook_path + "/JsonRequest.txt", "w")
        f.write(json.dumps(input_json, indent=4, sort_keys=True))
        f.close()

        # Check that HostFile exists
        if not os.path.isfile(HostFile):
            cherrypy.log("Inventory file Not Found: " + HostFile)
            return {"StatusCode": 101, "StatusMessage": "PLAYBOOK INVENTORY FILE NOT FOUND"}

        # Cannot use thread because ansible module uses signals which are only supported in main thread.
        # So use multiprocess with shared object
        p = Process(target=RunAnsible_Playbook,
                    args=(callback, Id, HostFile, PlaybookDir + '/' + PlayBookFile,
                          NodeList, TestRecord, PlaybookDir + "/" + PlayBookFunction, ArchiveFlag))
        p.start()
        ActiveProcess[Id] = p
        return TestRecord[Id]['Result']


def handle_post_method(input_json, time_now):
    cherrypy.log("Payload: " + str(input_json))

    if 'Id' in input_json and 'PlaybookName' in input_json and 'EnvParameters' in input_json:
        if input_json['Id'] not in TestRecord:
            # check if Id exists in previous run directory, if so return error
            Id = input_json['Id']
            if glob.glob(ANSIBLE_TEMP + '/*_' + input_json['Id']):
                cherrypy.log("Old directory found for ID: " + Id)
                return {"StatusCode": 101, "StatusMessage": "TEST ID FILE ALREADY DEFINED"}

            # if required it should be passed as an argument
            EnvParameters = input_json.get('EnvParameters', {})

            # The lines below are to test multiple EnvParameters being passed
            # for i in EnvParameters:
            #   cherrypy.log("EnvParameter object: " + i)
            #   cherrypy.log("  EnvParameter Value: " + EnvParameters[ i ])

            pnf_flag = EnvParameters.get("pnf-flag", "")
            if pnf_flag == "true":
                return process_pnf_playbook(input_json, Id, EnvParameters, time_now)
            else:
                return process_vnf_playbook(input_json, Id, EnvParameters, time_now)
        else:
            cherrypy.log("TEST ID ALREADY DEFINED")
            return {"StatusCode": 101, "StatusMessage": "TEST ID ALREADY DEFINED"}
    else:
        return {"StatusCode": 500, "StatusMessage": "JSON OBJECT MUST INCLUDE: ID, PLAYBOOKNAME, EnvParameters"}


def handle_get_method(input_data):
    # Verify we have a Type passed in GET request
    if 'Type' not in input_data:
        return {"StatusCode": 500, "StatusMessage": "RESULTS TYPE UNDEFINED"}

    if AUTH:
        cherrypy.log("Request USER:             " + cherrypy.request.login)
    cherrypy.log("Payload: " + str(input_data) + " Type " + input_data['Type'])

    if 'LogRest' in input_data['Type']:
        sys.stdout.close()
        sys.stdout = open("/var/log/RestServer.log", "w")

    # Just a debug to dump any records
    if 'GetStatus' in input_data['Type']:
        cherrypy.log("******** Dump Records **********")
        if list(TestRecord.items()):
            for id, record in list(TestRecord.items()):
                cherrypy.log("    Id: " + id)
                cherrypy.log("Record: " + str(record))
        else:
            cherrypy.log(" No Records to dump")

    if 'Id' in input_data and 'Type' in input_data:
        if not ('GetResult' in input_data['Type'] or 'GetOutputLog' in input_data['Type'] or
                'GetTheOutput' in input_data['Type'] or 'GetOutput' in input_data['Type'] or
                'GetLog' in input_data['Type']):
            return {"StatusCode": 500, "StatusMessage": "RESULTS TYPE UNDEFINED"}

        if input_data['Id'] in TestRecord:
            if 'GetResult' in input_data['Type']:
                cherrypy.log(" ** GetResult for: " + str(input_data['Id']))
                if 'StatusMessage' in TestRecord[input_data['Id']]['Result'] and getresults_block:
                    # check if playbook is still running
                    while ActiveProcess[input_data['Id']].is_alive():
                        cherrypy.log("*** Playbook running returning PENDING for " + str(input_data['Id']))
                        # If still running return PENDING response
                        # if AnsibleServer != 'na':
                        #     return {"StatusCode": 100, "StatusMessage": 'PENDING', "AnsibleServer": str(AnsibleServer)}
                        # else:
                        #    return {"StatusCode": 100, "StatusMessage": 'PENDING'}
                        time.sleep(3)

                    # cherrypy.log( "*** Request released " + input_data['Id'])

                cherrypy.log(str(TestRecord[input_data['Id']]['Result']))
                cherrypy.log("Output: " + str(TestRecord[input_data['Id']]['Output']))
                cherrypy.log("StatusCode: " + str(TestRecord[input_data['Id']]['Result']['StatusCode']))
                cherrypy.log("StatusMessage: " + str(TestRecord[input_data['Id']]['Result']['StatusMessage']))

                # out_obj gets returned to GET request
                if TestRecord[input_data['Id']]['Result']['StatusCode'] == 500:
                    out_obj = TestRecord[input_data['Id']]['Result']['Results']
                else:
                    out_obj = {
                        "StatusCode": 200,
                        "StatusMessage": "FINISHED",
                        "PlaybookName": TestRecord[input_data['Id']]["PlaybookName"],
                        "Version": TestRecord[input_data['Id']]["Version"],
                        "Duration": TestRecord[input_data['Id']]["Duration"],
                        "Output": TestRecord[input_data['Id']]["Output"]["Output"],
                        "Results": TestRecord[input_data['Id']]['Result']['Results']
                    }
                if not TestRecord[input_data['Id']]['Output']['Output'] == {}:
                    cherrypy.log("TestRecord has Output:" + str(TestRecord[input_data['Id']]['Output']['Output']))
                    # PAP
                    for key in out_obj["Results"]:
                        cherrypy.log("Output key: " + str(key))
                        if key in TestRecord[input_data['Id']]['Output']['Output']:
                            out_obj["Results"][key]["Output"] = TestRecord[input_data['Id']]['Output']['Output'][key]

                cherrypy.log("***** GET RETURNING RESULTS Back ****")
                cherrypy.log(str(out_obj))
                return out_obj
            elif 'GetStatus' in input_data['Type']:
                cherrypy.log(" Dump Records")
                for id, record in list(TestRecord.items()):
                    cherrypy.log(" id: " + id)
                    cherrypy.log("   Record:" + str(record))
            elif 'GetTheOutput' in input_data['Type'] or 'GetOutput' in input_data['Type']:
                if TestRecord[input_data['Id']]['Output'] == {} and getresults_block:
                    cherrypy.log("*** Request blocked " + input_data['Id'])

                    # while TestRecord[input_data['Id']]['Output'] == {} \
                    #         or 'StatusMessage' in TestRecord[input_data['Id']]['Result']:
                    while ActiveProcess[input_data['Id']].is_alive():
                        time.sleep(3)

                    cherrypy.log("*** Request released " + input_data['Id'])

                cherrypy.log("Output: " + str(TestRecord[input_data['Id']]['Output']))
                return {"Output": TestRecord[input_data['Id']]['Output']['Output']}
            elif 'GetOutputLog' in input_data['Type']:
                cherrypy.log("GetOutputLog: processing.")
                if glob.glob(ANSIBLE_TEMP + '/*_' + input_data['Id']):
                    id = input_data['Id']
                    cherrypy.log("Old directory found for ID: " + id)
                    run_dir = glob.glob(ANSIBLE_TEMP + '/*_' + input_data['Id'])
                    for dir in run_dir:
                        rdir = dir
                    if os.path.exists(rdir + "/PlaybookExDir.txt"):
                        cherrypy.log("Found PlaybookExDir.txt file")
                        f = open(rdir + '/PlaybookExDir.txt', 'r')
                        playbookexdir = f.readline()
                        rdir = playbookexdir
                        f.close()
                    cherrypy.log("Id:     " + id)
                    cherrypy.log("RunDir: " + rdir)
                    if os.path.exists(rdir + "/output.log"):
                        cherrypy.log("Found output.log file")
                        f = open(rdir + '/output.log', 'r')
                        output_log = f.readline()
                        f.close()
                        return output_log
                else:
                    cherrypy.log("Globglob failed:")
                    return

            else:
                # GetLog
                if TestRecord[input_data['Id']]['Log'] == '' and \
                        getresults_block:

                    cherrypy.log("*** Request blocked " + input_data['Id'])

                    while TestRecord[input_data['Id']]['Log'] == '' \
                            or 'StatusMessage' in TestRecord[input_data['Id']]['Result']:
                        time.sleep(5)

                    cherrypy.log("*** Request released " + input_data['Id'])

                cherrypy.log("Log:" + str(TestRecord[input_data['Id']]['Log']))
                return {"Log": TestRecord[input_data['Id']]['Log']}
        else:
            # Not in memory check for a file
            if glob.glob(ANSIBLE_TEMP + '/*_' + input_data['Id']):
                id = input_data['Id']
                cherrypy.log("Old directory found for ID: " + id)
                run_dir = glob.glob(ANSIBLE_TEMP + '/*_' + input_data['Id'])
                for dir in run_dir:
                    rdir = dir
                if os.path.exists(rdir + "/PlaybookExDir.txt"):
                    cherrypy.log("Found PlaybookExDir.txt file")
                    f = open(rdir + '/PlaybookExDir.txt', 'r')
                    playbookexdir = f.readline()
                    rdir = playbookexdir
                    f.close()
                cherrypy.log("Id:     " + id)
                cherrypy.log("RunDir: " + rdir)
                if 'GetLog' in input_data['Type']:
                    if os.path.exists(rdir + "/output.log"):
                        cherrypy.log("Found output.log file")
                        f = open(rdir + '/output.log', 'r')
                        output_log = f.readline()
                        f.close()
                        return output_log
                elif 'GetOutputLog' in input_data['Type']:
                    if os.path.exists(rdir + "/output.log"):
                        cherrypy.log("Found output.log file")
                        f = open(rdir + '/output.log', 'r')
                        output_log = f.readline()
                        f.close()
                        return output_log
                elif 'GetResult' in input_data['Type']:
                    if os.path.exists(rdir + "/PlaybookName.txt"):
                        cherrypy.log("Found PlaybookName.txt file")
                        f = open(rdir + '/PlaybookName.txt', 'r')
                        playbooknametxt = f.readline()
                        f.close()
                    else:
                        playbooknametxt = "NA"

                    # Add code to get other items not just output.log from files
                    if os.path.exists(rdir + "/log.file"):
                        cherrypy.log("Found log.file")
                        out_results = "NA:"

                        f = open(rdir + '/log.file', 'r')
                        line = f.readline()
                        while line:
                            if "fatal" in line:
                                out_results = out_results + line
                            elif "RECAP" in line:
                                out_results = out_results + line
                                recap_line = f.readline()
                                while recap_line:
                                    out_results = out_results + recap_line
                                    recap_line = f.readline()
                            line = f.readline()
                        f.close()
                        out_obj = {
                            "StatusCode": 200,
                            "StatusMessage": "FINISHED",
                            "PlaybookName": playbooknametxt,
                            "Version": "Version",
                            "Duration": 200,
                            "Results": out_results
                        }
                        return out_obj
                else:
                    return {"StatusCode": 500, "StatusMessage": "PLAYBOOK FAILED "}

            return {"StatusCode": 500, "StatusMessage": "TEST ID UNDEFINED"}
    else:
        return {"StatusCode": 500, "StatusMessage": "MALFORMED REQUEST"}


def handle_delete_method(input_data):
    cherrypy.log("***> in RestServer.DELETE")
    cherrypy.log("Payload: " + str(input_data))

    if input_data['Id'] in TestRecord:
        if 'PENDING' not in TestRecord[input_data['Id']]['Result']:
            cherrypy.log(" Path: " + str(TestRecord[input_data['Id']]['Path']))
            TestRecord.pop(input_data['Id'])
            if input_data['Id'] in ActiveProcess:
                ActiveProcess.pop(input_data['Id'])
            return {"StatusCode": 200, "StatusMessage": "PLAYBOOK EXECUTION RECORDS DELETED"}
        else:
            return {"StatusCode": 200, "StatusMessage": "PENDING"}
    else:
        return {"StatusCode": 500, "StatusMessage": "TEST ID UNDEFINED"}


class TestManager(object):
    @cherrypy.expose
    @cherrypy.tools.json_out()
    @cherrypy.tools.json_in()
    @cherrypy.tools.allow(methods=['POST', 'GET', 'DELETE'])
    def Dispatch(self, **kwargs):
        # Let cherrypy error handler deal with malformed requests
        # No need for explicit error handler, we use default ones

        time_now = datetime.datetime.utcnow()

        # Erase old test results (2x timeout)
        # Do cleanup too of ActiveProcess list and old Records - PAP
        if TestRecord:
            for key in TestRecord.copy():
                cherrypy.log("LOOKING AT ALL TestRecords: " + str(key))
                if key in ActiveProcess:
                    if not ActiveProcess[key].is_alive():  # Just to cleanup defunct processes
                        cherrypy.log("Not ActiveProcess for ID: " + str(key))
                delta_time = (time_now - TestRecord[key]['Time']).seconds
                if delta_time > 2 * TestRecord[key]['Timeout']:
                    cherrypy.log("DELETED HISTORY for ID: " + str(key))
                    if key in ActiveProcess:
                        if not ActiveProcess[key].is_alive():
                            ActiveProcess.pop(key)
                            cherrypy.log("DELETED ActiveProcess for ID: " + str(key))
                    # if os.path.exists(TestRecord[key]['Path']):
                    # don't remove run dirrectory
                    # shutil.rmtree (TestRecord[key]['Path'])
                    del TestRecord[key]

        cherrypy.log("RestServer.Dispatch: " + cherrypy.request.method)

        if 'POST' in cherrypy.request.method:
            input_json = cherrypy.request.json
            return handle_post_method(input_json, time_now)
        elif 'GET' in cherrypy.request.method:
            # Lets pause for a second just in case the request was just kicked off
            time.sleep(1)

            input_data = parse_query_string(cherrypy.request.query_string)
            return handle_get_method(input_data)
        elif 'DELETE' in cherrypy.request.method:
            input_data = parse_query_string(cherrypy.request.query_string)
            return handle_delete_method(input_data)


if __name__ == '__main__':

    # Read configuration

    config_file_path = "RestServer_config"

    if not os.path.exists(config_file_path):
        cherrypy.log('[INFO] The config file does not exist')
        sys.exit(0)

    ip = 'na'
    AnsibleServer = 'na'
    port = 'na'
    tls = False
    AUTH = False
    pub = 'na'
    priv = 'na'
    intermediate = 'na'
    timeout_seconds = 'na'
    ANSIBLE_PATH = 'na'
    ANSIBLE_TEMP = 'na'
    host = 'na'
    users = 'na'
    getresults_block = False
    from_files = False

    config_file = open(config_file_path, 'r')
    for config_line in config_file.readlines():
        if '#' not in config_line:
            if 'ip:' in config_line:
                ip = config_line.split(':')[1].strip()
            elif 'AnsibleServer:' in config_line:
                AnsibleServer = config_line.split(':')[1].strip()
            elif 'port:' in config_line:
                port = config_line.split(':')[1].strip()
            elif 'ksalt:' in config_line:
                salt = config_line.split(':')[1].strip()
            elif 'tls:' in config_line:
                tls = 'YES' in config_line.split(':')[1].strip().upper()
            elif 'auth:' in config_line:
                AUTH = 'YES' in config_line.split(':')[1].strip().upper()
            if tls and 'priv:' in config_line:
                priv = config_line.split(':')[1].strip()
            if tls and 'pub:' in config_line:
                pub = config_line.split(':')[1].strip()
            if tls and 'inter_cert:' in config_line:
                intermediate = config_line.split(':')[1].strip()
            if 'timeout_seconds' in config_line:
                timeout_seconds = int(config_line.split(':')[1].strip())
            if 'ansible_path' in config_line:
                ANSIBLE_PATH = config_line.split(':')[1].strip()
            if 'ansible_inv' in config_line:
                ANSIBLE_INV = config_line.split(':')[1].strip()
                if not os.path.exists(ANSIBLE_PATH + "/" + ANSIBLE_INV):
                    print('[INFO] The ansible_inv file does not exist')
                    sys.exit(0)
            if 'ansible_temp' in config_line:
                ANSIBLE_TEMP = config_line.split(':')[1].strip()
            if 'host' in config_line:
                host = config_line.split(':')[1].strip()
            if 'users' in config_line:
                users = config_line.split(':')[1].strip()
            if 'getresults_block' in config_line:
                getresults_block = 'YES' in config_line.split(':')[1].strip().upper()
            if 'from_files' in config_line:
                from_files = 'YES' in config_line.split(':')[1].strip().upper()
    config_file.close()

    # Initialization

    global_conf = {
        'global': {
            'log.screen': True,
            'response.timeout': 5400,
            'server.socket_host': ip,
            'server.socket_port': int(port),
            'server.protocol_version': 'HTTP/1.1'
        }
    }

    if tls:
        # Use pythons built-in SSL
        cherrypy.server.ssl_module = 'builtin'

        # Point to certificate files

        if not os.path.exists(pub):
            cherrypy.log('[INFO] The public certificate does not exist')
            sys.exit(0)

        if not os.path.exists(priv):
            cherrypy.log('[INFO] The private key does not exist')
            sys.exit(0)

        if not os.path.exists(intermediate):
            cherrypy.log('[INFO] The intermediate certificate does not exist')
            sys.exit(0)

        cherrypy.server.ssl_certificate = pub
        cherrypy.server.ssl_certificate_chain = intermediate
        cherrypy.server.ssl_private_key = priv

    if AUTH:
        # Read in and build user dictionary
        if not os.path.exists(users):
            cherrypy.log('[INFO] The users file does not exist: ' + users)
            sys.exit(0)
        userpassdict = {}
        user_file = open(users, 'r')
        for config_line in user_file.readlines():
            if '#' not in config_line:
                uid = config_line.split(':')[0].strip()
                pw = config_line.split(':')[1].strip()
                userpassdict[uid] = pw

        app_conf = {
            '/':
                {'tools.auth_basic.on': True,
                 'tools.auth_basic.realm': 'earth',
                 'tools.auth_basic.checkpassword': validate_password
                 }
        }

        application = cherrypy.tree.mount(TestManager(), '/', app_conf)
    else:
        application = cherrypy.tree.mount(TestManager(), '/')

    cherrypy.config.update({
        'log.access_file': "/var/log/RestServer.access"
    })
    accessLogName = "/var/log/RestServer.access"
    applicationLogName = "/var/log/RestServer.log"
    cherrypy.config.update(global_conf)

    log = application.log
    log.error_file = ""
    log.access_file = ""
    from logging import handlers

    applicationLogFileHandler = handlers.RotatingFileHandler(applicationLogName, 'a', 1000000, 5000)
    accessLogFileHandler = handlers.RotatingFileHandler(accessLogName, 'a', 1000000, 5000)
    import logging

    applicationLogFileHandler.setLevel(logging.DEBUG)
    log.error_log.addHandler(applicationLogFileHandler)
    log.access_log.addHandler(accessLogFileHandler)

    # Start server

    cherrypy.engine.start()
    cherrypy.engine.block()