diff options
Diffstat (limited to 'openecomp-be/tools/scripts/generate-manifest.py')
| -rw-r--r-- | openecomp-be/tools/scripts/generate-manifest.py | 435 |
1 files changed, 435 insertions, 0 deletions
diff --git a/openecomp-be/tools/scripts/generate-manifest.py b/openecomp-be/tools/scripts/generate-manifest.py new file mode 100644 index 0000000000..87968e0f89 --- /dev/null +++ b/openecomp-be/tools/scripts/generate-manifest.py @@ -0,0 +1,435 @@ +#!/usr/bin/python + +############################################################################## +### +### generate-manifest.py +### +### A Vendor utility to generate a valid heat zip manifest file for the AT&T onboarding. +### +### Usage: +### +### generate-manifest.py [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description +### +### For example: +### +### ./generate-manifest.py --folder ./vota --name vOTA --description "HOT template to create vOTA server" +### +### Help: +### The script is doing the following: +### 1) Split the files into different types +### a. .env files +### b. Network files (anything containing the string network) +### c. Volume files (anything containing the string volume) +### d. Top level Heat files +### e. Other types +### 2) Match env files to heat files – looking for same name ignoring suffix and extension +### 3) Match Network childs +### a. Look for Top level heats which name is a substring of the name of the Network heat name. +### 4) Match Volume childs +### a. Look for Top level heats which name is a substring of the name of the Volume heat name. +### 5) Generate the JSON file from the above +### +### +### Author: Avi Ziv +### Version 1.4 for OPENECOMP 1.0 +### Date: 13 July 2016 (c) OPENECOMP +### +############################################################################## + +# import os,sys,getopt,json,re +import os, sys, getopt, re +from collections import OrderedDict +from json import JSONEncoder +import json + +VERSION = "1.4" +ENV_EXT = ".env" +SHELL_EXT = ".sh" +YAML_EXT = [".yaml", ".yml"] +# VERSION_DELIMITER_PATTERN='_v\d{*}.\d{*}' +# VERSION_DELIMITER_PATTERN='_v*.*' +#v1.0 +VERSION_DELIMITER_PATTERN = '_v\d+.\d+' +#07_12_2016 +VERSION_DELIMITER_PATTERN2 = '_\d+-\d+-\d+' + +# types +HEAT = "HEAT" +HEAT_BASE = "HEAT_BASE" +HEAT_NET = "HEAT_NET" +HEAT_VOL = "HEAT_VOL" +HEAT_ENV = "HEAT_ENV" +SHELL = "SHELL" +OTHER = "OTHER" + +globalVolumeVal = "VOLUME" +globalNetworkVal = "NETWORK" +globalBaseVal = "BASE" + + +def version(): + return VERSION + + +def usage(): + print ('usage: ' + sys.argv[0] + ' [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description' ) + + +def header(): + print ("\nASDC Vendor manifest file generator, version " + version() + "\n") + + +def getEnvVariables(value, defaultValue): + try: + eVal = os.environ[value] + return eVal + except KeyError: + print ("Missing ${" + value + "} envirunment variable. Using default value: " + defaultValue) + return defaultValue + + +def getF(listFiles): + print ("Analyzing files ...") + foundABase = False + files = listFiles + jsons = {} + lOfEnvs = {} + lOfVolumes = {} + lOfNetworks = {} + lOfHeats = {} + lOfShels = {} + lOfArtifacts = {} + + for f in files: + fullFilename = f[1] + fObj = ManifestFileInfo(fullFilename) + if fObj.isEnv(): + lOfEnvs[fObj.file_name] = fObj + elif fObj.isShell(): + lOfShels[fObj.file_name] = fObj + elif fObj.isVolume(): + lOfVolumes[fObj.file_name] = fObj + elif fObj.isNetwork(): + lOfNetworks[fObj.file_name] = fObj + elif (fObj.isYaml() and not fObj.isBase()): + lOfHeats[fObj.file_name] = fObj + elif fObj.isArtifact(): + lOfArtifacts[fObj.file_name] = fObj + elif (fObj.isBase() and fObj.isYaml()): + foundABase = True + lOfHeats[fObj.file_name] = fObj + + jsons['heats'] = lOfHeats + jsons['envs'] = lOfEnvs + jsons['shells'] = lOfShels + jsons['volumes'] = lOfVolumes + jsons['networks'] = lOfNetworks + jsons['artifacts'] = lOfArtifacts + + if not foundABase: + print (">>> Warning: No Base was found") + return jsons + +def loadFilesToList(folder): + print ("Analyzing files in folder: << " + folder + " >>") + files = os.listdir(folder) + listOfFiles = [] + for f in files: + if os.path.isdir(os.path.join(folder, f)): + ConsoleLogger.warning("Sub folders are ignored by this script, you may want to remove it before archiving") + continue + + filename, file_extension = os.path.splitext(f) + if filename == 'MANIFEST': + ConsoleLogger.warning("Your folder already contains a manifest file that will be overridden") + continue + listOfFiles.append([filename, f]) + return listOfFiles + + +def make(files): + flist = [] + dEnvs = {} + dEnvs = files['envs'] + dHeats = files['heats'] + dNetworks = files['networks'] + dVolumes = files['volumes'] + dArtifacts = files['artifacts'] + dShells = files['shells'] + + env_items = dEnvs.items() + for fileItem in env_items: + env_name = fileItem[1].file_name + env_base = fileItem[1].base_file_name + if env_name in dHeats: + dHeats[env_name].add(fileItem[1]) + continue + + if env_name in dNetworks.items(): + dNetworks[env_name].add(fileItem[1]) + continue + + if env_name in dVolumes.items(): + dVolumes[env_name[0]].add(env_name[1]) + continue + + for fName in dHeats: + heat_base = dHeats[fName].base_file_name + if env_base in heat_base: + dHeats[fName].add(dEnvs[env_name]) + break + else: + for fName in dNetworks: + net_base = dNetworks[fName].base_file_name + if env_base in net_base: + dNetworks[fName].add(dEnvs[env_name]) + break + else: + for fName in dVolumes: + vol_base = dVolumes[fName].base_file_name + if env_base in vol_base: + dVolumes[fName].add(dEnvs[env_name]) + break + + else: + flist.append(dEnvs[env_name]) + + for fName in dVolumes: + vol_base = dVolumes[fName].base_file_name + for hfName in dHeats: + heat_base = dHeats[hfName].base_file_name + if heat_base in vol_base: + dHeats[hfName].add(dVolumes[fName]) + break + else: + flist.append(dVolumes[fName]) + + for fName in dNetworks: + net_base = dNetworks[fName].base_file_name + for hfName in dHeats: + heat_base = dHeats[hfName].base_file_name + if heat_base in net_base: + dHeats[hfName].add(dNetworks[fName]) + break + else: + flist.append(dNetworks[fName]) + + for fName in dHeats: + flist.append(dHeats[fName]) + for fName in dShells: + flist.append(dShells[fName]) + for fName in dArtifacts: + flist.append(dArtifacts[fName]) + + print ("\n------------------------------------------------------------\n") + return flist + + +def generate(folder, name, description): + print ("Checking envirunment variables ...") + global globalVolumeVal + globalVolumeVal = getEnvVariables("VOLUME", globalVolumeVal) + + global globalNetworkVal + globalNetworkVal = getEnvVariables("NETWORK", globalNetworkVal) + + global globalBaseVal + globalBaseVal = getEnvVariables("BASE", globalBaseVal) + + YamlTabCleaner(folder).cleanYamlTabs() + + print ("Generating manifest file ...") + jsons = getF(loadFilesToList(folder)) + + lFiles = make(jsons) + manifest = Manifest(name, description, '1.0', lFiles) + output_json = json.dumps(manifest, default=jdefault, indent=4, sort_keys=False) + + f = open(os.path.join(folder, 'MANIFEST.json'), 'w') + f.write(output_json) + print("MANIFEST file created") + + +################ + +def jdefault(obj): + if hasattr(obj, '__json__'): + return obj.__json__() + else: + return obj.__dict__ + + +class ManifestFileInfo(object): + def __init__(self, filename): + self.name = filename + self.base = 'false' + self.data = [] + self.file_name, self.file_extension = os.path.splitext(filename) + self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN, '', self.file_name) + self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN2, '', self.base_file_name) + + if self.isEnv(): + self.heat_type = Types.ENV + elif self.isShell(): + self.heat_type = Types.SHELL + elif self.isVolume(): + self.heat_type = Types.VOL + elif self.isNetwork(): + self.heat_type = Types.NET + elif self.isYaml() and not self.isBase(): + self.heat_type = Types.HEAT + elif self.isArtifact(): + self.heat_type = Types.OTHER + elif (self.isBase() and self.isYaml()): + self.heat_type = Types.HEAT + self.base = 'true' + + def set(self, data): + self.data = data + + def add(self, item): + self.data.append(item) + + def isYaml(self): + return any(val in self.file_extension.lower() for val in YAML_EXT) + + def isEnv(self): + return self.file_extension.lower() == ENV_EXT.lower() + + def isShell(self): + return self.file_extension.lower() == SHELL_EXT.lower() + + def isVolume(self): + res = globalVolumeVal.lower() in self.file_name.lower() + return res + + def isNetwork(self): + res = globalNetworkVal.lower() in self.file_name.lower() + return res + + def isBase(self): + res = globalBaseVal.lower() in self.file_name.lower() + return res + + def isArtifact(self): + return (not self.isBase() and not self.isVolume() and not self.isNetwork() and not self.isEnv()) + + def isHEAT(self): + return ((self.heat_type == Types.HEAT) | (self.heat_type == Types.BASE) | (self.heat_type == Types.NET) | ( + self.heat_type == Types.VOL)) + + def __json__(self): + dict = OrderedDict( + [('file', self.name), ('type', self.heat_type)]) + if self.isHEAT(): + dict['isBase'] = self.base + if self.data != []: + dict['data'] = self.data + + return dict + + +class Manifest(object): + def __init__(self, name, description, version, data): + self.name = name + self.description = description + self.version = version + self.data = data + + def add(self, data): + self.data.append(data) + + def __json__(self): + return OrderedDict([('name', self.name), ('description', self.description), ('data', self.data)]) + + +class YamlTabCleaner(object): + def __init__(self, folder): + self.folder = folder + + def replaceTabs(self, sourceFile, targetFile): + with open(sourceFile, "rt") as fin: + if '\t' in fin.read(): + print("\'tab\' character was found in the file: " + sourceFile + "\na clean version of the file can be found under \'clean\' folder") + target = os.path.dirname(targetFile) + if not os.path.exists(target): + os.makedirs(target) + fin.seek(0) + with open(targetFile, "wt") as fout: + for line in fin: + fout.write(line.replace('\t', ' ')) + + def cleanYamlTabs(self): + included_extenstions = ['yml', 'yaml'] + files = [fn for fn in os.listdir(self.folder) + if any(fn.endswith(ext) for ext in included_extenstions)] + target = os.path.join(self.folder, "clean") + for file in files: + self.replaceTabs(os.path.join(self.folder, file), os.path.join(target, file)) + +class ConsoleLogger(object): + @classmethod + def error(cls, message): + print(">>> Error: " + message) + + @classmethod + def warning(cls, message): + print(">>> Warning: " + message) + + + @classmethod + def info(cls, message): + print(">>> Info: " + message) + + +def enum(**named_values): + return type('Enum', (), named_values) + + +################ + +def main(argv): + action = '' + folderName = '.' + name = '' + description = '' + version = '' + + try: + opts, args = getopt.getopt(argv, "h:f:n:d", ["folder=", "name=", "description=", ]) + except getopt.GetoptError as err: + # print help information and exit: + print ('>>>>' + str(err)) + usage() + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + usage() + sys.exit() + elif opt in ('-f', '--folder'): + action = 'generate' + if not arg: + print ("Error: missing heat files directory") + usage() + sys.exit(2) + else: + folderName = arg + elif opt in ('-n', '--name'): + name = arg + elif opt in ('-d', '--description'): + description = arg + else: + usage() + + if action == 'generate': + generate(folderName, name, description) + sys.exit() + else: + usage() + + +if __name__ == "__main__": + header() + Types = enum(HEAT='HEAT', BASE='HEAT_BASE', NET='HEAT_NET', VOL='HEAT_VOL', ENV='HEAT_ENV', SHELL='SHELL', + OTHER='OTHER') + main(sys.argv[1:]) |