push addional code

Change-Id: Ia427bb3460cda3a896f8faced2de69eaf3807b74
Signed-off-by: Michael Lando <ml636r@att.com>

1 files changed, 435 insertions, 0 deletions

diff --git a/openecomp-be/tools/scripts/generate-manifest.py b/openecomp-be/tools/scripts/generate-manifest.py
new file mode 100644
index 0000000000..87968e0f89
--- /dev/null
+++ b/openecomp-be/tools/scripts/generate-manifest.py
@@ -0,0 +1,435 @@
+#!/usr/bin/python
+
+##############################################################################
+###
+### generate-manifest.py
+###
+### A Vendor utility to generate a valid heat zip manifest file for the AT&T onboarding. 
+###
+### Usage:
+###
+###    generate-manifest.py [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description
+###
+### For example:
+###
+###    ./generate-manifest.py --folder ./vota --name vOTA --description "HOT template to create vOTA server"
+###
+### Help:
+### The script is doing the following:
+###  1) Split the files into different types
+###     a. .env files
+###     b. Network files (anything containing the string network)
+###     c. Volume files (anything containing the string volume)
+###     d. Top level Heat files
+###     e. Other types
+###  2) Match env files to heat files – looking for same name ignoring suffix and extension
+###  3) Match Network childs
+###     a. Look for Top level heats which name is a substring of the name of the Network heat name.
+###  4) Match Volume childs
+###     a. Look for Top level heats which name is a substring of the name of the Volume heat name.
+###  5)      Generate the JSON file from the above
+###
+###
+### Author: Avi Ziv
+### Version 1.4 for OPENECOMP 1.0
+### Date: 13 July 2016 (c) OPENECOMP
+###
+##############################################################################
+
+# import os,sys,getopt,json,re
+import os, sys, getopt, re
+from collections import OrderedDict
+from json import JSONEncoder
+import json
+
+VERSION = "1.4"
+ENV_EXT = ".env"
+SHELL_EXT = ".sh"
+YAML_EXT = [".yaml", ".yml"]
+# VERSION_DELIMITER_PATTERN='_v\d{*}.\d{*}'
+# VERSION_DELIMITER_PATTERN='_v*.*'
+#v1.0
+VERSION_DELIMITER_PATTERN = '_v\d+.\d+'
+#07_12_2016
+VERSION_DELIMITER_PATTERN2 = '_\d+-\d+-\d+'
+
+# types
+HEAT = "HEAT"
+HEAT_BASE = "HEAT_BASE"
+HEAT_NET = "HEAT_NET"
+HEAT_VOL = "HEAT_VOL"
+HEAT_ENV = "HEAT_ENV"
+SHELL = "SHELL"
+OTHER = "OTHER"
+
+globalVolumeVal = "VOLUME"
+globalNetworkVal = "NETWORK"
+globalBaseVal = "BASE"
+
+
+def version():
+    return VERSION
+
+
+def usage():
+    print ('usage: ' + sys.argv[0] + ' [-f|--folder] vendor-heat-directory [-n|--name] manifest-name [-d|--description] manifet-description' )
+
+
+def header():
+    print ("\nASDC Vendor manifest file generator, version " + version() + "\n")
+
+
+def getEnvVariables(value, defaultValue):
+    try:
+        eVal = os.environ[value]
+        return eVal
+    except KeyError:
+        print ("Missing ${" + value + "} envirunment variable. Using default value: " + defaultValue)
+    return defaultValue
+
+
+def getF(listFiles):
+    print ("Analyzing files ...")
+    foundABase = False
+    files = listFiles
+    jsons = {}
+    lOfEnvs = {}
+    lOfVolumes = {}
+    lOfNetworks = {}
+    lOfHeats = {}
+    lOfShels = {}
+    lOfArtifacts = {}
+
+    for f in files:
+        fullFilename = f[1]
+        fObj = ManifestFileInfo(fullFilename)
+        if fObj.isEnv():
+            lOfEnvs[fObj.file_name] = fObj
+        elif fObj.isShell():
+            lOfShels[fObj.file_name] = fObj
+        elif fObj.isVolume():
+            lOfVolumes[fObj.file_name] = fObj
+        elif fObj.isNetwork():
+            lOfNetworks[fObj.file_name] = fObj
+        elif (fObj.isYaml() and not fObj.isBase()):
+            lOfHeats[fObj.file_name] = fObj
+        elif fObj.isArtifact():
+            lOfArtifacts[fObj.file_name] = fObj
+        elif (fObj.isBase() and fObj.isYaml()):
+            foundABase = True
+            lOfHeats[fObj.file_name] = fObj
+
+    jsons['heats'] = lOfHeats
+    jsons['envs'] = lOfEnvs
+    jsons['shells'] = lOfShels
+    jsons['volumes'] = lOfVolumes
+    jsons['networks'] = lOfNetworks
+    jsons['artifacts'] = lOfArtifacts
+
+    if not foundABase:
+        print (">>> Warning: No Base was found")
+    return jsons
+
+def loadFilesToList(folder):
+    print ("Analyzing files in folder: << " + folder + " >>")
+    files = os.listdir(folder)
+    listOfFiles = []
+    for f in files:
+        if os.path.isdir(os.path.join(folder, f)):
+            ConsoleLogger.warning("Sub folders are ignored by this script, you may want to remove it before archiving")
+            continue
+
+        filename, file_extension = os.path.splitext(f)
+        if filename == 'MANIFEST':
+            ConsoleLogger.warning("Your folder already contains a manifest file that will be overridden")
+            continue
+        listOfFiles.append([filename, f])
+    return listOfFiles
+
+
+def make(files):
+    flist = []
+    dEnvs = {}
+    dEnvs = files['envs']
+    dHeats = files['heats']
+    dNetworks = files['networks']
+    dVolumes = files['volumes']
+    dArtifacts = files['artifacts']
+    dShells = files['shells']
+
+    env_items = dEnvs.items()
+    for fileItem in env_items:
+        env_name = fileItem[1].file_name
+        env_base = fileItem[1].base_file_name
+        if env_name in dHeats:
+            dHeats[env_name].add(fileItem[1])
+            continue
+
+        if env_name in dNetworks.items():
+            dNetworks[env_name].add(fileItem[1])
+            continue
+
+        if env_name in dVolumes.items():
+            dVolumes[env_name[0]].add(env_name[1])
+            continue
+
+        for fName in dHeats:
+            heat_base = dHeats[fName].base_file_name
+            if env_base in heat_base:
+                dHeats[fName].add(dEnvs[env_name])
+                break
+        else:
+            for fName in dNetworks:
+                net_base = dNetworks[fName].base_file_name
+                if env_base in net_base:
+                    dNetworks[fName].add(dEnvs[env_name])
+                    break
+            else:
+                for fName in dVolumes:
+                    vol_base = dVolumes[fName].base_file_name
+                    if env_base in vol_base:
+                        dVolumes[fName].add(dEnvs[env_name])
+                        break
+
+                else:
+                    flist.append(dEnvs[env_name])
+
+    for fName in dVolumes:
+        vol_base = dVolumes[fName].base_file_name
+        for hfName in dHeats:
+            heat_base = dHeats[hfName].base_file_name
+            if heat_base in vol_base:
+                dHeats[hfName].add(dVolumes[fName])
+                break
+        else:
+            flist.append(dVolumes[fName])
+
+    for fName in dNetworks:
+        net_base = dNetworks[fName].base_file_name
+        for hfName in dHeats:
+            heat_base = dHeats[hfName].base_file_name
+            if heat_base in net_base:
+                dHeats[hfName].add(dNetworks[fName])
+                break
+        else:
+            flist.append(dNetworks[fName])
+
+    for fName in dHeats:
+        flist.append(dHeats[fName])
+    for fName in dShells:
+        flist.append(dShells[fName])
+    for fName in dArtifacts:
+        flist.append(dArtifacts[fName])
+
+    print ("\n------------------------------------------------------------\n")
+    return flist
+
+
+def generate(folder, name, description):
+    print ("Checking envirunment variables ...")
+    global globalVolumeVal
+    globalVolumeVal = getEnvVariables("VOLUME", globalVolumeVal)
+
+    global globalNetworkVal
+    globalNetworkVal = getEnvVariables("NETWORK", globalNetworkVal)
+
+    global globalBaseVal
+    globalBaseVal = getEnvVariables("BASE", globalBaseVal)
+
+    YamlTabCleaner(folder).cleanYamlTabs()
+
+    print ("Generating manifest file ...")
+    jsons = getF(loadFilesToList(folder))
+
+    lFiles = make(jsons)
+    manifest = Manifest(name, description, '1.0', lFiles)
+    output_json = json.dumps(manifest, default=jdefault, indent=4, sort_keys=False)
+
+    f = open(os.path.join(folder, 'MANIFEST.json'), 'w')
+    f.write(output_json)
+    print("MANIFEST file created")
+
+
+################
+
+def jdefault(obj):
+    if hasattr(obj, '__json__'):
+        return obj.__json__()
+    else:
+        return obj.__dict__
+
+
+class ManifestFileInfo(object):
+    def __init__(self, filename):
+        self.name = filename
+        self.base = 'false'
+        self.data = []
+        self.file_name, self.file_extension = os.path.splitext(filename)
+        self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN, '', self.file_name)
+        self.base_file_name = re.sub(VERSION_DELIMITER_PATTERN2, '', self.base_file_name)
+
+        if self.isEnv():
+            self.heat_type = Types.ENV
+        elif self.isShell():
+            self.heat_type = Types.SHELL
+        elif self.isVolume():
+            self.heat_type = Types.VOL
+        elif self.isNetwork():
+            self.heat_type = Types.NET
+        elif self.isYaml() and not self.isBase():
+            self.heat_type = Types.HEAT
+        elif self.isArtifact():
+            self.heat_type = Types.OTHER
+        elif (self.isBase() and self.isYaml()):
+            self.heat_type = Types.HEAT
+            self.base = 'true'
+
+    def set(self, data):
+        self.data = data
+
+    def add(self, item):
+        self.data.append(item)
+
+    def isYaml(self):
+        return any(val in self.file_extension.lower() for val in YAML_EXT)
+
+    def isEnv(self):
+        return self.file_extension.lower() == ENV_EXT.lower()
+
+    def isShell(self):
+        return self.file_extension.lower() == SHELL_EXT.lower()
+
+    def isVolume(self):
+        res = globalVolumeVal.lower() in self.file_name.lower()
+        return res
+
+    def isNetwork(self):
+        res = globalNetworkVal.lower() in self.file_name.lower()
+        return res
+
+    def isBase(self):
+        res = globalBaseVal.lower() in self.file_name.lower()
+        return res
+
+    def isArtifact(self):
+        return (not self.isBase() and not self.isVolume() and not self.isNetwork() and not self.isEnv())
+
+    def isHEAT(self):
+        return ((self.heat_type == Types.HEAT) | (self.heat_type == Types.BASE) | (self.heat_type == Types.NET) | (
+            self.heat_type == Types.VOL))
+
+    def __json__(self):
+        dict = OrderedDict(
+            [('file', self.name), ('type', self.heat_type)])
+        if self.isHEAT():
+            dict['isBase'] = self.base
+            if self.data != []:
+                dict['data'] = self.data
+
+        return dict
+
+
+class Manifest(object):
+    def __init__(self, name, description, version, data):
+        self.name = name
+        self.description = description
+        self.version = version
+        self.data = data
+
+    def add(self, data):
+        self.data.append(data)
+
+    def __json__(self):
+        return OrderedDict([('name', self.name), ('description', self.description), ('data', self.data)])
+
+
+class YamlTabCleaner(object):
+    def __init__(self, folder):
+        self.folder = folder
+
+    def replaceTabs(self, sourceFile, targetFile):
+        with open(sourceFile, "rt") as fin:
+            if '\t' in fin.read():
+                print("\'tab\' character was found in the file: " + sourceFile + "\na clean version of the file can be found under \'clean\' folder")
+                target = os.path.dirname(targetFile)
+                if not os.path.exists(target):
+                    os.makedirs(target)
+                fin.seek(0)
+                with open(targetFile, "wt") as fout:
+                    for line in fin:
+                        fout.write(line.replace('\t', ' '))
+
+    def cleanYamlTabs(self):
+        included_extenstions = ['yml', 'yaml']
+        files = [fn for fn in os.listdir(self.folder)
+                 if any(fn.endswith(ext) for ext in included_extenstions)]
+        target = os.path.join(self.folder, "clean")
+        for file in files:
+            self.replaceTabs(os.path.join(self.folder, file), os.path.join(target, file))
+
+class ConsoleLogger(object):
+    @classmethod
+    def error(cls, message):
+        print(">>> Error: " + message)
+
+    @classmethod
+    def warning(cls, message):
+        print(">>> Warning: " + message)
+
+
+    @classmethod
+    def info(cls, message):
+        print(">>> Info: " + message)
+
+
+def enum(**named_values):
+    return type('Enum', (), named_values)
+
+
+################
+
+def main(argv):
+    action = ''
+    folderName = '.'
+    name = ''
+    description = ''
+    version = ''
+
+    try:
+        opts, args = getopt.getopt(argv, "h:f:n:d", ["folder=", "name=", "description=", ])
+    except getopt.GetoptError as err:
+        # print help information and exit:
+        print ('>>>>' + str(err))
+        usage()
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-h':
+            usage()
+            sys.exit()
+        elif opt in ('-f', '--folder'):
+            action = 'generate'
+            if not arg:
+                print ("Error: missing heat files directory")
+                usage()
+                sys.exit(2)
+            else:
+                folderName = arg
+        elif opt in ('-n', '--name'):
+            name = arg
+        elif opt in ('-d', '--description'):
+            description = arg
+        else:
+            usage()
+
+    if action == 'generate':
+        generate(folderName, name, description)
+        sys.exit()
+    else:
+        usage()
+
+
+if __name__ == "__main__":
+    header()
+    Types = enum(HEAT='HEAT', BASE='HEAT_BASE', NET='HEAT_NET', VOL='HEAT_VOL', ENV='HEAT_ENV', SHELL='SHELL',
+                 OTHER='OTHER')
+    main(sys.argv[1:])