migrate parents to phosphorus-sr1

generate sr1 parents and add script for this

Issue-ID: CCSDK-3565
Signed-off-by: Michael DÜrre <michael.duerre@highstreet-technologies.com>
Change-Id: I53c5bed11d3d336df5ca99f521a210067663571d
Signed-off-by: Michael DÜrre <michael.duerre@highstreet-technologies.com>

4 files changed, 514 insertions, 0 deletions

diff --git a/tools/README.md b/tools/README.md
new file mode 100644
index 00000000..62a8520d
--- /dev/null
+++ b/tools/README.md
@@ -0,0 +1,30 @@
+# CCSDK parent tools
+
+## migrateOdl.py
+
+This script tries to generate the ccsdk odl parents out of the existing odl parents and a downloaded and extracted odl folder.
+
+Usage
+
+```
+$ python3 tools/migrateOdlParents.py --src ~/Downloads/karaf-0.15.1
+```
+
+args:
+
+ * --src  opendaylight source folder
+ * --group-id parent group-id to set (default=org.onap.ccsdk.parent)
+ * --version parent version to set
+ * --non-strict  flag to stop on fail (default=True)
+
+
+## mkbom.sh
+
+This script searches for all artifacts in the local odl repository folder $ODL_HOME/system and writes out of this a pom file. This is used to generate the installed-odl-bom/pom.xml.
+
+Usage:
+
+```
+$ cd ~/Downloads/opendaylight-15.0.0/system
+$ ./your-path-to-odl-parents/tools/mkbom.sh your-group-id  your-artifact-id your-version > /your-path-to-odl-parents/installed-odl-bom/pom.xml
+```
\ No newline at end of file
diff --git a/tools/lib/pomfile.py b/tools/lib/pomfile.py
new file mode 100644
index 00000000..35dfd5a3
--- /dev/null
+++ b/tools/lib/pomfile.py
@@ -0,0 +1,109 @@
+import re
+import tempfile
+import tempfile
+import glob
+import shutil
+from .xpath import XPath
+
+class PomFile:
+
+    def __init__(self, filename):
+        self.filename=filename
+
+    def hasParent(self) -> bool:
+        pattern_compiled = re.compile('<project[>\ ]')
+        inProject=False
+        with open(self.filename,'r') as src_file:
+                for line in src_file:
+                    m = pattern_compiled.search(line)
+                    if m is not None:
+                        if inProject == True:
+                            return True
+                        inProject=True
+                        pattern_compiled = re.compile('<parent[>\ ]')
+        return False
+                    
+
+    def setDependencyVersion(self, groupId, artifactId, version) -> bool:
+        return self.setXmlValue('/project/dependencies/dependency[groupId={},artifactId={}]/version'.format(groupId,artifactId),version)
+    def setDependencyManagementVersion(self, groupId, artifactId, version) -> bool:
+        return self.setXmlValue('/project/dependencyManagement/dependencies/dependency[groupId={},artifactId={}]/version'.format(groupId,artifactId),version)
+    # set xmlElementValue (just simple values - no objects)
+    # valuePath: xpath
+    #    e.g. /project/parent/version
+    #         /project/dependencies/dependency[groupId=org.opendaylight.netconf]/version
+    # value: value to set
+    def setXmlValue(self, valuePath, value, replaceMultiple=False) -> bool:
+        
+        found=False
+        pathToFind = XPath(valuePath)
+        pattern = re.compile('<([^>^\ ^?^!]+)')
+        curPath=XPath()
+        curParent=None
+        isComment=False
+        with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
+            with open(self.filename) as src_file:
+                for line in src_file:
+                    if found == False or replaceMultiple:
+                        x=line.find('<!--')
+                        y=line.find('-->')
+                        if x>=0:
+                            isComment=True
+                        if y>=0 and y > x:
+                            isComment=False
+                        if not isComment:
+                            matches = pattern.finditer(line,y)
+                            for matchNum, match in enumerate(matches, 1):
+                                f = match.group(1)
+                                # end tag detected
+                                if f.startswith("/"):
+                                    curPath.remove(f[1:])
+                                # start tag detected (not autoclosing xml like <br />)
+                                elif not f.endswith("/"):
+                                    x = curPath.add(f)
+                                    if curParent is None:
+                                        curParent = x
+                                    else:
+                                        curParent = curPath.last(1)
+                                else:
+                                    continue
+                                if pathToFind.equals(curPath, False):
+                                    pre=line[0:line.index('<')]
+                                    line=pre+'<{x}>{v}</{x}>\n'.format(x=f,v=value)
+                                    found=True
+                                    curPath.remove(f)
+                                    break
+                                elif pathToFind.parentParamIsNeeded(curPath.subpath(1), f):
+                                    v = self.tryToGetValue(line, f)
+                                    if v is not None:
+                                        curParent.setFilter(f, v)
+
+                    tmp_file.write(line)
+            # Overwrite the original file with the munged temporary file in a
+            # manner preserving file attributes (e.g., permissions).
+            shutil.copystat(self.filename, tmp_file.name)
+            shutil.move(tmp_file.name, self.filename)
+        print("set {} to {} in {}: {}".format(valuePath, value, self.filename, str(found)))
+        return found
+
+    def tryToGetValue(self, line, xmlTag=None):
+        pattern = re.compile('<([^>^\ ^?^!]+)>([^<]+)<\/([^>^\ ^?^!]+)>' if xmlTag is None else '<('+xmlTag+')>([^<]+)<\/('+xmlTag+')>') 
+        matches = pattern.finditer(line)
+        match = next(matches)
+        if match is not None:
+            return match.group(2)
+        return None
+
+    @staticmethod
+    def findAll(folder, excludes=[]):
+        files= glob.glob(folder + "/**/pom.xml", recursive = True)
+        r=[]
+        for file in files:
+            doExclude=False
+            for exclude in excludes:
+                if exclude in file:
+                    doExclude=True
+                    break
+            if not doExclude:
+                r.append(file)
+        return r
diff --git a/tools/lib/xpath.py b/tools/lib/xpath.py
new file mode 100644
index 00000000..07f8e389
--- /dev/null
+++ b/tools/lib/xpath.py
@@ -0,0 +1,84 @@
+
+import re
+
+
+class XPathComponent:
+    regex = r"([^\/^\[]+)(\[([^\]]+)\])?"
+    def __init__(self, expr):
+        matches = re.finditer(XPathComponent.regex, expr, re.DOTALL | re.IGNORECASE)
+        match = next(matches)
+        self.name = match.group(1)
+        tmp = match.group(3) if len(match.groups())>2 else None
+        self.filter = tmp.split(',') if tmp is not None else [] 
+
+    def equals(self, comp, ignoreFilter=False) -> bool:
+        if ignoreFilter:
+            return self.name == comp.name
+        if self.name == comp.name:
+            return set(self.filter) == set(comp.filter)
+        return False
+    
+    def setFilter(self, f, v):
+        self.filter.append('{}={}'.format(f,v))
+
+    def hasFilter(self, propertyName):
+        search=propertyName+'='
+        for filter in self.filter:
+            if filter.startswith(search):
+                return True
+        return False
+
+    def __str__(self) -> str:
+        return "XPathComponent[name={}, filter={}]".format(self.name, self.filter)
+    
+class XPath:
+
+    def __init__(self, expr=None):
+        self.raw = expr
+        tmp = expr.split('/') if expr is not None else []
+        self.components=[]
+        if len(tmp)>0 and len(tmp[0])==0:
+            tmp.pop(0)
+        for x in tmp:
+            self.components.append(XPathComponent(x))
+    
+    def add(self, c: str) -> XPathComponent:
+        xc=XPathComponent(c)
+        self.components.append(xc)
+        return xc
+
+    def remove(self, c: str) -> bool:
+        if self.components[len(self.components)-1].equals(XPathComponent(c), True):
+            self.components.pop()
+            return True
+        return False
+
+    def parentParamIsNeeded(self, xp, paramName) -> bool:
+        for i in range(len(xp.components)):
+            if i>=len(self.components):
+                return False
+            if not self.components[i].equals(xp.components[i], True):
+                return False
+        return self.components[len(xp.components)-1].hasFilter(paramName)
+
+    def equals(self, path, ignoreFilter=False) -> bool:
+        if len(self.components) != len(path.components):
+            return False
+
+        for i in range(len(self.components)):
+            if not self.components[i].equals(path.components[i], ignoreFilter):
+                return False
+        return True
+
+    def lastname(self) -> str:
+        tmp = self.last()
+        return tmp.name if tmp is not None else ""
+
+    def last(self, off=0) -> XPathComponent:
+        return self.components[len(self.components)-1-off] if len(self.components)>off else None
+
+    def subpath(self, off=0):
+        tmp =XPath()
+        for i in range(len(self.components)-off):
+            tmp.components.append(self.components[i])
+        return tmp
\ No newline at end of file
diff --git a/tools/migrateOdlParents.py b/tools/migrateOdlParents.py
new file mode 100755
index 00000000..d46085f1
--- /dev/null
+++ b/tools/migrateOdlParents.py
@@ -0,0 +1,291 @@
+#!/usr/bin/python3
+import os
+import argparse
+import subprocess
+import re
+import shutil
+import tempfile
+from lib.pomfile import PomFile
+
+DEFAULT_PARENT_GROUPID="org.onap.ccsdk.parent"
+DEFAULT_PARENT_VERSION="2.3.3-SNAPSHOT"
+DEFAULT_STRICT=True
+USE_OLD_SERVLET_API=True
+
+class OdlParentMigrator:
+
+    def __init__(self,odlSourcePath, odlParentPath=None, groupId=DEFAULT_PARENT_GROUPID, version=DEFAULT_PARENT_VERSION, strict=DEFAULT_STRICT):
+        self.odlSourcePath=odlSourcePath
+        self.mvnbin = "/usr/bin/mvn"
+        self.version = version
+        self.groupId = groupId
+        self.strict = strict
+        if odlParentPath is None:
+            odlParentPath = os.path.abspath(os.path.dirname(os.path.realpath(__file__))+"/../odlparent")
+        self.odlParentPath=odlParentPath
+        self.parentPath =os.path.abspath(odlParentPath+'/../') 
+
+
+    def getMvnRepoVersion(self, groupId, artifactId):
+        path="{}/system/{}/{}".format(self.odlSourcePath,groupId.replace('.','/'),artifactId)
+        if not os.path.exists(path):
+            return None
+        folders =[f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f))]
+        if len(folders)<1:
+            return None
+        return folders[0]
+    
+    def getMvnRepoVersions(self, groupId, artifactId):
+        path="{}/system/{}/{}".format(self.odlSourcePath,groupId.replace('.','/'),artifactId)
+        if not os.path.exists(path):
+            return None
+        folders =[f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f))]
+        if len(folders)<1:
+            return None
+        return folders
+
+    def migrateInstalledOdlBom(self) -> bool:
+        success = True
+        print("migrating installed-odl-bom")
+        root=os.path.abspath(self.odlParentPath+'/..')
+        self.exec(('cd {src}/system && '+
+            '{root}/tools/mkbom.sh {groupId} {artifactId} {version}> '+
+            '{root}/installed-odl-bom/pom.xml && cd -').format(
+            root=root,src=self.odlSourcePath,
+            parent=self.odlParentPath,groupId=self.groupId,
+            artifactId='installed-odl-bom', version=self.version))
+        if USE_OLD_SERVLET_API:
+            pom = PomFile('{}/installed-odl-bom/pom.xml'.format(root))
+            success = pom.setDependencyManagementVersion('javax.servlet','javax.servlet-api','3.1.0')
+
+        print("done")
+        return success
+
+    def migrateDependenciesBom(self) -> bool:
+        success = True
+        print("migrating dependencies-bom")
+    
+        print("done" if success else "failed")
+        return success
+
+    def migrateSetupProperties(self) -> bool:
+        success = True
+        print("migrating setup")
+        mdsalVersion=self.getMvnRepoVersion('org.opendaylight.mdsal','mdsal-binding-api')
+        odlBundleVersion=self.getMvnRepoVersion('org.opendaylight.odlparent','features-odlparent')
+        mdsalItVersion=self.getMvnRepoVersion('org.opendaylight.controller','features-controller')
+        yangVersion = self.getMvnRepoVersion('org.opendaylight.yangtools','yang-common')
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/binding-parent.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(mdsalVersion))
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/bundle-parent.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/feature-repo-parent.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/karaf4-parent.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/mdsal-it-parent.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(mdsalItVersion))
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/odlparent-lite.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/odlparent.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+        self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/single-feature-parent.properties',
+            'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+
+
+        templatePom = PomFile(self.odlParentPath+'/setup/src/main/resources/pom-template.xml')
+        x = templatePom.setXmlValue('/project/properties/odl.controller.mdsal.version',mdsalVersion)
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/odl.mdsal.version',odlBundleVersion)
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/odl.mdsal.model.version',odlBundleVersion)
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/odl.netconf.restconf.version',mdsalVersion)
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/odl.netconf.netconf.version',mdsalVersion)
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/odl.netconf.sal.rest.docgen.version',mdsalVersion)
+        success = success and x
+        
+        x = templatePom.setXmlValue('/project/properties/commons.codec.version',
+            self.getMvnRepoVersion('commons-codec','commons-codec'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/commons.lang3.version',
+            self.getMvnRepoVersion('org.apache.commons','commons-lang3'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/commons.lang.version',
+            self.getMvnRepoVersion('commons-lang','commons-lang'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/commons.net.version',
+            self.getMvnRepoVersion('commons-net','commons-net'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/eclipse.persistence.version',
+            self.getMvnRepoVersion('org.eclipse.persistence','org.eclipse.persistence.core'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/gson.version',
+            self.getMvnRepoVersion('com.google.code.gson','gson'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/guava.version',
+            self.getMvnRepoVersion('com.google.guava','guava'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/jackson.version',
+            self.getMvnRepoVersion('com.fasterxml.jackson.core','jackson-core'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/javassist.version',
+            self.getMvnRepoVersion('org.javassist','javassist'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/jersey.version',
+            self.getMvnRepoVersion('org.glassfish.jersey.core','jersey-common'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/jersey.client.version',
+            self.getMvnRepoVersion('org.glassfish.jersey.core','jersey-client'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/org.json.version',
+            self.getMvnRepoVersion('org.json','json'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/netty.version',
+            self.getMvnRepoVersion('io.netty','netty-common'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/slf4j.version',
+            self.getMvnRepoVersion('org.slf4j','slf4j-api'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/derby.version',
+            self.getMvnRepoVersion('org.apache.derby','derby'))
+        success = success and x
+        x = templatePom.setXmlValue('/project/properties/jetty.version',
+            self.getMvnRepoVersion('org.eclipse.jetty','jetty-http'))        
+        success = success and x
+        print("done" if success else "failed")
+        return success
+
+    def migrateDependenciesOdlBom(self):
+        success = True
+        print("migrating dependencies-odl-bom")
+        bgpVersion = self.getMvnRepoVersion('org.opendaylight.bgpcep','topology-api')
+        controllerVersion = self.getMvnRepoVersion('org.opendaylight.controller', 'blueprint')
+        mdsalVersion=self.getMvnRepoVersion('org.opendaylight.mdsal','mdsal-binding-api')
+        netconfVersion = self.getMvnRepoVersion('org.opendaylight.netconf','ietf-netconf')
+
+        pomFile = PomFile(os.path.abspath(self.parentPath+'/dependencies-odl-bom/pom.xml'))
+        x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=bgp-artifacts]/version',bgpVersion)
+        success = success and x
+        x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=controller-artifacts]/version',controllerVersion)
+        success = success and x
+        x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=mdsal-artifacts]/version',mdsalVersion)
+        success = success and x
+        x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=netconf-artifacts]/version',netconfVersion)
+        success = success and x
+        x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=sal-binding-broker-impl]/version',netconfVersion, True)
+        success = success and x
+        # at the moment not possible because of dependent variable in path after value to set
+        # x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=sal-binding-broker-impl,type=test-jar]/version',netconfVersion)
+        # success = success and x
+        x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=sal-test-model]/version',netconfVersion)
+        success = success and x
+        print("done" if success else "failed")
+        return success
+
+    def setParentValues(self):
+        print("setting all other parents")
+        # find all pom.xml files with parent to set
+        pomfiles=[os.path.abspath(self.parentPath+'/pom.xml'),
+            os.path.abspath(self.parentPath+'/dependencies-bom/pom.xml'),
+            os.path.abspath(self.odlParentPath+'/pom.xml'),
+            os.path.abspath(self.odlParentPath+'/setup/pom.xml'),
+            os.path.abspath(self.parentPath+'/springboot/pom.xml'),
+            os.path.abspath(self.parentPath+'/springboot/spring-boot-setup/pom.xml')]
+        
+        success=True
+        for file in pomfiles:
+            pomfile = PomFile(file)
+            if pomfile.hasParent():
+                x = pomfile.setXmlValue('/project/parent/groupId',self.groupId)
+                success = success and x
+                x = pomfile.setXmlValue('/project/parent/version',self.version)
+                success = success and x
+            
+        # find all pom.xml files with groupId and version to set
+        pomfiles=PomFile.findAll(os.path.abspath(self.odlParentPath+'/..'))
+        for file in pomfiles:
+            pomfile = PomFile(file)
+            x = pomfile.setXmlValue('/project/groupId',self.groupId)
+            success = success and x
+            x = pomfile.setXmlValue('/project/version',self.version)
+            success = success and x
+        
+        # set only groupId for odl template
+        pomfile = PomFile(self.odlParentPath+'/setup/src/main/resources/pom-template.xml')
+        x = pomfile.setXmlValue('/project/groupId',self.groupId)
+        success = success and x
+        print("done" if success else "failed")
+        return success
+        
+    def execMaven(self, command):
+        print(self.execToStdOut(self.mvnbin,command))
+
+
+        
+
+    '''
+    Perform the pure-Python equivalent of in-place `sed` substitution: e.g.,
+    `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
+    '''
+    def replaceInFile(self, filename, pattern, replacement):
+   
+        # For efficiency, precompile the passed regular expression.
+        pattern_compiled = re.compile(pattern)
+
+        # For portability, NamedTemporaryFile() defaults to mode "w+b" (i.e., binary
+        # writing with updating). This is usually a good thing. In this case,
+        # however, binary writing imposes non-trivial encoding constraints trivially
+        # resolved by switching to text writing. Let's do that.
+        with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
+            with open(filename) as src_file:
+                for line in src_file:
+                    tmp_file.write(pattern_compiled.sub(replacement, line))
+
+        # Overwrite the original file with the munged temporary file in a
+        # manner preserving file attributes (e.g., permissions).
+        shutil.copystat(filename, tmp_file.name)
+        shutil.move(tmp_file.name, filename)
+
+    def exec(self, bin, params=""):
+        output = subprocess.Popen(
+            bin+" "+params, shell=True, stdout=subprocess.PIPE).stdout.read()
+        return output
+    def execToStdOut(self, bin, params=""):
+        process = subprocess.Popen(
+            (bin+" "+params).split(' '), shell=False)
+        process.communicate()
+    
+    def run(self):
+        print("starting ONAP odl parent migration")
+        print("odl src={}".format(self.odlSourcePath))
+        print("target ={}".format(self.odlParentPath))
+        x = self.migrateInstalledOdlBom()
+        if self.strict and not x:
+            exit(1)
+        x = self.migrateDependenciesBom()
+        if self.strict and not x:
+            exit(1)
+        x = self.migrateDependenciesOdlBom()
+        if self.strict and not x:
+            exit(1)
+        x = self.migrateSetupProperties()
+        if self.strict and not x:
+            exit(1)
+        x = self.setParentValues()
+        if self.strict and not x:
+            exit(1)
+#        self.execMaven('clean install -f {}'.format(self.odlParentPath+'/setup'))
+#        self.execMaven('clean install -f {}'.format(self.parentPath))
+
+parser = argparse.ArgumentParser(description='ONAP odl parent migration tool')
+parser.add_argument('--src', type=str, required=True, help='the source folder where odl is located')
+parser.add_argument('--group-id', type=str, required=False,default=DEFAULT_PARENT_GROUPID, help='groupid for the parents')
+parser.add_argument('--version', type=str, required=False,default=DEFAULT_PARENT_VERSION, help='version')
+parser.add_argument('--non-strict', action='store_false' if DEFAULT_STRICT else 'store_true', help='determine stopping script if something cannot be set')
+args = parser.parse_args()
+
+migrator = OdlParentMigrator(args.src,None,args.group_id, args.version, args.non_strict)
+migrator.run()
\ No newline at end of file