summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authorMichael DÜrre <michael.duerre@highstreet-technologies.com>2022-01-18 10:06:31 +0100
committerDan Timoney <dtimoney@att.com>2022-03-22 13:08:35 +0000
commit7bb133e401f31623104ecd3534e2aab3ffe72009 (patch)
tree35b0115f52ec7d067cbae31c459a429de3bfb073 /tools
parentcce01d4d7b326ec8283479f3ddf397a306b32a29 (diff)
migrate parents to phosphorus-sr1
generate sr1 parents and add script for this Issue-ID: CCSDK-3565 Signed-off-by: Michael DÜrre <michael.duerre@highstreet-technologies.com> Change-Id: I53c5bed11d3d336df5ca99f521a210067663571d Signed-off-by: Michael DÜrre <michael.duerre@highstreet-technologies.com>
Diffstat (limited to 'tools')
-rw-r--r--tools/README.md30
-rw-r--r--tools/lib/pomfile.py109
-rw-r--r--tools/lib/xpath.py84
-rwxr-xr-xtools/migrateOdlParents.py291
4 files changed, 514 insertions, 0 deletions
diff --git a/tools/README.md b/tools/README.md
new file mode 100644
index 00000000..62a8520d
--- /dev/null
+++ b/tools/README.md
@@ -0,0 +1,30 @@
+# CCSDK parent tools
+
+## migrateOdl.py
+
+This script tries to generate the ccsdk odl parents out of the existing odl parents and a downloaded and extracted odl folder.
+
+Usage
+
+```
+$ python3 tools/migrateOdlParents.py --src ~/Downloads/karaf-0.15.1
+```
+
+args:
+
+ * --src opendaylight source folder
+ * --group-id parent group-id to set (default=org.onap.ccsdk.parent)
+ * --version parent version to set
+ * --non-strict flag to stop on fail (default=True)
+
+
+## mkbom.sh
+
+This script searches for all artifacts in the local odl repository folder $ODL_HOME/system and writes out of this a pom file. This is used to generate the installed-odl-bom/pom.xml.
+
+Usage:
+
+```
+$ cd ~/Downloads/opendaylight-15.0.0/system
+$ ./your-path-to-odl-parents/tools/mkbom.sh your-group-id your-artifact-id your-version > /your-path-to-odl-parents/installed-odl-bom/pom.xml
+``` \ No newline at end of file
diff --git a/tools/lib/pomfile.py b/tools/lib/pomfile.py
new file mode 100644
index 00000000..35dfd5a3
--- /dev/null
+++ b/tools/lib/pomfile.py
@@ -0,0 +1,109 @@
+import re
+import tempfile
+import tempfile
+import glob
+import shutil
+from .xpath import XPath
+
+class PomFile:
+
+ def __init__(self, filename):
+ self.filename=filename
+
+ def hasParent(self) -> bool:
+ pattern_compiled = re.compile('<project[>\ ]')
+ inProject=False
+ with open(self.filename,'r') as src_file:
+ for line in src_file:
+ m = pattern_compiled.search(line)
+ if m is not None:
+ if inProject == True:
+ return True
+ inProject=True
+ pattern_compiled = re.compile('<parent[>\ ]')
+ return False
+
+
+ def setDependencyVersion(self, groupId, artifactId, version) -> bool:
+ return self.setXmlValue('/project/dependencies/dependency[groupId={},artifactId={}]/version'.format(groupId,artifactId),version)
+ def setDependencyManagementVersion(self, groupId, artifactId, version) -> bool:
+ return self.setXmlValue('/project/dependencyManagement/dependencies/dependency[groupId={},artifactId={}]/version'.format(groupId,artifactId),version)
+ # set xmlElementValue (just simple values - no objects)
+ # valuePath: xpath
+ # e.g. /project/parent/version
+ # /project/dependencies/dependency[groupId=org.opendaylight.netconf]/version
+ # value: value to set
+ def setXmlValue(self, valuePath, value, replaceMultiple=False) -> bool:
+
+ found=False
+ pathToFind = XPath(valuePath)
+ pattern = re.compile('<([^>^\ ^?^!]+)')
+ curPath=XPath()
+ curParent=None
+ isComment=False
+ with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
+ with open(self.filename) as src_file:
+ for line in src_file:
+ if found == False or replaceMultiple:
+ x=line.find('<!--')
+ y=line.find('-->')
+ if x>=0:
+ isComment=True
+ if y>=0 and y > x:
+ isComment=False
+ if not isComment:
+ matches = pattern.finditer(line,y)
+ for matchNum, match in enumerate(matches, 1):
+ f = match.group(1)
+ # end tag detected
+ if f.startswith("/"):
+ curPath.remove(f[1:])
+ # start tag detected (not autoclosing xml like <br />)
+ elif not f.endswith("/"):
+ x = curPath.add(f)
+ if curParent is None:
+ curParent = x
+ else:
+ curParent = curPath.last(1)
+ else:
+ continue
+ if pathToFind.equals(curPath, False):
+ pre=line[0:line.index('<')]
+ line=pre+'<{x}>{v}</{x}>\n'.format(x=f,v=value)
+ found=True
+ curPath.remove(f)
+ break
+ elif pathToFind.parentParamIsNeeded(curPath.subpath(1), f):
+ v = self.tryToGetValue(line, f)
+ if v is not None:
+ curParent.setFilter(f, v)
+
+ tmp_file.write(line)
+ # Overwrite the original file with the munged temporary file in a
+ # manner preserving file attributes (e.g., permissions).
+ shutil.copystat(self.filename, tmp_file.name)
+ shutil.move(tmp_file.name, self.filename)
+ print("set {} to {} in {}: {}".format(valuePath, value, self.filename, str(found)))
+ return found
+
+ def tryToGetValue(self, line, xmlTag=None):
+ pattern = re.compile('<([^>^\ ^?^!]+)>([^<]+)<\/([^>^\ ^?^!]+)>' if xmlTag is None else '<('+xmlTag+')>([^<]+)<\/('+xmlTag+')>')
+ matches = pattern.finditer(line)
+ match = next(matches)
+ if match is not None:
+ return match.group(2)
+ return None
+
+ @staticmethod
+ def findAll(folder, excludes=[]):
+ files= glob.glob(folder + "/**/pom.xml", recursive = True)
+ r=[]
+ for file in files:
+ doExclude=False
+ for exclude in excludes:
+ if exclude in file:
+ doExclude=True
+ break
+ if not doExclude:
+ r.append(file)
+ return r
diff --git a/tools/lib/xpath.py b/tools/lib/xpath.py
new file mode 100644
index 00000000..07f8e389
--- /dev/null
+++ b/tools/lib/xpath.py
@@ -0,0 +1,84 @@
+
+import re
+
+
+class XPathComponent:
+ regex = r"([^\/^\[]+)(\[([^\]]+)\])?"
+ def __init__(self, expr):
+ matches = re.finditer(XPathComponent.regex, expr, re.DOTALL | re.IGNORECASE)
+ match = next(matches)
+ self.name = match.group(1)
+ tmp = match.group(3) if len(match.groups())>2 else None
+ self.filter = tmp.split(',') if tmp is not None else []
+
+ def equals(self, comp, ignoreFilter=False) -> bool:
+ if ignoreFilter:
+ return self.name == comp.name
+ if self.name == comp.name:
+ return set(self.filter) == set(comp.filter)
+ return False
+
+ def setFilter(self, f, v):
+ self.filter.append('{}={}'.format(f,v))
+
+ def hasFilter(self, propertyName):
+ search=propertyName+'='
+ for filter in self.filter:
+ if filter.startswith(search):
+ return True
+ return False
+
+ def __str__(self) -> str:
+ return "XPathComponent[name={}, filter={}]".format(self.name, self.filter)
+
+class XPath:
+
+ def __init__(self, expr=None):
+ self.raw = expr
+ tmp = expr.split('/') if expr is not None else []
+ self.components=[]
+ if len(tmp)>0 and len(tmp[0])==0:
+ tmp.pop(0)
+ for x in tmp:
+ self.components.append(XPathComponent(x))
+
+ def add(self, c: str) -> XPathComponent:
+ xc=XPathComponent(c)
+ self.components.append(xc)
+ return xc
+
+ def remove(self, c: str) -> bool:
+ if self.components[len(self.components)-1].equals(XPathComponent(c), True):
+ self.components.pop()
+ return True
+ return False
+
+ def parentParamIsNeeded(self, xp, paramName) -> bool:
+ for i in range(len(xp.components)):
+ if i>=len(self.components):
+ return False
+ if not self.components[i].equals(xp.components[i], True):
+ return False
+ return self.components[len(xp.components)-1].hasFilter(paramName)
+
+ def equals(self, path, ignoreFilter=False) -> bool:
+ if len(self.components) != len(path.components):
+ return False
+
+ for i in range(len(self.components)):
+ if not self.components[i].equals(path.components[i], ignoreFilter):
+ return False
+ return True
+
+ def lastname(self) -> str:
+ tmp = self.last()
+ return tmp.name if tmp is not None else ""
+
+ def last(self, off=0) -> XPathComponent:
+ return self.components[len(self.components)-1-off] if len(self.components)>off else None
+
+ def subpath(self, off=0):
+ tmp =XPath()
+ for i in range(len(self.components)-off):
+ tmp.components.append(self.components[i])
+ return tmp \ No newline at end of file
diff --git a/tools/migrateOdlParents.py b/tools/migrateOdlParents.py
new file mode 100755
index 00000000..d46085f1
--- /dev/null
+++ b/tools/migrateOdlParents.py
@@ -0,0 +1,291 @@
+#!/usr/bin/python3
+import os
+import argparse
+import subprocess
+import re
+import shutil
+import tempfile
+from lib.pomfile import PomFile
+
+DEFAULT_PARENT_GROUPID="org.onap.ccsdk.parent"
+DEFAULT_PARENT_VERSION="2.3.3-SNAPSHOT"
+DEFAULT_STRICT=True
+USE_OLD_SERVLET_API=True
+
+class OdlParentMigrator:
+
+ def __init__(self,odlSourcePath, odlParentPath=None, groupId=DEFAULT_PARENT_GROUPID, version=DEFAULT_PARENT_VERSION, strict=DEFAULT_STRICT):
+ self.odlSourcePath=odlSourcePath
+ self.mvnbin = "/usr/bin/mvn"
+ self.version = version
+ self.groupId = groupId
+ self.strict = strict
+ if odlParentPath is None:
+ odlParentPath = os.path.abspath(os.path.dirname(os.path.realpath(__file__))+"/../odlparent")
+ self.odlParentPath=odlParentPath
+ self.parentPath =os.path.abspath(odlParentPath+'/../')
+
+
+ def getMvnRepoVersion(self, groupId, artifactId):
+ path="{}/system/{}/{}".format(self.odlSourcePath,groupId.replace('.','/'),artifactId)
+ if not os.path.exists(path):
+ return None
+ folders =[f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f))]
+ if len(folders)<1:
+ return None
+ return folders[0]
+
+ def getMvnRepoVersions(self, groupId, artifactId):
+ path="{}/system/{}/{}".format(self.odlSourcePath,groupId.replace('.','/'),artifactId)
+ if not os.path.exists(path):
+ return None
+ folders =[f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f))]
+ if len(folders)<1:
+ return None
+ return folders
+
+ def migrateInstalledOdlBom(self) -> bool:
+ success = True
+ print("migrating installed-odl-bom")
+ root=os.path.abspath(self.odlParentPath+'/..')
+ self.exec(('cd {src}/system && '+
+ '{root}/tools/mkbom.sh {groupId} {artifactId} {version}> '+
+ '{root}/installed-odl-bom/pom.xml && cd -').format(
+ root=root,src=self.odlSourcePath,
+ parent=self.odlParentPath,groupId=self.groupId,
+ artifactId='installed-odl-bom', version=self.version))
+ if USE_OLD_SERVLET_API:
+ pom = PomFile('{}/installed-odl-bom/pom.xml'.format(root))
+ success = pom.setDependencyManagementVersion('javax.servlet','javax.servlet-api','3.1.0')
+
+ print("done")
+ return success
+
+ def migrateDependenciesBom(self) -> bool:
+ success = True
+ print("migrating dependencies-bom")
+
+ print("done" if success else "failed")
+ return success
+
+ def migrateSetupProperties(self) -> bool:
+ success = True
+ print("migrating setup")
+ mdsalVersion=self.getMvnRepoVersion('org.opendaylight.mdsal','mdsal-binding-api')
+ odlBundleVersion=self.getMvnRepoVersion('org.opendaylight.odlparent','features-odlparent')
+ mdsalItVersion=self.getMvnRepoVersion('org.opendaylight.controller','features-controller')
+ yangVersion = self.getMvnRepoVersion('org.opendaylight.yangtools','yang-common')
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/binding-parent.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(mdsalVersion))
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/bundle-parent.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/feature-repo-parent.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/karaf4-parent.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/mdsal-it-parent.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(mdsalItVersion))
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/odlparent-lite.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/odlparent.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+ self.replaceInFile(self.odlParentPath+'/setup/src/main/properties/single-feature-parent.properties',
+ 'odlparent.version=.*','odlparent.version={}'.format(odlBundleVersion))
+
+
+ templatePom = PomFile(self.odlParentPath+'/setup/src/main/resources/pom-template.xml')
+ x = templatePom.setXmlValue('/project/properties/odl.controller.mdsal.version',mdsalVersion)
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/odl.mdsal.version',odlBundleVersion)
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/odl.mdsal.model.version',odlBundleVersion)
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/odl.netconf.restconf.version',mdsalVersion)
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/odl.netconf.netconf.version',mdsalVersion)
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/odl.netconf.sal.rest.docgen.version',mdsalVersion)
+ success = success and x
+
+ x = templatePom.setXmlValue('/project/properties/commons.codec.version',
+ self.getMvnRepoVersion('commons-codec','commons-codec'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/commons.lang3.version',
+ self.getMvnRepoVersion('org.apache.commons','commons-lang3'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/commons.lang.version',
+ self.getMvnRepoVersion('commons-lang','commons-lang'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/commons.net.version',
+ self.getMvnRepoVersion('commons-net','commons-net'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/eclipse.persistence.version',
+ self.getMvnRepoVersion('org.eclipse.persistence','org.eclipse.persistence.core'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/gson.version',
+ self.getMvnRepoVersion('com.google.code.gson','gson'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/guava.version',
+ self.getMvnRepoVersion('com.google.guava','guava'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/jackson.version',
+ self.getMvnRepoVersion('com.fasterxml.jackson.core','jackson-core'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/javassist.version',
+ self.getMvnRepoVersion('org.javassist','javassist'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/jersey.version',
+ self.getMvnRepoVersion('org.glassfish.jersey.core','jersey-common'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/jersey.client.version',
+ self.getMvnRepoVersion('org.glassfish.jersey.core','jersey-client'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/org.json.version',
+ self.getMvnRepoVersion('org.json','json'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/netty.version',
+ self.getMvnRepoVersion('io.netty','netty-common'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/slf4j.version',
+ self.getMvnRepoVersion('org.slf4j','slf4j-api'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/derby.version',
+ self.getMvnRepoVersion('org.apache.derby','derby'))
+ success = success and x
+ x = templatePom.setXmlValue('/project/properties/jetty.version',
+ self.getMvnRepoVersion('org.eclipse.jetty','jetty-http'))
+ success = success and x
+ print("done" if success else "failed")
+ return success
+
+ def migrateDependenciesOdlBom(self):
+ success = True
+ print("migrating dependencies-odl-bom")
+ bgpVersion = self.getMvnRepoVersion('org.opendaylight.bgpcep','topology-api')
+ controllerVersion = self.getMvnRepoVersion('org.opendaylight.controller', 'blueprint')
+ mdsalVersion=self.getMvnRepoVersion('org.opendaylight.mdsal','mdsal-binding-api')
+ netconfVersion = self.getMvnRepoVersion('org.opendaylight.netconf','ietf-netconf')
+
+ pomFile = PomFile(os.path.abspath(self.parentPath+'/dependencies-odl-bom/pom.xml'))
+ x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=bgp-artifacts]/version',bgpVersion)
+ success = success and x
+ x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=controller-artifacts]/version',controllerVersion)
+ success = success and x
+ x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=mdsal-artifacts]/version',mdsalVersion)
+ success = success and x
+ x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=netconf-artifacts]/version',netconfVersion)
+ success = success and x
+ x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=sal-binding-broker-impl]/version',netconfVersion, True)
+ success = success and x
+ # at the moment not possible because of dependent variable in path after value to set
+ # x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=sal-binding-broker-impl,type=test-jar]/version',netconfVersion)
+ # success = success and x
+ x = pomFile.setXmlValue('/project/dependencyManagement/dependencies/dependency[artifactId=sal-test-model]/version',netconfVersion)
+ success = success and x
+ print("done" if success else "failed")
+ return success
+
+ def setParentValues(self):
+ print("setting all other parents")
+ # find all pom.xml files with parent to set
+ pomfiles=[os.path.abspath(self.parentPath+'/pom.xml'),
+ os.path.abspath(self.parentPath+'/dependencies-bom/pom.xml'),
+ os.path.abspath(self.odlParentPath+'/pom.xml'),
+ os.path.abspath(self.odlParentPath+'/setup/pom.xml'),
+ os.path.abspath(self.parentPath+'/springboot/pom.xml'),
+ os.path.abspath(self.parentPath+'/springboot/spring-boot-setup/pom.xml')]
+
+ success=True
+ for file in pomfiles:
+ pomfile = PomFile(file)
+ if pomfile.hasParent():
+ x = pomfile.setXmlValue('/project/parent/groupId',self.groupId)
+ success = success and x
+ x = pomfile.setXmlValue('/project/parent/version',self.version)
+ success = success and x
+
+ # find all pom.xml files with groupId and version to set
+ pomfiles=PomFile.findAll(os.path.abspath(self.odlParentPath+'/..'))
+ for file in pomfiles:
+ pomfile = PomFile(file)
+ x = pomfile.setXmlValue('/project/groupId',self.groupId)
+ success = success and x
+ x = pomfile.setXmlValue('/project/version',self.version)
+ success = success and x
+
+ # set only groupId for odl template
+ pomfile = PomFile(self.odlParentPath+'/setup/src/main/resources/pom-template.xml')
+ x = pomfile.setXmlValue('/project/groupId',self.groupId)
+ success = success and x
+ print("done" if success else "failed")
+ return success
+
+ def execMaven(self, command):
+ print(self.execToStdOut(self.mvnbin,command))
+
+
+
+
+ '''
+ Perform the pure-Python equivalent of in-place `sed` substitution: e.g.,
+ `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
+ '''
+ def replaceInFile(self, filename, pattern, replacement):
+
+ # For efficiency, precompile the passed regular expression.
+ pattern_compiled = re.compile(pattern)
+
+ # For portability, NamedTemporaryFile() defaults to mode "w+b" (i.e., binary
+ # writing with updating). This is usually a good thing. In this case,
+ # however, binary writing imposes non-trivial encoding constraints trivially
+ # resolved by switching to text writing. Let's do that.
+ with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
+ with open(filename) as src_file:
+ for line in src_file:
+ tmp_file.write(pattern_compiled.sub(replacement, line))
+
+ # Overwrite the original file with the munged temporary file in a
+ # manner preserving file attributes (e.g., permissions).
+ shutil.copystat(filename, tmp_file.name)
+ shutil.move(tmp_file.name, filename)
+
+ def exec(self, bin, params=""):
+ output = subprocess.Popen(
+ bin+" "+params, shell=True, stdout=subprocess.PIPE).stdout.read()
+ return output
+ def execToStdOut(self, bin, params=""):
+ process = subprocess.Popen(
+ (bin+" "+params).split(' '), shell=False)
+ process.communicate()
+
+ def run(self):
+ print("starting ONAP odl parent migration")
+ print("odl src={}".format(self.odlSourcePath))
+ print("target ={}".format(self.odlParentPath))
+ x = self.migrateInstalledOdlBom()
+ if self.strict and not x:
+ exit(1)
+ x = self.migrateDependenciesBom()
+ if self.strict and not x:
+ exit(1)
+ x = self.migrateDependenciesOdlBom()
+ if self.strict and not x:
+ exit(1)
+ x = self.migrateSetupProperties()
+ if self.strict and not x:
+ exit(1)
+ x = self.setParentValues()
+ if self.strict and not x:
+ exit(1)
+# self.execMaven('clean install -f {}'.format(self.odlParentPath+'/setup'))
+# self.execMaven('clean install -f {}'.format(self.parentPath))
+
+parser = argparse.ArgumentParser(description='ONAP odl parent migration tool')
+parser.add_argument('--src', type=str, required=True, help='the source folder where odl is located')
+parser.add_argument('--group-id', type=str, required=False,default=DEFAULT_PARENT_GROUPID, help='groupid for the parents')
+parser.add_argument('--version', type=str, required=False,default=DEFAULT_PARENT_VERSION, help='version')
+parser.add_argument('--non-strict', action='store_false' if DEFAULT_STRICT else 'store_true', help='determine stopping script if something cannot be set')
+args = parser.parse_args()
+
+migrator = OdlParentMigrator(args.src,None,args.group_id, args.version, args.non_strict)
+migrator.run() \ No newline at end of file