diff options
author | Gautam Shah <gautams@amdocs.com> | 2018-05-27 13:32:07 +0530 |
---|---|---|
committer | GAUTAMS <gautams@amdocs.com> | 2018-05-28 23:34:45 +0530 |
commit | b145c0825c0bd163a0a2643aeee4c8b283e35ada (patch) | |
tree | e322d65c1fba0abfeb4e4a615c0868ddc23e6f72 /openecomp-be/tools/compile-helper-plugin/src/main/java | |
parent | bd6665a88439c0c6459c32fdbe1f979b33a9068c (diff) |
Onboarding build optimization incl Qual Control
Sonar fixes, incremental build enhancements and Quality control mechanism integration.
Change-Id: I118d7fc0cc50c1eddb94137310c00afaaa3aaffb
Issue-ID: SDC-1189
Signed-off-by: GAUTAMS <gautams@amdocs.com>
Diffstat (limited to 'openecomp-be/tools/compile-helper-plugin/src/main/java')
9 files changed, 796 insertions, 337 deletions
diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/BuildHelper.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/BuildHelper.java new file mode 100644 index 0000000000..42f3166ad6 --- /dev/null +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/BuildHelper.java @@ -0,0 +1,182 @@ +/* + * Copyright © 2018 European Support Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on a "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openecomp.sdc.onboarding; + +import static org.openecomp.sdc.onboarding.Constants.JAVA_EXT; +import static org.openecomp.sdc.onboarding.Constants.UNICORN; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.RecursiveTask; +import java.util.stream.Collectors; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.project.MavenProject; + +class BuildHelper { + + private BuildHelper() { + // donot remove. + } + + static long getChecksum(File file, String fileType) { + try { + return readSources(file, fileType).hashCode(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + static String getSourceChecksum(String data, String hashType) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance(hashType); + md.update(data.getBytes()); + byte[] hashBytes = md.digest(); + + StringBuffer buffer = new StringBuffer(); + for (byte hashByte : hashBytes) { + buffer.append(Integer.toString((hashByte & 0xff) + 0x100, 16).substring(1)); + } + return buffer.toString(); + } + + + private static Map<String, String> readSources(File file, String fileType) throws IOException { + Map<String, String> source = new HashMap<>(); + if (file.exists()) { + List<File> list = Files.walk(Paths.get(file.getAbsolutePath())) + .filter(JAVA_EXT.equals(fileType) ? BuildHelper::isRegularJavaFile : + Files::isRegularFile).map(p -> p.toFile()) + .collect(Collectors.toList()); + source.putAll(ForkJoinPool.commonPool() + .invoke(new FileReadTask(list.toArray(new File[0]), file.getAbsolutePath()))); + } + return source; + } + + private static boolean isRegularJavaFile(Path path) { + File file = path.toFile(); + return file.isFile() && file.getName().endsWith(JAVA_EXT); + } + + private static String getData(File file, byte[] buffer) { + try (FileInputStream fis = new FileInputStream(file); + BufferedInputStream bis = new BufferedInputStream(fis, 64 * 1024)) { + bis.read(buffer, 0, ((int) file.length())); + if (file.getAbsolutePath().contains(File.separator + "generated-sources" + File.separator)) { + StringBuffer sb = new StringBuffer(); + List<String> coll = Files.readAllLines(file.toPath()); + for (String s : coll) { + if (s != null && !s.trim().startsWith("/") && !s.trim().startsWith("*")) { + sb.append(s); + } + } + return sb.toString(); + } + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + return new String(buffer, 0, ((int) file.length())); + } + + + private static class FileReadTask extends RecursiveTask<Map<String, String>> { + + Map<String, String> store = new HashMap<>(); + private byte[] buffer = new byte[1024 * 1024]; + File[] files; + String pathPrefix; + private final int MAX_FILES = 10; + + FileReadTask(File[] files, String pathPrefix) { + this.files = files; + this.pathPrefix = pathPrefix; + } + + @Override + protected Map<String, String> compute() { + if (files.length > MAX_FILES) { + FileReadTask task1 = new FileReadTask(Arrays.copyOfRange(files, 0, files.length / 2), pathPrefix); + FileReadTask task2 = + new FileReadTask(Arrays.copyOfRange(files, files.length / 2, files.length), pathPrefix); + task1.fork(); + task2.fork(); + store.putAll(task1.join()); + store.putAll(task2.join()); + } else { + for (File toRead : files) { + store.put(toRead.getAbsolutePath().substring(pathPrefix.length()), getData(toRead, buffer)); + } + } + + return store; + } + } + + static Optional<String> getArtifactPathInLocalRepo(String repoPath, MavenProject project, byte[] sourceChecksum) + throws MojoFailureException { + + URI uri = null; + try { + uri = new URI(repoPath + (project.getGroupId().replace('.', '/')) + '/' + project.getArtifactId() + '/' + + project.getVersion()); + } catch (URISyntaxException e) { + throw new MojoFailureException(e.getMessage()); + } + File f = new File(uri); + File[] list = f.listFiles(t -> t.getName().equals(project.getArtifactId() + "-" + project.getVersion() + "." + + project.getPackaging())); + if (list != null && list.length > 0) { + File checksumFile = new File(list[0].getParentFile(), project.getBuild().getFinalName() + "." + UNICORN); + try { + if (checksumFile.exists() && Arrays.equals(sourceChecksum, Files.readAllBytes(checksumFile.toPath()))) { + return Optional.of(list[0].getAbsolutePath()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return Optional.empty(); + } + + static <T> Optional<T> readState(String fileName, Class<T> clazz) { + try (InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName); + ObjectInputStream ois = new ObjectInputStream(is)) { + return Optional.of(clazz.cast(ois.readObject())); + } catch (Exception ignored) { + //ignore. it is taken care. + return Optional.empty(); + } + } + +} diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/BuildState.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/BuildState.java index 888622f959..17ff7c9500 100644 --- a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/BuildState.java +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/BuildState.java @@ -16,7 +16,16 @@ package org.openecomp.sdc.onboarding; +import static org.openecomp.sdc.onboarding.BuildHelper.readState; +import static org.openecomp.sdc.onboarding.Constants.ANSI_COLOR_RESET; +import static org.openecomp.sdc.onboarding.Constants.ANSI_YELLOW; +import static org.openecomp.sdc.onboarding.Constants.FULL_BUILD_DATA; +import static org.openecomp.sdc.onboarding.Constants.FULL_RESOURCE_BUILD_DATA; +import static org.openecomp.sdc.onboarding.Constants.JAR; +import static org.openecomp.sdc.onboarding.Constants.MODULE_BUILD_DATA; import static org.openecomp.sdc.onboarding.Constants.RESOURCES_CHANGED; +import static org.openecomp.sdc.onboarding.Constants.RESOURCE_BUILD_DATA; +import static org.openecomp.sdc.onboarding.Constants.SKIP_MAIN_SOURCE_COMPILE; import java.io.File; import java.io.FileInputStream; @@ -24,64 +33,65 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; +import java.util.HashSet; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Stream; +import org.apache.maven.artifact.Artifact; import org.apache.maven.project.MavenProject; public class BuildState { - private static Map<String, Long> fullBuildData = new HashMap<>(); - private static Map<String, Long> fullResourceBuildData = new HashMap<>(); + private static Map<String, Map> compileDataStore = new HashMap<>(); private static Map<String, Object> moduleBuildData = new HashMap<>(); private static Map<String, Object> resourceBuildData = new HashMap<>(); + private static Map<String, Artifact> artifacts = new HashMap<>(); + private static Set<String> executeTestsIfDependsOnStore = new HashSet<>(); + private static Set<String> pmdExecutedInRun = new HashSet<>(); - private static File buildStateFile; - private static File resourceStateFile; - private File moduleBuildDataFile; - private File resourceBuildDataFile; + private static File compileStateFile; private MavenProject project; - private String buildStateFilePath; - private String resourceStateFilePath; + private String compileStateFilePath; - private void readFullBuildState() { - buildStateFile = initialize(this::getBuildStateFile, fullBuildData, - buildStateFilePath.substring(0, buildStateFilePath.indexOf('/')), project); + static { + initializeStore(); + Optional<HashMap> masterStore = readState("compile.dat", HashMap.class); + compileDataStore = masterStore.isPresent() ? masterStore.get() : compileDataStore; } - private void readResourceBuildState() { - resourceStateFile = initialize(this::getResourceStateFile, fullResourceBuildData, - resourceStateFilePath.substring(0, resourceStateFilePath.indexOf('/')), project); + void init() { + artifacts.clear(); + for (Artifact artifact : project.getArtifacts()) { + if (artifact.isSnapshot() && JAR.equals(artifact.getType())) { + artifacts.put(artifact.getGroupId() + ":" + artifact.getArtifactId(), artifact); + } + } + compileStateFile = + getCompileStateFile(compileStateFilePath.substring(0, compileStateFilePath.indexOf('/')), project); + } + static void initializeStore() { + compileDataStore.put(FULL_BUILD_DATA, new HashMap<>()); + compileDataStore.put(FULL_RESOURCE_BUILD_DATA, new HashMap<>()); + compileDataStore.put(MODULE_BUILD_DATA, new HashMap<>()); + compileDataStore.put(RESOURCE_BUILD_DATA, new HashMap<>()); } - private File initialize(BiFunction<String, MavenProject, File> funct, Map store, String moduleCoordinate, - MavenProject proj) { - File file = funct.apply(moduleCoordinate, proj); - file.getParentFile().mkdirs(); - try (FileInputStream fis = new FileInputStream(file); ObjectInputStream ois = new ObjectInputStream(fis);) { - if (store.isEmpty()) { - store.putAll(HashMap.class.cast(ois.readObject())); - } - } catch (Exception e) { - store.clear(); - } - return file; + + static void recordPMDRun(String moduleCoordinates) { + pmdExecutedInRun.add(moduleCoordinates); } - private void writeFullBuildState() throws IOException { - writeState(buildStateFile, fullBuildData); + static boolean isPMDRun(String moduleCoordintes) { + return pmdExecutedInRun.contains(moduleCoordintes); } - private void writeFullResourceBuildState() throws IOException { - writeState(resourceStateFile, fullResourceBuildData); + private void writeCompileState() throws IOException { + writeState(compileStateFile, compileDataStore); } private void writeState(File file, Map store) throws IOException { @@ -90,20 +100,18 @@ public class BuildState { } } - private File getBuildStateFile(String moduleCoordinate, MavenProject proj) { - return getStateFile(moduleCoordinate, proj, buildStateFilePath); - } - private File getResourceStateFile(String moduleCoordinate, MavenProject proj) { - return getStateFile(moduleCoordinate, proj, resourceStateFilePath); + private File getCompileStateFile(String moduleCoordinate, MavenProject proj) { + return getStateFile(moduleCoordinate, proj, compileStateFilePath); } + private File getStateFile(String moduleCoordinate, MavenProject proj, String filePath) { return new File(getTopParentProject(moduleCoordinate, proj).getBasedir(), filePath.substring(filePath.indexOf('/') + 1)); } - private MavenProject getTopParentProject(String moduleCoordinate, MavenProject proj) { + MavenProject getTopParentProject(String moduleCoordinate, MavenProject proj) { if (getModuleCoordinate(proj).equals(moduleCoordinate) || proj.getParent() == null) { return proj; } else { @@ -116,10 +124,12 @@ public class BuildState { } void addModuleBuildTime(String moduleCoordinates, Long buildTime) { - Long lastTime = fullBuildData.put(moduleCoordinates, buildTime); + Long lastTime = Long.class.cast(compileDataStore.get(FULL_BUILD_DATA).put(moduleCoordinates, buildTime)); try { if (lastTime == null || !lastTime.equals(buildTime)) { - writeFullBuildState(); + if (!project.getProperties().containsKey(SKIP_MAIN_SOURCE_COMPILE)) { + writeCompileState(); + } } } catch (IOException ignored) { // ignored. No need to handle. System will take care. @@ -127,10 +137,11 @@ public class BuildState { } void addResourceBuildTime(String moduleCoordinates, Long buildTime) { - if (project.getProperties().containsKey(RESOURCES_CHANGED)) { - Long lastTime = fullResourceBuildData.put(moduleCoordinates, buildTime); + if (project.getProperties().containsKey(RESOURCES_CHANGED) + || compileDataStore.get(FULL_RESOURCE_BUILD_DATA).get(moduleCoordinates) == null) { try { - writeFullResourceBuildState(); + compileDataStore.get(FULL_RESOURCE_BUILD_DATA).put(moduleCoordinates, buildTime); + writeCompileState(); } catch (IOException ignored) { // ignored. No need to handle. System will take care. } @@ -142,15 +153,29 @@ public class BuildState { } Map<String, Object> readModuleBuildData() { - return readBuildData(moduleBuildDataFile); + return HashMap.class.cast(compileDataStore.get(MODULE_BUILD_DATA).get(getModuleCoordinate(project))); } void saveModuleBuildData(String moduleCoordinate) { - saveBuildData(moduleBuildDataFile, moduleBuildData.get(moduleCoordinate)); + if (moduleBuildData.get(moduleCoordinate) != null) { + compileDataStore.get(MODULE_BUILD_DATA).put(moduleCoordinate, moduleBuildData.get(moduleCoordinate)); + } + saveCompileData(); } void saveResourceBuildData(String moduleCoordinate) { - saveBuildData(resourceBuildDataFile, resourceBuildData.get(moduleCoordinate)); + if (resourceBuildData.get(moduleCoordinate) != null) { + compileDataStore.get(RESOURCE_BUILD_DATA).put(moduleCoordinate, resourceBuildData.get(moduleCoordinate)); + } + saveCompileData(); + } + + void saveCompileData() { + saveBuildData(compileStateFile, compileDataStore); + } + + void markTestsMandatoryModule(String moduleCoordinates) { + executeTestsIfDependsOnStore.add(moduleCoordinates); } private void saveBuildData(File file, Object dataToSave) { @@ -166,41 +191,50 @@ public class BuildState { } Map<String, Object> readResourceBuildData() { - return readBuildData(resourceBuildDataFile); + return HashMap.class.cast(compileDataStore.get(RESOURCE_BUILD_DATA).get(getModuleCoordinate(project))); } - private Map<String, Object> readBuildData(File file) { - try (FileInputStream fis = new FileInputStream(file); ObjectInputStream ois = new ObjectInputStream(fis)) { - return HashMap.class.cast(ois.readObject()); - } catch (Exception e) { - return new HashMap<>(); - } - } void addResourceBuildData(String moduleCoordinates, Map currentModuleResourceBuildData) { resourceBuildData.put(moduleCoordinates, currentModuleResourceBuildData); } Long getBuildTime(String moduleCoordinates) { - if (fullBuildData.isEmpty()) { - readFullBuildState(); - readResourceBuildState(); - } - Long buildTime = fullBuildData.get(moduleCoordinates); + Long buildTime = Long.class.cast(compileDataStore.get(FULL_BUILD_DATA).get(moduleCoordinates)); return buildTime == null ? 0 : buildTime; } Long getResourceBuildTime(String moduleCoordinates) { - Long resourceBuildTime = fullResourceBuildData.get(moduleCoordinates); + Long resourceBuildTime = Long.class.cast(compileDataStore.get(FULL_RESOURCE_BUILD_DATA).get(moduleCoordinates)); return resourceBuildTime == null ? 0 : resourceBuildTime; } boolean isCompileMust(String moduleCoordinates, Collection<String> dependencies) { + for (String d : dependencies) { + if (artifacts.containsKey(d) && JAR.equals(artifacts.get(d).getType())) { + if (artifacts.get(d).getVersion().equals(project.getVersion()) && getBuildTime(d) == 0) { + System.out.println(ANSI_YELLOW + "[WARNING:]" + "You have module[" + d + + "] not locally compiled even once, please compile your project once daily from root to have reliable build results." + + ANSI_COLOR_RESET); + return true; + } + } + } return isMust(this::getBuildTime, moduleCoordinates, dependencies); } - boolean isTestMust(String moduleCoordinates, Collection<String> dependencies) { - return isMust(this::getResourceBuildTime, moduleCoordinates, dependencies); + boolean isTestExecutionMandatory() { + for (String d : artifacts.keySet()) { + if (executeTestsIfDependsOnStore.contains(d)) { + return true; + } + } + return false; + } + + boolean isTestMust(String moduleCoordinates) { + return getBuildTime(moduleCoordinates) > getResourceBuildTime(moduleCoordinates) || isMust( + this::getResourceBuildTime, moduleCoordinates, artifacts.keySet()); } private boolean isMust(Function<String, Long> funct, String moduleCoordinates, Collection<String> dependencies) { @@ -217,22 +251,4 @@ public class BuildState { return false; } - void markModuleDirty(File file) throws IOException { - if (file.exists()) { - Stream<String> lines = Files.lines(file.toPath()); - Iterator<String> itr = lines.iterator(); - while (itr.hasNext()) { - String line = itr.next(); - Path path = Paths.get(line); - if (path.toFile().exists()) { - if (path.toFile().setLastModified(System.currentTimeMillis())) { - break; - } else { - continue; - } - } - } - } - } - } diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/Constants.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/Constants.java index 96abc47882..c639c6d790 100644 --- a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/Constants.java +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/Constants.java @@ -2,14 +2,44 @@ package org.openecomp.sdc.onboarding; public class Constants { + private Constants() { + } + public static final String JACOCO_SKIP = "jacoco.skip"; public static final String FORK_COUNT = "fork.count"; + public static final String FORK_MODE = "fork.mode"; + public static final String SKIP_PMD = "skipPMD"; public static final String JAVA_EXT = ".java"; - public static final String CLASS_EXT = ".class"; + public static final String ANY_EXT = "*"; public static final String SKIP_TEST_RUN = "skipTestRun"; + public static final String SKIP_TESTS = "skipTests"; public static final String MAIN = "main"; public static final String TEST = "test"; - public static final String JAVA = "java"; public static final String RESOURCES_CHANGED = "resourcesChanged"; public static final String UNICORN = "unicorn"; + public static final String ANSI_YELLOW = "\u001B[43m"; + public static final String ANSI_COLOR_RESET = "\u001B[0m"; + public static final String SKIP_MAIN_SOURCE_COMPILE = "skipMainSourceCompile"; + public static final String SKIP_TEST_SOURCE_COMPILE = "skipTestSourceCompile"; + public static final String MAIN_CHECKSUM = "mainChecksum"; + public static final String TEST_CHECKSUM = "testChecksum"; + public static final String RESOURCE_CHECKSUM = "resourceChecksum"; + public static final String MAIN_SOURCE_CHECKSUM = "mainSourceChecksum"; + public static final String TEST_SOURCE_CHECKSUM = "testSourceChecksum"; + public static final String GENERATED_SOURCE_CHECKSUM = "generatedSourceChecksum"; + public static final String EMPTY_JAR = "emptyJAR"; + public static final String JAR = "jar"; + public static final String SHA1 = "sha1"; + public static final String COLON = ":"; + public static final String DOT = "."; + public static final String FULL_BUILD_DATA = "fullBuildData"; + public static final String FULL_RESOURCE_BUILD_DATA = "fullResourceBuildData"; + public static final String MODULE_BUILD_DATA = "moduleBuildData"; + public static final String RESOURCE_BUILD_DATA = "resourceBuildData"; + public static final String RESOURCE_ONLY = "resourceOnly"; + public static final String TEST_RESOURCE_ONLY = "testResourceOnly"; + public static final String INSTRUMENT_WITH_TEST_ONLY = "instrumentWithTestOnly"; + public static final String RESOURCE_WITH_TEST_ONLY = "resourceWithTestOnly"; + public static final String INSTRUMENT_ONLY = "instrumentOnly"; + public static final String TEST_ONLY = "testOnly"; } diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/InitializationHelperMojo.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/InitializationHelperMojo.java index cbf6f69c8a..8b0ff0eba2 100644 --- a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/InitializationHelperMojo.java +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/InitializationHelperMojo.java @@ -17,7 +17,9 @@ package org.openecomp.sdc.onboarding; import static org.openecomp.sdc.onboarding.Constants.FORK_COUNT; +import static org.openecomp.sdc.onboarding.Constants.FORK_MODE; import static org.openecomp.sdc.onboarding.Constants.JACOCO_SKIP; +import static org.openecomp.sdc.onboarding.Constants.SKIP_PMD; import static org.openecomp.sdc.onboarding.Constants.UNICORN; import org.apache.maven.plugin.AbstractMojo; @@ -29,29 +31,39 @@ import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.plugins.annotations.ResolutionScope; import org.apache.maven.project.MavenProject; -@Mojo(name = "init-helper", threadSafe = true, defaultPhase = LifecyclePhase.PRE_CLEAN, - requiresDependencyResolution = ResolutionScope.NONE) +@Mojo(name = "init-helper", threadSafe = true, defaultPhase = LifecyclePhase.GENERATE_RESOURCES, + requiresDependencyResolution = ResolutionScope.TEST) public class InitializationHelperMojo extends AbstractMojo { @Parameter(defaultValue = "${project}", readonly = true) private MavenProject project; + @Parameter + private BuildState buildState; + @Parameter + private String excludePackaging; public void execute() throws MojoExecutionException, MojoFailureException { + if (project.getPackaging().equals(excludePackaging)) { + return; + } + project.getProperties().setProperty("skipGet", "false"); if (System.getProperties().containsKey(JACOCO_SKIP) && Boolean.FALSE.equals(Boolean.valueOf( System.getProperties().getProperty(JACOCO_SKIP)))) { project.getProperties().setProperty(FORK_COUNT, "1"); + project.getProperties().setProperty(FORK_MODE, "once"); } else { project.getProperties().setProperty(FORK_COUNT, "0"); + project.getProperties().setProperty(FORK_MODE, "never"); } - if (System.getProperties().containsKey(UNICORN)) { - project.getProperties().setProperty("classes", "classes/**/*.class"); - project.getProperties().setProperty("testClasses", "test-classes/**/*.class"); - project.getProperties().setProperty("mavenStatus", "maven-status/**"); - project.getProperties().setProperty("pmd", "pmd/**"); - project.getProperties().setProperty("customGeneratedSources", "generated-sources/custom/**"); + project.getProperties().setProperty(SKIP_PMD, Boolean.TRUE.toString()); + if (System.getProperties().containsKey(UNICORN)) { + buildState.init(); + } else { + project.getProperties().setProperty("skipMainSourceCompile", "false"); + project.getProperties().setProperty("skipTestSourceCompile", "false"); } } diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostCompileHelperMojo.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostCompileHelperMojo.java index 9ab373509e..04e0ca8e23 100644 --- a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostCompileHelperMojo.java +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostCompileHelperMojo.java @@ -16,26 +16,18 @@ package org.openecomp.sdc.onboarding; -import static org.openecomp.sdc.onboarding.Constants.CLASS_EXT; -import static org.openecomp.sdc.onboarding.Constants.JAVA_EXT; -import static org.openecomp.sdc.onboarding.Constants.MAIN; -import static org.openecomp.sdc.onboarding.Constants.RESOURCES_CHANGED; -import static org.openecomp.sdc.onboarding.Constants.SKIP_TEST_RUN; +import static org.openecomp.sdc.onboarding.Constants.INSTRUMENT_ONLY; +import static org.openecomp.sdc.onboarding.Constants.INSTRUMENT_WITH_TEST_ONLY; +import static org.openecomp.sdc.onboarding.Constants.RESOURCE_ONLY; +import static org.openecomp.sdc.onboarding.Constants.RESOURCE_WITH_TEST_ONLY; +import static org.openecomp.sdc.onboarding.Constants.SKIP_MAIN_SOURCE_COMPILE; +import static org.openecomp.sdc.onboarding.Constants.SKIP_PMD; +import static org.openecomp.sdc.onboarding.Constants.SKIP_TEST_SOURCE_COMPILE; +import static org.openecomp.sdc.onboarding.Constants.TEST_ONLY; +import static org.openecomp.sdc.onboarding.Constants.TEST_RESOURCE_ONLY; import static org.openecomp.sdc.onboarding.Constants.UNICORN; import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.maven.artifact.Artifact; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.LifecyclePhase; @@ -44,7 +36,7 @@ import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.plugins.annotations.ResolutionScope; import org.apache.maven.project.MavenProject; -@Mojo(name = "post-compile-helper", threadSafe = true, defaultPhase = LifecyclePhase.PROCESS_TEST_CLASSES, +@Mojo(name = "post-compile-helper", threadSafe = true, defaultPhase = LifecyclePhase.TEST_COMPILE, requiresDependencyResolution = ResolutionScope.TEST) public class PostCompileHelperMojo extends AbstractMojo { @@ -53,88 +45,14 @@ public class PostCompileHelperMojo extends AbstractMojo { @Parameter(defaultValue = "${project.artifact.groupId}:${project.artifact.artifactId}") private String moduleCoordinates; @Parameter - private Long staleThreshold; - @Parameter private String excludePackaging; @Parameter - private List<String> excludeDependencies; - @Parameter - private File mainSourceLocation; - @Parameter - private File testSourceLocation; - @Parameter - private File mainCompiledLocation; - @Parameter - private File testCompiledLocation; - @Parameter - private File inputSourceFilesList; - @Parameter - private File inputTestFilesList; - @Parameter private BuildState buildState; @Parameter private File mainResourceLocation; @Parameter private File testResourceLocation; - @Parameter - private File compiledTestFilesList; - - - private File[] getCompiledClasses(File compiledFiles) { - if (!compiledFiles.exists()) { - return new File[0]; - } - File[] list = null; - try { - list = Files.walk(Paths.get(compiledFiles.getAbsolutePath())) - .filter(p -> p.toFile().getAbsolutePath().endsWith(CLASS_EXT)).map(p -> p.toFile()) - .sorted(this::compare).collect(Collectors.toList()).toArray(new File[0]); - } catch (IOException e) { - e.printStackTrace(); - } - if (list == null || list.length == 0) { - return new File[0]; - } - return list; - } - - private int compare(File file1, File file2) { - if (file1.lastModified() > file2.lastModified()) { - return 1; - } - if (file1.lastModified() < file2.lastModified()) { - return -1; - } - return 0; - } - private File[] getStaleCompiledClasses(File[] compiledClasses, File javaSourceLocation) { - List<File> staleFiles = new ArrayList<>(); - for (File file : compiledClasses) { - String classLocation = file.getAbsolutePath().replace( - project.getBasedir().getAbsolutePath() + File.separator + "target" + File.separator, ""); - String classLocationWithPackageOnly = - classLocation.substring(classLocation.indexOf(File.separatorChar) + 1); - String sourceFilePath = javaSourceLocation.getAbsolutePath() + File.separator + classLocationWithPackageOnly - .replace(CLASS_EXT, - JAVA_EXT); - if (Paths.get(sourceFilePath).toFile().exists()) { - return staleFiles.toArray(new File[0]); - } else { - staleFiles.add(file); - } - } - return staleFiles.toArray(new File[0]); - } - - private boolean deleteAll(File[] files) { - for (File file : files) { - if (!file.delete()) { - return false; - } - } - return true; - } public void execute() throws MojoExecutionException { if (!System.getProperties().containsKey(UNICORN)) { @@ -143,93 +61,40 @@ public class PostCompileHelperMojo extends AbstractMojo { if (project.getPackaging().equals(excludePackaging)) { return; } - String moduleLocation = project.getBasedir().getAbsolutePath(); - - File[] mainClasses = getCompiledClasses(mainCompiledLocation); - processStaleClassesIfAny(mainClasses, mainSourceLocation, inputSourceFilesList); - - File[] testClasses = getCompiledClasses(testCompiledLocation); - processStaleClassesIfAny(testClasses, testSourceLocation, inputTestFilesList); - - if (mainClasses.length == 0 && testClasses.length == 0) { - return; + if (project.getProperties().containsKey(TEST_ONLY)) { + project.getProperties().setProperty(SKIP_MAIN_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().remove(TEST_ONLY); } - buildState.addModuleBuildTime(project.getGroupId() + ":" + project.getArtifactId(), - mainClasses.length > 0 ? mainClasses[mainClasses.length - 1].lastModified() : - testClasses.length > 0 ? testClasses[testClasses.length - 1].lastModified() : 0); - buildState.saveModuleBuildData(moduleCoordinates); - Map<String, Object> resourceBuildData = getCurrentResourceBuildData(); - Map<String, Object> lastTimeResourceBuildData = buildState.readResourceBuildData(); - boolean resourceDataSame = resourceBuildData.equals(lastTimeResourceBuildData); - if (!resourceDataSame) { - buildState.addResourceBuildData(moduleCoordinates, resourceBuildData); - project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); + if (project.getProperties().containsKey(INSTRUMENT_ONLY)) { + project.getProperties().setProperty(SKIP_MAIN_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().setProperty(SKIP_TEST_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().remove(INSTRUMENT_ONLY); } - boolean resourceMainBuildDataSameWithPreviousBuild = - lastTimeResourceBuildData.get(MAIN) != null && resourceBuildData.get(MAIN) - .equals(lastTimeResourceBuildData - .get(MAIN)); - if (!resourceMainBuildDataSameWithPreviousBuild) { - project.getProperties().setProperty(RESOURCES_CHANGED, Boolean.TRUE.toString()); + if (project.getProperties().containsKey(INSTRUMENT_WITH_TEST_ONLY)) { + project.getProperties().setProperty(SKIP_MAIN_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().remove(INSTRUMENT_WITH_TEST_ONLY); } - if (!project.getProperties().containsKey(SKIP_TEST_RUN)) { - if (compiledTestFilesList.exists() - && compiledTestFilesList.lastModified() > System.currentTimeMillis() - staleThreshold) { - project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); - } + if (project.getProperties().containsKey(RESOURCE_WITH_TEST_ONLY)) { + project.getProperties().setProperty(SKIP_MAIN_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().remove(RESOURCE_WITH_TEST_ONLY); } - } - - private void processStaleClassesIfAny(File[] classes, File sourceLocation, File listFile) - throws MojoExecutionException { - if (classes.length > 0) { - List<File> list = new ArrayList<>(Arrays.asList(classes)); - File[] staleClasses = null; - boolean allStale = listFile.isFile() && listFile.length() == 0; - if (allStale) { - staleClasses = classes; - listFile.delete(); - } else { - list.removeIf(f -> f.lastModified() > classes[classes.length - 1].lastModified() - staleThreshold); - staleClasses = getStaleCompiledClasses(list.toArray(new File[0]), sourceLocation); - } - if (!deleteAll(staleClasses)) { - throw new MojoExecutionException( - "****** Please remove 'target' directory manually under path " + project.getBasedir() - .getAbsolutePath()); - } + if (project.getProperties().containsKey(RESOURCE_ONLY)) { + project.getProperties().setProperty(SKIP_MAIN_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().setProperty(SKIP_TEST_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().remove(RESOURCE_ONLY); } - } - - private Map<String, Object> getCurrentResourceBuildData() { - HashMap<String, Object> resourceBuildStateData = new HashMap<>(); - try { - resourceBuildStateData.put("main", readResources(mainResourceLocation)); - resourceBuildStateData.put("test", readResources(testResourceLocation)); - resourceBuildStateData.put("dependency", getDependencies()); - } catch (IOException ioException) { - throw new UncheckedIOException(ioException); + if (project.getProperties().containsKey(TEST_RESOURCE_ONLY)) { + project.getProperties().setProperty(SKIP_MAIN_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().setProperty(SKIP_TEST_SOURCE_COMPILE, Boolean.TRUE.toString()); + project.getProperties().remove(TEST_RESOURCE_ONLY); } - return resourceBuildStateData; - } - - private Map<String, Long> readResources(File file) throws IOException { - Map<String, Long> resources = new HashMap<>(); - if (file.exists()) { - List<Path> list = Files.walk(Paths.get(file.getAbsolutePath())).filter(Files::isRegularFile) - .collect(Collectors.toList()); - for (Path path : list) { - resources.put(path.toFile().getAbsolutePath(), path.toFile().lastModified()); - } + if (!project.getProperties().containsKey(SKIP_MAIN_SOURCE_COMPILE)) { + buildState.addModuleBuildTime(moduleCoordinates, System.currentTimeMillis()); + project.getProperties().setProperty(SKIP_PMD, Boolean.FALSE.toString()); } - return resources; - } - - private Map<String, String> getDependencies() { - Map<String, String> dependencies = new HashMap<>(); - for (Artifact d : project.getArtifacts()) { - dependencies.put(d.getGroupId() + ":" + d.getArtifactId(), d.getVersion()); + if (!project.getProperties().containsKey(SKIP_TEST_SOURCE_COMPILE)) { + project.getProperties().setProperty(SKIP_PMD, Boolean.FALSE.toString()); } - return dependencies; + buildState.saveModuleBuildData(moduleCoordinates); } } diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostSourceGeneratorMojo.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostSourceGeneratorMojo.java new file mode 100644 index 0000000000..36e5f8ac32 --- /dev/null +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostSourceGeneratorMojo.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2018 European Support Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on a "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openecomp.sdc.onboarding; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.ResolutionScope; + + +@Mojo(name = "post-source-generator-helper", threadSafe = true, defaultPhase = LifecyclePhase.GENERATE_RESOURCES, + requiresDependencyResolution = ResolutionScope.TEST) +public class PostSourceGeneratorMojo extends PreCompileHelperMojo { + + public void execute() throws MojoExecutionException, MojoFailureException { + if (isCodeGenerator()) { + super.execute(); + } + } + + // @Override + void deleteAllClasses(File file) { + if (!file.exists()) { + return; + } + try { + List<File> list = + Files.walk(Paths.get(file.getAbsolutePath())).filter(Files::isRegularFile).map(p -> p.toFile()) + .collect(Collectors.toList()); + for (File f : list) { + f.delete(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + } +} diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostTestRunHelperMojo.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostTestRunHelperMojo.java index 5b326f3ddf..c2816dbfbf 100644 --- a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostTestRunHelperMojo.java +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PostTestRunHelperMojo.java @@ -16,6 +16,7 @@ package org.openecomp.sdc.onboarding; +import static org.openecomp.sdc.onboarding.Constants.SKIP_TESTS; import static org.openecomp.sdc.onboarding.Constants.SKIP_TEST_RUN; import static org.openecomp.sdc.onboarding.Constants.UNICORN; @@ -49,11 +50,13 @@ public class PostTestRunHelperMojo extends AbstractMojo { if (project.getPackaging().equals(excludePackaging)) { return; } - buildState.saveResourceBuildData(moduleCoordinates); + if (project.getProperties().containsKey(SKIP_TEST_RUN) && !Boolean.valueOf( project.getProperties().getProperty(SKIP_TEST_RUN))) { - if (!System.getProperties().containsKey("skipTests")) { + if (!System.getProperties().containsKey(SKIP_TESTS)) { + buildState.saveResourceBuildData(moduleCoordinates); buildState.addResourceBuildTime(moduleCoordinates, System.currentTimeMillis()); + } } diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreCompileHelperMojo.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreCompileHelperMojo.java index faa3167e37..854ef7fa97 100644 --- a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreCompileHelperMojo.java +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreCompileHelperMojo.java @@ -16,127 +16,417 @@ package org.openecomp.sdc.onboarding; +import static org.openecomp.sdc.onboarding.BuildHelper.getArtifactPathInLocalRepo; +import static org.openecomp.sdc.onboarding.BuildHelper.getChecksum; +import static org.openecomp.sdc.onboarding.BuildHelper.getSourceChecksum; +import static org.openecomp.sdc.onboarding.BuildHelper.readState; +import static org.openecomp.sdc.onboarding.Constants.ANY_EXT; +import static org.openecomp.sdc.onboarding.Constants.COLON; +import static org.openecomp.sdc.onboarding.Constants.DOT; +import static org.openecomp.sdc.onboarding.Constants.EMPTY_JAR; +import static org.openecomp.sdc.onboarding.Constants.GENERATED_SOURCE_CHECKSUM; +import static org.openecomp.sdc.onboarding.Constants.INSTRUMENT_ONLY; +import static org.openecomp.sdc.onboarding.Constants.INSTRUMENT_WITH_TEST_ONLY; +import static org.openecomp.sdc.onboarding.Constants.JAR; +import static org.openecomp.sdc.onboarding.Constants.JAVA_EXT; import static org.openecomp.sdc.onboarding.Constants.MAIN; +import static org.openecomp.sdc.onboarding.Constants.MAIN_CHECKSUM; +import static org.openecomp.sdc.onboarding.Constants.MAIN_SOURCE_CHECKSUM; +import static org.openecomp.sdc.onboarding.Constants.RESOURCES_CHANGED; +import static org.openecomp.sdc.onboarding.Constants.RESOURCE_CHECKSUM; +import static org.openecomp.sdc.onboarding.Constants.RESOURCE_ONLY; +import static org.openecomp.sdc.onboarding.Constants.RESOURCE_WITH_TEST_ONLY; +import static org.openecomp.sdc.onboarding.Constants.SHA1; +import static org.openecomp.sdc.onboarding.Constants.SKIP_MAIN_SOURCE_COMPILE; +import static org.openecomp.sdc.onboarding.Constants.SKIP_PMD; import static org.openecomp.sdc.onboarding.Constants.SKIP_TEST_RUN; +import static org.openecomp.sdc.onboarding.Constants.SKIP_TEST_SOURCE_COMPILE; import static org.openecomp.sdc.onboarding.Constants.TEST; +import static org.openecomp.sdc.onboarding.Constants.TEST_CHECKSUM; +import static org.openecomp.sdc.onboarding.Constants.TEST_ONLY; +import static org.openecomp.sdc.onboarding.Constants.TEST_RESOURCE_ONLY; +import static org.openecomp.sdc.onboarding.Constants.TEST_SOURCE_CHECKSUM; import static org.openecomp.sdc.onboarding.Constants.UNICORN; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.apache.maven.artifact.Artifact; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.model.Plugin; +import org.apache.maven.model.PluginExecution; import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.InvalidPluginDescriptorException; +import org.apache.maven.plugin.MavenPluginManager; import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugin.MojoNotFoundException; +import org.apache.maven.plugin.PluginDescriptorParsingException; +import org.apache.maven.plugin.PluginResolutionException; +import org.apache.maven.plugin.descriptor.MojoDescriptor; +import org.apache.maven.plugins.annotations.Component; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.plugins.annotations.ResolutionScope; import org.apache.maven.project.MavenProject; -@Mojo(name = "pre-compile-helper", threadSafe = true, defaultPhase = LifecyclePhase.GENERATE_SOURCES, + +@Mojo(name = "pre-compile-helper", threadSafe = true, defaultPhase = LifecyclePhase.GENERATE_RESOURCES, requiresDependencyResolution = ResolutionScope.TEST) public class PreCompileHelperMojo extends AbstractMojo { - @Parameter(defaultValue = "${project}", readonly = true) + @Parameter(defaultValue = "${project}") private MavenProject project; @Parameter(defaultValue = "${project.artifact.groupId}:${project.artifact.artifactId}") private String moduleCoordinates; + @Parameter(defaultValue = "${session}") + private MavenSession session; @Parameter private String excludePackaging; @Parameter private List<String> excludeDependencies; @Parameter - private File mainCompiledLocation; + private BuildState buildState; @Parameter - private File testCompiledLocation; + private File mainSourceLocation; @Parameter - private File inputSourceFilesList; + private File testSourceLocation; @Parameter - private File inputTestFilesList; + private File generatedSourceLocation; + @Component + private MavenPluginManager pluginManager; @Parameter - private BuildState buildState; + private File mainResourceLocation; + @Parameter + private File testResourceLocation; + private Map<String, Object> resourceBuildData; - public void execute() throws MojoExecutionException { - if (!System.getProperties().containsKey(UNICORN)) { + private static Map<String, String> checksumMap; + + static { + checksumMap = readCurrentPMDState("pmd.dat"); + } + + public void execute() throws MojoExecutionException, MojoFailureException { + + long mainChecksum = 0, testChecksum = 0, resourceChecksum = 0; + Optional<String> artifactPath; + + if (project.getPackaging().equals(excludePackaging)) { return; } - if (project.getPackaging().equals(excludePackaging)) { + init(); + project.getProperties() + .setProperty(MAIN_CHECKSUM, String.valueOf(mainChecksum = getChecksum(mainSourceLocation, JAVA_EXT))); + project.getProperties() + .setProperty(TEST_CHECKSUM, String.valueOf(testChecksum = getChecksum(testSourceLocation, JAVA_EXT))); + String checksum = mainChecksum + COLON + testChecksum; + if (!checksum.equals(checksumMap.get(moduleCoordinates)) || isPMDMandatory(project.getArtifacts())) { + project.getProperties().setProperty(SKIP_PMD, Boolean.FALSE.toString()); + buildState.recordPMDRun(moduleCoordinates); + } + project.getProperties().setProperty(EMPTY_JAR, ""); + if (!System.getProperties().containsKey(UNICORN)) { return; } + project.getProperties().setProperty(RESOURCE_CHECKSUM, + String.valueOf(resourceChecksum = getChecksum(mainResourceLocation, ANY_EXT))); + byte[] sourceChecksum = calculateChecksum(mainChecksum, resourceChecksum).getBytes(); + boolean instrumented = isCurrentModuleInstrumented(); + artifactPath = getArtifactPathInLocalRepo(session.getLocalRepository().getUrl(), project, sourceChecksum); + + boolean isFirstBuild = buildState.getBuildTime(moduleCoordinates) == 0 || !artifactPath.isPresent(); + Map<String, Object> moduleBuildData = getCurrentModuleBuildData(); Map<String, Object> lastTimeModuleBuildData = buildState.readModuleBuildData(); + resourceBuildData = getCurrentResourceBuildData(); + Map<String, Object> lastTimeResourceBuildData = buildState.readResourceBuildData(); - boolean buildDataSameWithPreviousBuild = lastTimeModuleBuildData.get(MAIN) != null && moduleBuildData.get(MAIN) - .equals(lastTimeModuleBuildData - .get(MAIN)); - boolean isFirstBuild = buildState.getBuildTime(moduleCoordinates) == 0; - - if (isCompileNeeded(HashMap.class.cast(moduleBuildData.get(MAIN)).keySet(), isFirstBuild, - buildDataSameWithPreviousBuild)) { - try { - buildState.markModuleDirty(inputSourceFilesList); - buildState.markModuleDirty(inputTestFilesList); - project.getProperties().setProperty(SKIP_TEST_RUN, "false"); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + boolean buildDataSameWithPreviousBuild = + isBuildDataSameWithPreviousBuild(lastTimeModuleBuildData, moduleBuildData); + boolean resourceMainBuildDataSameWithPreviousBuild = + isResourceMainBuildDataSameWithPreviousBuild(lastTimeResourceBuildData); + + boolean mainToBeCompiled = isCompileNeeded(HashMap.class.cast(moduleBuildData.get(MAIN)).keySet(), isFirstBuild, + buildDataSameWithPreviousBuild); + + boolean resourceDataSame = resourceBuildData.equals(lastTimeResourceBuildData); + + boolean testToBeCompiled = + lastTimeModuleBuildData == null || !moduleBuildData.get(TEST).equals(lastTimeModuleBuildData.get(TEST)); + setMainBuildAttribute(mainToBeCompiled, testToBeCompiled); + generateSignature(sourceChecksum); + setTestBuild(resourceDataSame, resourceMainBuildDataSameWithPreviousBuild, testToBeCompiled, mainToBeCompiled); + setInstrumentedBuild(testToBeCompiled, mainToBeCompiled, instrumented); + + if (!moduleBuildData.equals(lastTimeModuleBuildData) || isFirstBuild) { + buildState.addModuleBuildData(moduleCoordinates, moduleBuildData); } + setResourceBuild(resourceMainBuildDataSameWithPreviousBuild, mainToBeCompiled, testToBeCompiled); + setJarFlags(mainToBeCompiled, instrumented, !resourceMainBuildDataSameWithPreviousBuild); + setInstallFlags(mainToBeCompiled, instrumented, project.getPackaging(), + !resourceMainBuildDataSameWithPreviousBuild); - if (!moduleBuildData.get(TEST).equals(lastTimeModuleBuildData.get(TEST))) { - try { - buildState.markModuleDirty(inputTestFilesList); - project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + if (!mainToBeCompiled && !instrumented && JAR.equals(project.getPackaging()) + && resourceMainBuildDataSameWithPreviousBuild) { + project.getProperties().setProperty("artifactPathToCopy", artifactPath.orElse(null)); } + } - if (!moduleBuildData.equals(lastTimeModuleBuildData)) { - buildState.addModuleBuildData(moduleCoordinates, moduleBuildData); + private void generateSignature(byte[] sourceChecksum) { + try { + Files.write(Paths.get(project.getBuild().getDirectory(), project.getBuild().getFinalName() + DOT + UNICORN), + sourceChecksum, StandardOpenOption.CREATE); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private String calculateChecksum(long mainChecksum, long resourceChecksum) throws MojoExecutionException { + try { + return getSourceChecksum(mainChecksum + COLON + resourceChecksum, SHA1); + } catch (NoSuchAlgorithmException e) { + throw new MojoExecutionException(e.getMessage(), e); + } + } + + private boolean isResourceMainBuildDataSameWithPreviousBuild(Map<String, Object> lastTimeResourceBuildData) { + return lastTimeResourceBuildData != null && (lastTimeResourceBuildData.get(MAIN) != null && resourceBuildData + .get(MAIN) + .equals(lastTimeResourceBuildData + .get(MAIN))); + } + + private boolean isBuildDataSameWithPreviousBuild(Map<String, Object> lastTimeModuleBuildData, + Map<String, Object> moduleBuildData) { + return lastTimeModuleBuildData != null && (lastTimeModuleBuildData.get(MAIN) != null && moduleBuildData + .get(MAIN) + .equals(lastTimeModuleBuildData + .get(MAIN))); + } + + private void setInstrumentedBuild(boolean testToBeCompiled, boolean mainToBeCompiled, boolean instrumented) { + if (!testToBeCompiled && !mainToBeCompiled && instrumented) { + project.getProperties().setProperty(INSTRUMENT_ONLY, Boolean.TRUE.toString()); + project.getProperties().remove(SKIP_MAIN_SOURCE_COMPILE); + project.getProperties().remove(SKIP_TEST_SOURCE_COMPILE); + } + if (testToBeCompiled && !mainToBeCompiled && instrumented) { + project.getProperties().setProperty(INSTRUMENT_WITH_TEST_ONLY, Boolean.TRUE.toString()); + project.getProperties().remove(SKIP_MAIN_SOURCE_COMPILE); } + if (instrumented) { + buildState.markTestsMandatoryModule(moduleCoordinates); + project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); + } + } - if (inputSourceFilesList.isFile() && inputSourceFilesList.length() == 0) { - if (!inputSourceFilesList.delete()) { - throw new MojoExecutionException( - "****** Please remove 'target' directory manually under path " + project.getBasedir() - .getAbsolutePath()); + private void setResourceBuild(boolean resourceMainBuildDataSameWithPreviousBuild, boolean mainToBeCompiled, + boolean testToBeCompiled) { + if (resourceMainBuildDataSameWithPreviousBuild) { + project.getProperties().setProperty("skipResourceCollection", Boolean.TRUE.toString()); + } else { + project.getProperties().setProperty(RESOURCES_CHANGED, Boolean.TRUE.toString()); + } + if (!resourceMainBuildDataSameWithPreviousBuild && !mainToBeCompiled) { + project.getProperties().remove(SKIP_MAIN_SOURCE_COMPILE); + if (!testToBeCompiled) { + project.getProperties().remove(SKIP_TEST_SOURCE_COMPILE); + project.getProperties().setProperty(RESOURCE_ONLY, Boolean.TRUE.toString()); + } else { + project.getProperties().setProperty(RESOURCE_WITH_TEST_ONLY, Boolean.TRUE.toString()); } } - if (inputTestFilesList.isFile() && inputTestFilesList.length() == 0) { - if (!inputTestFilesList.delete()) { - throw new MojoExecutionException( - "****** Please remove 'target' directory manually under path " + project.getBasedir() - .getAbsolutePath()); + } + + private void setTestBuild(boolean resourceDataSame, boolean resourceMainBuildDataSameWithPreviousBuild, + boolean testToBeCompiled, boolean mainToBeCompiled) { + if (!resourceDataSame) { + buildState.addResourceBuildData(moduleCoordinates, resourceBuildData); + project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); + if (resourceMainBuildDataSameWithPreviousBuild && !testToBeCompiled && !mainToBeCompiled) { + project.getProperties().setProperty(TEST_RESOURCE_ONLY, Boolean.TRUE.toString()); + project.getProperties().remove(SKIP_MAIN_SOURCE_COMPILE); + project.getProperties().remove(SKIP_TEST_SOURCE_COMPILE); } } } + private void setMainBuildAttribute(boolean mainToBeCompiled, boolean testToBeCompiled) { + if (!mainToBeCompiled) { + project.getProperties().setProperty(SKIP_MAIN_SOURCE_COMPILE, Boolean.TRUE.toString()); + } + if (testToBeCompiled && !mainToBeCompiled) { + project.getProperties().setProperty(TEST_ONLY, Boolean.TRUE.toString()); + project.getProperties().remove(SKIP_MAIN_SOURCE_COMPILE); + } + + if (mainToBeCompiled || testToBeCompiled) { + project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); + } else { + project.getProperties().setProperty(SKIP_TEST_SOURCE_COMPILE, Boolean.TRUE.toString()); + } + } + + private void setJarFlags(boolean compile, boolean instrumented, boolean resourceChanged) { + if (compile || instrumented || resourceChanged) { + project.getProperties().setProperty(EMPTY_JAR, ""); + } else { + project.getProperties().setProperty(EMPTY_JAR, "**/*"); + project.getProperties().setProperty("mvnDsc", "false"); + } + } + + private void setInstallFlags(boolean compile, boolean instrumented, String packaging, boolean resourceChanged) { + if (!compile && !instrumented && !resourceChanged && JAR.equals(packaging)) { + project.getProperties().setProperty("skipInstall", Boolean.TRUE.toString()); + } + } + private boolean isCompileNeeded(Collection<String> dependencyCoordinates, boolean isFirstBuild, - boolean buildDataSame) { + boolean buildDataSame) throws MojoFailureException { return isFirstBuild || !buildDataSame || buildState.isCompileMust(moduleCoordinates, dependencyCoordinates); } + private boolean isCurrentModuleInstrumented() { + try { + return scanModuleFor(LifecyclePhase.PROCESS_CLASSES.id(), LifecyclePhase.PROCESS_TEST_CLASSES.id(), + LifecyclePhase.COMPILE.id(), LifecyclePhase.TEST_COMPILE.id()); + } catch (Exception e) { + return true; + } + } + + boolean isCodeGenerator() { + try { + return scanModuleFor(LifecyclePhase.GENERATE_RESOURCES.id(), LifecyclePhase.GENERATE_SOURCES.id(), + LifecyclePhase.GENERATE_TEST_RESOURCES.id(), LifecyclePhase.GENERATE_TEST_SOURCES.id()); + } catch (Exception e) { + return true; + } + } + private Map<String, Object> getCurrentModuleBuildData() { Map<String, Object> moduleBuildData = new HashMap<>(); moduleBuildData.put(MAIN, new HashMap<String, String>()); moduleBuildData.put(TEST, new HashMap<String, String>()); + HashMap.class.cast(moduleBuildData.get(MAIN)) + .put(MAIN_SOURCE_CHECKSUM, project.getProperties().getProperty(MAIN_CHECKSUM)); + HashMap.class.cast(moduleBuildData.get(TEST)) + .put(TEST_SOURCE_CHECKSUM, project.getProperties().getProperty(TEST_CHECKSUM)); + if (isCodeGenerator()) { + HashMap.class.cast(moduleBuildData.get(MAIN)) + .put(GENERATED_SOURCE_CHECKSUM, getChecksum(generatedSourceLocation, JAVA_EXT)); + } if (project.getArtifacts() == null || project.getArtifacts().isEmpty()) { return moduleBuildData; } for (Artifact dependency : project.getArtifacts()) { + String fileNme = dependency.getFile().getName(); if (excludeDependencies.contains(dependency.getScope())) { HashMap.class.cast(moduleBuildData.get(TEST)) - .put(dependency.getGroupId() + ":" + dependency.getArtifactId(), dependency.getVersion()); + .put(dependency.getGroupId() + COLON + dependency.getArtifactId(), + fileNme.endsWith(dependency.getVersion() + DOT + JAR) ? dependency.getVersion() : + fileNme); continue; } HashMap.class.cast(moduleBuildData.get(MAIN)) - .put(dependency.getGroupId() + ":" + dependency.getArtifactId(), dependency.getVersion()); + .put(dependency.getGroupId() + COLON + dependency.getArtifactId(), + fileNme.endsWith(dependency.getVersion() + DOT + JAR) ? dependency.getVersion() : + fileNme); } return moduleBuildData; } + + private static Map<String, String> readCurrentPMDState(String fileName) { + Optional<HashMap> val = readState(fileName, HashMap.class); + return val.orElseGet(HashMap::new); + } + + private boolean isPMDMandatory(Set<Artifact> dependencies) { + for (Artifact artifact : dependencies) { + if (buildState.isPMDRun(artifact.getGroupId() + COLON + artifact.getArtifactId())) { + return true; + } + } + return false; + } + + private boolean scanModuleFor(String... types) + throws InvalidPluginDescriptorException, PluginResolutionException, MojoNotFoundException, + PluginDescriptorParsingException { + for (Plugin plugin : project.getBuildPlugins()) { + if (!(plugin.getGroupId().equals("org.apache.maven.plugins") && plugin.getArtifactId().startsWith("maven")) + && !plugin.getGroupId().startsWith("org.openecomp.sdc")) { + if (scanPlugin(plugin, types)) { + return true; + } + } + } + return false; + } + + private boolean scanPlugin(Plugin plugin, String... types) + throws InvalidPluginDescriptorException, PluginDescriptorParsingException, MojoNotFoundException, + PluginResolutionException { + for (PluginExecution pluginExecution : plugin.getExecutions()) { + if (pluginExecution.getPhase() != null) { + if (Arrays.asList(types).contains(pluginExecution.getPhase())) { + return true; + } + } + for (String goal : pluginExecution.getGoals()) { + MojoDescriptor md = pluginManager.getMojoDescriptor(plugin, goal, project.getRemotePluginRepositories(), + session.getRepositorySession()); + if (Arrays.asList(types).contains(md.getPhase())) { + return true; + } + } + } + return false; + } + + private Map<String, Object> getCurrentResourceBuildData() { + HashMap<String, Object> resourceBuildStateData = new HashMap<>(); + resourceBuildStateData.put(MAIN, project.getProperties().getProperty(RESOURCE_CHECKSUM)); + resourceBuildStateData.put(TEST, getChecksum(testResourceLocation, ANY_EXT)); + resourceBuildStateData.put("dependency", getDependencies().hashCode()); + return resourceBuildStateData; + } + + private Map<String, String> getDependencies() { + Map<String, String> dependencies = new HashMap<>(); + for (Artifact d : project.getArtifacts()) { + dependencies.put(d.getGroupId() + COLON + d.getArtifactId(), d.getVersion()); + } + return dependencies; + } + + private void init() { + if (mainSourceLocation == null) { + mainSourceLocation = Paths.get(project.getBuild().getSourceDirectory()).toFile(); + } + if (testSourceLocation == null) { + testSourceLocation = Paths.get(project.getBuild().getTestSourceDirectory()).toFile(); + } + if (mainResourceLocation == null) { + mainResourceLocation = Paths.get(project.getBuild().getResources().get(0).getDirectory()).toFile(); + } + if (testResourceLocation == null) { + testResourceLocation = Paths.get(project.getBuild().getTestResources().get(0).getDirectory()).toFile(); + } + } } diff --git a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreTestCompileHelperMojo.java b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreTestCompileHelperMojo.java index e711cb0ae7..a11d796464 100644 --- a/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreTestCompileHelperMojo.java +++ b/openecomp-be/tools/compile-helper-plugin/src/main/java/org/openecomp/sdc/onboarding/PreTestCompileHelperMojo.java @@ -17,14 +17,13 @@ package org.openecomp.sdc.onboarding; import static org.openecomp.sdc.onboarding.Constants.JACOCO_SKIP; -import static org.openecomp.sdc.onboarding.Constants.SKIP_TEST_RUN; import static org.openecomp.sdc.onboarding.Constants.RESOURCES_CHANGED; +import static org.openecomp.sdc.onboarding.Constants.SKIP_TESTS; +import static org.openecomp.sdc.onboarding.Constants.SKIP_TEST_RUN; import static org.openecomp.sdc.onboarding.Constants.UNICORN; -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.stream.Collectors; +import java.util.List; +import org.apache.maven.model.Plugin; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -33,23 +32,20 @@ import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.plugins.annotations.ResolutionScope; import org.apache.maven.project.MavenProject; +import org.codehaus.plexus.util.xml.Xpp3Dom; -@Mojo(name = "pre-test-compile-helper", threadSafe = true, defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES, +@Mojo(name = "pre-test-compile-helper", threadSafe = true, defaultPhase = LifecyclePhase.PROCESS_TEST_CLASSES, requiresDependencyResolution = ResolutionScope.TEST) public class PreTestCompileHelperMojo extends AbstractMojo { @Parameter - private File compiledFilesList; - @Parameter - private Long staleThreshold; - @Parameter - private File inputTestFilesList; - @Parameter private BuildState buildState; @Parameter(defaultValue = "${project}", readonly = true) private MavenProject project; @Parameter(defaultValue = "${project.artifact.groupId}:${project.artifact.artifactId}") private String moduleCoordinates; + @Parameter(defaultValue = "${project.buildPlugins}", readonly = true) + private List<Plugin> plugins; @Parameter private String excludePackaging; @@ -61,25 +57,17 @@ public class PreTestCompileHelperMojo extends AbstractMojo { if (project.getPackaging().equals(excludePackaging)) { return; } - if (compiledFilesList.exists() - && compiledFilesList.lastModified() > System.currentTimeMillis() - staleThreshold) { - try { - buildState.markModuleDirty(inputTestFilesList); - project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + if (buildState.isTestExecutionMandatory()) { + project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); } - boolean isTestMust = buildState.isTestMust(moduleCoordinates, - project.getDependencies().stream().map(d -> d.getGroupId() + ":" + d.getArtifactId()) - .collect(Collectors.toList())); + boolean isTestMust = buildState.isTestMust(moduleCoordinates); if (isTestMust) { project.getProperties().setProperty(RESOURCES_CHANGED, Boolean.TRUE.toString()); if (!project.getProperties().containsKey(SKIP_TEST_RUN)) { project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); } } - if (!project.getProperties().containsKey(SKIP_TEST_RUN)) { + if (!project.getProperties().containsKey(SKIP_TEST_RUN) || isTestSkippedExplicitly()) { project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.TRUE.toString()); } if (System.getProperties().containsKey(JACOCO_SKIP) && Boolean.FALSE.equals(Boolean.valueOf( @@ -87,4 +75,17 @@ public class PreTestCompileHelperMojo extends AbstractMojo { project.getProperties().setProperty(SKIP_TEST_RUN, Boolean.FALSE.toString()); } } + + + private boolean isTestSkippedExplicitly() { + for (Plugin p : plugins) { + if ("org.apache.maven.plugins:maven-surefire-plugin".equals(p.getKey())) { + Xpp3Dom dom = Xpp3Dom.class.cast(p.getConfiguration()); + if (dom.getChild(SKIP_TESTS) != null) { + return Boolean.TRUE.equals(Boolean.valueOf(dom.getValue())); + } + } + } + return false; + } } |