diff options
| -rw-r--r-- | feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/RepositoryAudit.java | 291 | ||||
| -rw-r--r-- | packages/docker/pom.xml | 1 |
2 files changed, 167 insertions, 125 deletions
diff --git a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/RepositoryAudit.java b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/RepositoryAudit.java index dc0f3784..66982ef9 100644 --- a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/RepositoryAudit.java +++ b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/RepositoryAudit.java @@ -32,7 +32,11 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.LinkedList; import java.util.List; import java.util.Properties; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,13 +44,18 @@ import org.slf4j.LoggerFactory; * This class audits the Maven repository. */ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { - private static final long DEFAULT_TIMEOUT = 60; //timeout in 60 seconds + // timeout in 60 seconds + private static final long DEFAULT_TIMEOUT = 60; // get an instance of logger - private static Logger logger = LoggerFactory.getLogger(RepositoryAudit.class); + private static final Logger logger = LoggerFactory.getLogger(RepositoryAudit.class); + // single global instance of this audit object private static RepositoryAudit instance = new RepositoryAudit(); + // Regex pattern used to find additional repos in the form "repository(number).id.url" + private static final Pattern repoPattern = Pattern.compile("(repository([1-9][0-9]*))[.]audit[.]id"); + /** * Constructor - set the name to 'Repository'. */ @@ -64,19 +73,46 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } /** + * First, get the names of each property from StateManagementProperties. For each property name, check if it is of + * the form "repository(number).audit.id" If so, we extract the number and determine if there exists another + * property in the form "repository(number).audit.url" with the same "number". Only the + * 'repository(number).audit.id' and 'repository(number).audit.url" properties need to be specified. If both 'id' + * and 'url' properties are found, we add it to our set. InvokeData.getProperty(String, boolean) will determine the + * other 4 properties: '*.username', '*.password', '*.is.active', and '*.ignore.errors', or use default values. + * + * @return set of Integers representing a repository to support + */ + private static TreeSet<Integer> countAdditionalNexusRepos() { + TreeSet<Integer> returnIndices = new TreeSet<>(); + Properties properties = StateManagementProperties.getProperties(); + Set<String> propertyNames = properties.stringPropertyNames(); + + for (String currName : propertyNames) { + Matcher matcher = repoPattern.matcher(currName); + + if (matcher.matches()) { + int currRepoNum = Integer.parseInt(matcher.group(2)); + if (propertyNames.contains(matcher.group(1) + ".audit.url")) { + returnIndices.add(currRepoNum); + } + } + } + return returnIndices; + } + + /** * Invoke the audit. * * @param properties properties to be passed to the audit */ @Override - public void invoke(Properties properties) - throws IOException, InterruptedException { + public void invoke(Properties properties) throws IOException, InterruptedException { logger.debug("Running 'RepositoryAudit.invoke'"); InvokeData data = new InvokeData(); - logger.debug("RepositoryAudit.invoke: repoAuditIsActive = {}" - + ", repoAuditIgnoreErrors = {}",data.repoAuditIsActive, data.repoAuditIgnoreErrors); + logger.debug("RepositoryAudit.invoke: repoAuditIsActive = {}" + ", repoAuditIgnoreErrors = {}", + data.repoAuditIsActive, data.repoAuditIgnoreErrors); data.initIsActive(); @@ -85,6 +121,29 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { return; } + // Run audit for first nexus repository + logger.debug("Running read-only audit on first nexus repository: repository"); + runAudit(data); + + // set of indices for supported nexus repos (ex: repository2 -> 2) + // TreeSet is used to maintain order so repos can be audited in numerical order + TreeSet<Integer> repoIndices = countAdditionalNexusRepos(); + logger.debug("Additional nexus repositories: {}", repoIndices); + + // Run audit for remaining 'numNexusRepos' repositories + for (int index : repoIndices) { + logger.debug("Running read-only audit on nexus repository = repository{}", index); + + data = new InvokeData(index); + data.initIsActive(); + + if (data.isActive) { + runAudit(data); + } + } + } + + private void runAudit(InvokeData data) throws IOException, InterruptedException { data.initIgnoreErrors(); data.initTimeout(); @@ -99,8 +158,7 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { final Path repo = data.dir.resolve("repo"); /* - * 2) Create test file, and upload to repository - * (only if repository information is specified) + * 2) Create test file, and upload to repository (only if repository information is specified) */ if (data.upload) { data.uploadTestFile(); @@ -122,23 +180,20 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { int rval = data.runMaven(output); /* - * 4a) Check attempted and successful downloads from output file - * Note: at present, this step just generates log messages, - * but doesn't do any verification. + * 4a) Check attempted and successful downloads from output file Note: at present, this step just generates log + * messages, but doesn't do any verification. */ if (rval == 0 && output != null) { generateDownloadLogs(output); } /* - * 5) Check the contents of the directory to make sure the downloads - * were successful + * 5) Check the contents of the directory to make sure the downloads were successful */ data.verifyDownloads(repo); /* - * 6) Use 'curl' to delete the uploaded test file - * (only if repository information is specified) + * 6) Use 'curl' to delete the uploaded test file (only if repository information is specified) */ if (data.upload) { data.deleteUploadedTestFile(); @@ -150,6 +205,20 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { Files.walkFileTree(data.dir, new RecursivelyDeleteDirectory()); } + + /** + * Set the response string to the specified value. Overrides 'setResponse(String value)' from + * DroolsPdpIntegrityMonitor This method prevents setting a response string that indicates whether the caller should + * receive an error list from the audit. By NOT setting the response string to a value, this indicates that there + * are no errors. + * + * @param value the new value of the response string (null = no errors) + */ + @Override + public void setResponse(String value) { + // Do nothing, prevent the caller from receiving a list of errors. + } + private class InvokeData { private boolean isActive = true; @@ -180,18 +249,39 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { // artifacts to be downloaded private final List<Artifact> artifacts = new LinkedList<>(); + // 0 = base repository, 2-n = additional repositories + private final int index; + public InvokeData() { - repoAuditIsActive = StateManagementProperties.getProperty("repository.audit.is.active"); - repoAuditIgnoreErrors = StateManagementProperties.getProperty("repository.audit.ignore.errors"); + this(0); + } + + public InvokeData(int index) { + this.index = index; + repoAuditIsActive = getProperty("audit.is.active", true); + repoAuditIgnoreErrors = getProperty("audit.ignore.errors", true); // Fetch repository information from 'IntegrityMonitorProperties' - repositoryId = StateManagementProperties.getProperty("repository.audit.id"); - repositoryUrl = StateManagementProperties.getProperty("repository.audit.url"); - repositoryUsername = StateManagementProperties.getProperty("repository.audit.username"); - repositoryPassword = StateManagementProperties.getProperty("repository.audit.password"); + repositoryId = getProperty("audit.id", false); + repositoryUrl = getProperty("audit.url", false); + repositoryUsername = getProperty("audit.username", true); + repositoryPassword = getProperty("audit.password", true); - upload = repositoryId != null && repositoryUrl != null - && repositoryUsername != null && repositoryPassword != null; + logger.debug("Nexus Repository Information retrieved from 'IntegrityMonitorProperties':"); + logger.debug("repositoryId: " + repositoryId); + logger.debug("repositoryUrl: " + repositoryUrl); + + // Setting upload to be false so that files can no longer be created/deleted + upload = false; + } + + private String getProperty(String property, boolean useDefault) { + String fullProperty = (index == 0 ? "repository." + property : "repository" + index + "." + property); + String rval = StateManagementProperties.getProperty(fullProperty); + if (rval == null && index != 0 && useDefault) { + rval = StateManagementProperties.getProperty("repository." + property); + } + return rval; } public void initIsActive() { @@ -203,6 +293,9 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { repoAuditIsActive); } } + if (repositoryId == null || repositoryUrl == null) { + isActive = false; + } } public void initIgnoreErrors() { @@ -212,8 +305,8 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } catch (NumberFormatException e) { ignoreErrors = true; logger.warn( - "RepositoryAudit.invoke: Ignoring invalid property: repository.audit.ignore.errors = {}", - repoAuditIgnoreErrors); + "RepositoryAudit.invoke: Ignoring invalid property: repository.audit.ignore.errors = {}", + repoAuditIgnoreErrors); } } else { ignoreErrors = true; @@ -221,17 +314,15 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } public void initTimeout() { - String timeoutString = - StateManagementProperties.getProperty("repository.audit.timeout"); + String timeoutString = getProperty("audit.timeout", true); if (timeoutString != null && !timeoutString.isEmpty()) { try { timeoutInSeconds = Long.valueOf(timeoutString); } catch (NumberFormatException e) { - logger.error("RepositoryAudit: Invalid 'repository.audit.timeout' value: '{}'", - timeoutString, e); + logger.error("RepositoryAudit: Invalid 'repository.audit.timeout' value: '{}'", timeoutString, e); if (!ignoreErrors) { - response.append("Invalid 'repository.audit.timeout' value: '") - .append(timeoutString).append("'\n"); + response.append("Invalid 'repository.audit.timeout' value: '").append(timeoutString) + .append("'\n"); setResponse(response.toString()); } } @@ -249,29 +340,21 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } // create text file to write - try (FileOutputStream fos = - new FileOutputStream(dir.resolve("repository-audit.txt").toFile())) { + try (FileOutputStream fos = new FileOutputStream(dir.resolve("repository-audit.txt").toFile())) { fos.write(version.getBytes()); } // try to install file in repository - if (runProcess(timeoutInSeconds, dir.toFile(), null, - "mvn", "deploy:deploy-file", - "-DrepositoryId=" + repositoryId, - "-Durl=" + repositoryUrl, - "-Dfile=repository-audit.txt", - "-DgroupId=" + groupId, - "-DartifactId=" + artifactId, - "-Dversion=" + version, - "-Dpackaging=txt", - "-DgeneratePom=false") != 0) { + if (runProcess(timeoutInSeconds, dir.toFile(), null, "mvn", "deploy:deploy-file", + "-DrepositoryId=" + repositoryId, "-Durl=" + repositoryUrl, "-Dfile=repository-audit.txt", + "-DgroupId=" + groupId, "-DartifactId=" + artifactId, "-Dversion=" + version, "-Dpackaging=txt", + "-DgeneratePom=false") != 0) { logger.error("RepositoryAudit: 'mvn deploy:deploy-file' failed"); if (!ignoreErrors) { response.append("'mvn deploy:deploy-file' failed\n"); setResponse(response.toString()); } - } - else { + } else { logger.info("RepositoryAudit: 'mvn deploy:deploy-file succeeded"); // we also want to include this new artifact in the download @@ -280,64 +363,36 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } } - private void createPomFile(final Path repo, final Path pom) - throws IOException { + private void createPomFile(final Path repo, final Path pom) throws IOException { artifacts.add(new Artifact("org.apache.maven/maven-embedder/3.2.2")); StringBuilder sb = new StringBuilder(); - sb.append("<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" - + " xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd\">\n" - + "\n" - + " <modelVersion>4.0.0</modelVersion>\n" - + " <groupId>empty</groupId>\n" - + " <artifactId>empty</artifactId>\n" - + " <version>1.0-SNAPSHOT</version>\n" - + " <packaging>pom</packaging>\n" - + "\n" - + " <build>\n" - + " <plugins>\n" - + " <plugin>\n" - + " <groupId>org.apache.maven.plugins</groupId>\n" - + " <artifactId>maven-dependency-plugin</artifactId>\n" - + " <version>2.10</version>\n" - + " <executions>\n" - + " <execution>\n" - + " <id>copy</id>\n" - + " <goals>\n" - + " <goal>copy</goal>\n" - + " </goals>\n" - + " <configuration>\n" - + " <localRepositoryDirectory>") - .append(repo) - .append("</localRepositoryDirectory>\n") - .append(" <artifactItems>\n"); + sb.append( + "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + + " xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd\">\n" + + "\n" + " <modelVersion>4.0.0</modelVersion>\n" + " <groupId>empty</groupId>\n" + + " <artifactId>empty</artifactId>\n" + " <version>1.0-SNAPSHOT</version>\n" + + " <packaging>pom</packaging>\n" + "\n" + " <build>\n" + " <plugins>\n" + + " <plugin>\n" + " <groupId>org.apache.maven.plugins</groupId>\n" + + " <artifactId>maven-dependency-plugin</artifactId>\n" + + " <version>2.10</version>\n" + " <executions>\n" + + " <execution>\n" + " <id>copy</id>\n" + " <goals>\n" + + " <goal>copy</goal>\n" + " </goals>\n" + + " <configuration>\n" + " <localRepositoryDirectory>") + .append(repo).append("</localRepositoryDirectory>\n").append(" <artifactItems>\n"); for (Artifact artifact : artifacts) { // each artifact results in an 'artifactItem' element - sb.append(" <artifactItem>\n" - + " <groupId>") - .append(artifact.groupId) - .append("</groupId>\n" - + " <artifactId>") - .append(artifact.artifactId) - .append("</artifactId>\n" - + " <version>") - .append(artifact.version) - .append("</version>\n" - + " <type>") - .append(artifact.type) - .append("</type>\n" - + " </artifactItem>\n"); + sb.append(" <artifactItem>\n" + " <groupId>").append(artifact.groupId) + .append("</groupId>\n" + " <artifactId>").append(artifact.artifactId) + .append("</artifactId>\n" + " <version>").append(artifact.version) + .append("</version>\n" + " <type>").append(artifact.type) + .append("</type>\n" + " </artifactItem>\n"); } - sb.append(" </artifactItems>\n" - + " </configuration>\n" - + " </execution>\n" - + " </executions>\n" - + " </plugin>\n" - + " </plugins>\n" - + " </build>\n" - + "</project>\n"); + sb.append(" </artifactItems>\n" + " </configuration>\n" + + " </execution>\n" + " </executions>\n" + " </plugin>\n" + + " </plugins>\n" + " </build>\n" + "</project>\n"); try (FileOutputStream fos = new FileOutputStream(pom.toFile())) { fos.write(sb.toString().getBytes()); @@ -359,19 +414,17 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { private void verifyDownloads(final Path repo) { for (Artifact artifact : artifacts) { - if (repo.resolve(artifact.groupId.replace('.','/')) - .resolve(artifact.artifactId) + if (repo.resolve(artifact.groupId.replace('.', '/')).resolve(artifact.artifactId) .resolve(artifact.version) - .resolve(artifact.artifactId + "-" + artifact.version + "." - + artifact.type).toFile().exists()) { + .resolve(artifact.artifactId + "-" + artifact.version + "." + artifact.type).toFile() + .exists()) { // artifact exists, as expected logger.info("RepositoryAudit: {} : exists", artifact.toString()); } else { // Audit ERROR: artifact download failed for some reason logger.error("RepositoryAudit: {}: does not exist", artifact.toString()); if (!ignoreErrors) { - response.append("Failed to download artifact: ") - .append(artifact).append('\n'); + response.append("Failed to download artifact: ").append(artifact).append('\n'); setResponse(response.toString()); } } @@ -379,13 +432,9 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } private void deleteUploadedTestFile() throws IOException, InterruptedException { - if (runProcess(timeoutInSeconds, dir.toFile(), null, - "curl", - "--request", "DELETE", - "--user", repositoryUsername + ":" + repositoryPassword, - repositoryUrl + "/" + groupId.replace('.', '/') + "/" - + artifactId + "/" + version) - != 0) { + if (runProcess(timeoutInSeconds, dir.toFile(), null, "curl", "--request", "DELETE", "--user", + repositoryUsername + ":" + repositoryPassword, + repositoryUrl + "/" + groupId.replace('.', '/') + "/" + artifactId + "/" + version) != 0) { logger.error("RepositoryAudit: delete of uploaded artifact failed"); if (!ignoreErrors) { response.append("delete of uploaded artifact failed\n"); @@ -401,7 +450,7 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { private void generateDownloadLogs(File output) throws IOException { // place output in 'fileContents' (replacing the Return characters // with Newline) - byte[] outputData = new byte[(int)output.length()]; + byte[] outputData = new byte[(int) output.length()]; String fileContents; try (FileInputStream fis = new FileInputStream(output)) { // @@ -410,7 +459,7 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { // int bytesRead = fis.read(outputData); logger.info("fileContents read {} bytes", bytesRead); - fileContents = new String(outputData).replace('\r','\n'); + fileContents = new String(outputData).replace('\r', '\n'); } // generate log messages from 'Downloading' and 'Downloaded' @@ -421,13 +470,12 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { if (fileContents.regionMatches(index, "loading: ", 0, 9)) { index += 9; int endIndex = fileContents.indexOf('\n', index); - logger.info("RepositoryAudit: Attempted download: '{}'", - fileContents.substring(index, endIndex)); + logger.info("RepositoryAudit: Attempted download: '{}'", fileContents.substring(index, endIndex)); index = endIndex; } else if (fileContents.regionMatches(index, "loaded: ", 0, 8)) { index += 8; int endIndex = fileContents.indexOf(' ', index); - logger.info("RepositoryAudit: Successful download: '{}'",fileContents.substring(index, endIndex)); + logger.info("RepositoryAudit: Successful download: '{}'", fileContents.substring(index, endIndex)); index = endIndex; } } @@ -443,9 +491,8 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { * @return the return value of the process * @throws IOException InterruptedException */ - static int runProcess(long timeoutInSeconds, - File directory, File stdout, String... command) - throws IOException, InterruptedException { + static int runProcess(long timeoutInSeconds, File directory, File stdout, String... command) + throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(command); if (directory != null) { pb.directory(directory); @@ -466,8 +513,7 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } /** - * This class is used to recursively delete a directory and all of its - * contents. + * This class is used to recursively delete a directory and all of its contents. */ private final class RecursivelyDeleteDirectory extends SimpleFileVisitor<Path> { @Override @@ -477,8 +523,7 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { } @Override - public FileVisitResult postVisitDirectory(Path file, IOException ex) - throws IOException { + public FileVisitResult postVisitDirectory(Path file, IOException ex) throws IOException { if (ex == null) { file.toFile().delete(); return FileVisitResult.CONTINUE; @@ -491,8 +536,7 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { /* ============================================================ */ /** - * An instance of this class exists for each artifact that we are trying - * to download. + * An instance of this class exists for each artifact that we are trying to download. */ static class Artifact { String groupId; @@ -518,8 +562,7 @@ public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase { /** * Constructor - populate an 'Artifact' instance. * - * @param artifact a string of the form: - * {@code"<groupId>/<artifactId>/<version>[/<type>]"} + * @param artifact a string of the form: {@code"<groupId>/<artifactId>/<version>[/<type>]"} * @throws IllegalArgumentException if 'artifact' has the incorrect format */ Artifact(String artifact) { diff --git a/packages/docker/pom.xml b/packages/docker/pom.xml index b961a6ad..9a5979a8 100644 --- a/packages/docker/pom.xml +++ b/packages/docker/pom.xml @@ -83,7 +83,6 @@ <plugin> <groupId>io.fabric8</groupId> <artifactId>docker-maven-plugin</artifactId> - <version>0.30.0</version> <!-- Extract this as a property from the policy-parent --> <configuration> <verbose>true</verbose> <apiVersion>1.23</apiVersion> |