Init ncomp.utils

Change-Id: Ib092dff1d165296e2d86265137cf0b5ae8b20323
Signed-off-by: lj1412 <lji@research.att.com>

11 files changed, 499 insertions, 0 deletions

diff --git a/ncomp-utils-java-extra/.classpath b/ncomp-utils-java-extra/.classpath
new file mode 100644
index 0000000..79ccca7
--- /dev/null
+++ b/ncomp-utils-java-extra/.classpath
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" output="target/classes" path="src/main/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="output" path="target/classes"/>
+</classpath>
diff --git a/ncomp-utils-java-extra/.gitignore b/ncomp-utils-java-extra/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/ncomp-utils-java-extra/.gitignore
@@ -0,0 +1 @@
+/target/
diff --git a/ncomp-utils-java-extra/.project b/ncomp-utils-java-extra/.project
new file mode 100644
index 0000000..3ba37c0
--- /dev/null
+++ b/ncomp-utils-java-extra/.project
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>ncomp-utils-java-extra</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.ManifestBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.SchemaBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.ManifestBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.SchemaBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
+		<nature>org.eclipse.pde.PluginNature</nature>
+	</natures>
+</projectDescription>
diff --git a/ncomp-utils-java-extra/.settings/org.eclipse.core.resources.prefs b/ncomp-utils-java-extra/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..e9441bb
--- /dev/null
+++ b/ncomp-utils-java-extra/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,3 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding/<project>=UTF-8
diff --git a/ncomp-utils-java-extra/.settings/org.eclipse.jdt.core.prefs b/ncomp-utils-java-extra/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..ec4300d
--- /dev/null
+++ b/ncomp-utils-java-extra/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.source=1.7
diff --git a/ncomp-utils-java-extra/.settings/org.eclipse.m2e.core.prefs b/ncomp-utils-java-extra/.settings/org.eclipse.m2e.core.prefs
new file mode 100644
index 0000000..f897a7f
--- /dev/null
+++ b/ncomp-utils-java-extra/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
diff --git a/ncomp-utils-java-extra/LICENSE.txt b/ncomp-utils-java-extra/LICENSE.txt
new file mode 100644
index 0000000..30471b5
--- /dev/null
+++ b/ncomp-utils-java-extra/LICENSE.txt
@@ -0,0 +1,22 @@
+
+/*-
+ * ============LICENSE_START==========================================
+ * OPENECOMP - DCAE
+ * ===================================================================
+ * Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+ * ===================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0 
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END============================================
+ */
+	
+ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property.
diff --git a/ncomp-utils-java-extra/META-INF/MANIFEST.MF b/ncomp-utils-java-extra/META-INF/MANIFEST.MF
new file mode 100644
index 0000000..e855152
--- /dev/null
+++ b/ncomp-utils-java-extra/META-INF/MANIFEST.MF
@@ -0,0 +1,13 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: %pluginName
+Bundle-SymbolicName: ncomp-utils-java-extra
+Bundle-Version: 0.1.0.qualifier
+Export-Package: org.openecomp.ncomp.utils.extra
+Require-Bundle: org.eclipse.emf.ecore;bundle-version="2.10.2",
+ org.eclipse.emf.ecore.xmi;bundle-version="2.10.2",
+ org.junit;bundle-version="4.11.0",
+ org.apache.log4j;bundle-version="1.2.15",
+ org.apache.commons.cli,
+ org.apache.commons.compress;bundle-version="1.6.0"
+Bundle-Vendor: %providerName
diff --git a/ncomp-utils-java-extra/build.properties b/ncomp-utils-java-extra/build.properties
new file mode 100644
index 0000000..a1ec8c4
--- /dev/null
+++ b/ncomp-utils-java-extra/build.properties
@@ -0,0 +1,3 @@
+source.. = src/main/java/
+bin.includes = META-INF/,\
+               .
diff --git a/ncomp-utils-java-extra/pom.xml b/ncomp-utils-java-extra/pom.xml
new file mode 100644
index 0000000..9b6640c
--- /dev/null
+++ b/ncomp-utils-java-extra/pom.xml
@@ -0,0 +1,162 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>org.openecomp.ncomp.utils</groupId>
+	<artifactId>ncomp-utils-java-extra</artifactId>
+	<version>0.1.0-SNAPSHOT</version>
+
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<maven.compiler.target>1.7</maven.compiler.target>
+		<maven.compiler.source>1.7</maven.compiler.source>
+	</properties>
+
+	
+	<pluginRepositories>
+		<!-- Black Duck plugin dependencies -->
+		<pluginRepository>
+			<id>JCenter</id>
+			<name>JCenter Repository</name>
+			<url>http://jcenter.bintray.com</url>
+		</pluginRepository>
+
+		<pluginRepository>
+			<id>Restlet</id>
+			<name>Restlet Repository</name>
+			<url>http://maven.restlet.com</url>
+		</pluginRepository>
+	</pluginRepositories>
+<build>
+		<plugins>
+		<!-- blackduck maven plugin -->
+			<plugin>
+				<groupId>com.blackducksoftware.integration</groupId>
+				<artifactId>hub-maven-plugin</artifactId>
+				<version>1.4.0</version>
+				<inherited>false</inherited>
+				<configuration>
+					<hubProjectName>${project.name}</hubProjectName>
+					<outputDirectory>${project.basedir}</outputDirectory>
+				</configuration>
+				<executions>
+					<execution>
+						<id>create-bdio-file</id>
+						<phase>package</phase>
+						<goals>
+							<goal>createHubOutput</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+       <!-- site maven plugin -->
+       <plugin>
+           <groupId>org.apache.maven.plugins</groupId>
+           <artifactId>maven-site-plugin</artifactId>
+           <version>3.6</version>
+           <dependencies>
+             <dependency>
+               <groupId>org.apache.maven.wagon</groupId>
+               <artifactId>wagon-webdav-jackrabbit</artifactId>
+               <version>2.10</version>
+             </dependency>
+           </dependencies>
+       </plugin>
+    
+			<plugin>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.1</version>
+				<configuration>
+					<source />
+					<target />
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+
+	<dependencies>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>1.2.17</version>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.11</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-compress</artifactId>
+			<version>1.9</version>
+		</dependency>
+
+		<dependency>
+			<groupId>commons-cli</groupId>
+			<artifactId>commons-cli</artifactId>
+			<version>1.3</version>
+		</dependency>
+
+
+		<dependency>
+			<groupId>org.eclipse.emf</groupId>
+			<artifactId>org.eclipse.emf.ecore</artifactId>
+			<version>2.11.0-v20150123-0347</version>
+		</dependency>
+		<dependency>
+			<groupId>org.eclipse.emf</groupId>
+			<artifactId>org.eclipse.emf.common</artifactId>
+			<version>2.11.0-v20150123-0347</version>
+		</dependency>
+		<dependency>
+			<groupId>org.eclipse.emf</groupId>
+			<artifactId>org.eclipse.emf.ecore.xmi</artifactId>
+			<version>2.11.0-v20150123-0347</version>
+		</dependency>
+		<dependency>
+			<groupId>org.openecomp.ncomp.utils</groupId>
+			<artifactId>ncomp-utils-journaling</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+	</dependencies>
+
+	
+		<distributionManagement>
+			<repository>
+				<id>osecomp-nexus-releases</id>
+				<name>OSECOMP Release Repository</name>
+				<url>${openecomp.nexus.releases}</url>
+			</repository>
+			<snapshotRepository>
+				<id>osecomp-nexus-snapshots</id>
+				<name>OSECOMP Snapshot Repository</name>
+				<url>${openecomp.nexus.snapshots}</url>
+			</snapshotRepository>
+			<site>
+				<id>dcae-javadoc</id>
+				<url>${site.urlroot}/${project.artifactId}/${project.version}/</url>
+			</site>
+		</distributionManagement>
+	
+   <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.10.4</version>
+                <configuration>
+                    <failOnError>false</failOnError>
+                    <doclet>org.umlgraph.doclet.UmlGraphDoc</doclet>
+                    <docletArtifact>
+                        <groupId>org.umlgraph</groupId>
+                        <artifactId>umlgraph</artifactId>
+                        <version>5.6</version>
+                    </docletArtifact>
+                    <additionalparam>-views</additionalparam>
+                    <useStandardDocletOptions>true</useStandardDocletOptions>
+                </configuration>
+            </plugin>
+        </plugins>
+    </reporting>
+
+</project>
diff --git a/ncomp-utils-java-extra/src/main/java/org/openecomp/ncomp/utils/extra/FileTail.java b/ncomp-utils-java-extra/src/main/java/org/openecomp/ncomp/utils/extra/FileTail.java
new file mode 100644
index 0000000..623733d
--- /dev/null
+++ b/ncomp-utils-java-extra/src/main/java/org/openecomp/ncomp/utils/extra/FileTail.java
@@ -0,0 +1,215 @@
+
+/*-
+ * ============LICENSE_START==========================================
+ * OPENECOMP - DCAE
+ * ===================================================================
+ * Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+ * ===================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0 
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END============================================
+ */
+	
+package org.openecomp.ncomp.utils.extra;
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.json.JSONObject;
+
+import org.openecomp.ncomp.utils.FindFiles;
+import org.openecomp.ncomp.utils.FindFiles.ParameterizedFile;
+import org.openecomp.ncomp.utils.journaling.JournalingHashMap;
+import org.openecomp.ncomp.utils.logging.LoggingUtils;
+import org.openecomp.ncomp.webservice.utils.DateUtils;
+
+public class FileTail {
+	public static final Logger logger = Logger.getLogger(FileTail.class);
+	private String format;
+	private String directory;
+	private NewLineHandler handler;
+	private long checkFrequency = 60000; // milliseconds
+	private long checkNewFrequency = 300000; // milliseconds
+	private JournalingHashMap<Long> filePointerMap;
+	private long scanDuration = DateUtils.stringToDuration("2day");
+	private Object context; 
+
+	public interface NewLineHandler {
+		void newLine(String file, String line, Object context);
+		void fixFilePermissions(File file);
+	}
+
+	@SuppressWarnings("unchecked")
+	public FileTail(String format, String directory, String scanFreq, String scanNewFreq, String scanDuration, NewLineHandler handler, Object context) {
+		super();
+		if (scanFreq != null)
+			checkFrequency = DateUtils.stringToDuration(scanFreq);
+		if (scanNewFreq != null)
+			checkNewFrequency = DateUtils.stringToDuration(scanNewFreq);
+		if (scanDuration != null)
+			this.scanDuration  = DateUtils.stringToDuration(scanDuration);
+
+		this.format = format;
+		this.directory = directory;
+		this.handler = handler;
+		this.context = context;
+		filePointerMap = JournalingHashMap.create(new File(directory));
+		logger.info("initial status: " + getStatus().toString(2));
+		logger.info("Created: " + this);
+		new MonitorThread(format);
+	}
+	@SuppressWarnings("unchecked")
+	public FileTail(String filename, String directory, String scanFreq, NewLineHandler handler, Object context) {
+		if (scanFreq != null)
+			checkFrequency = DateUtils.stringToDuration(scanFreq);
+		this.directory = directory;
+		this.handler = handler;
+		this.context = context;
+		filePointerMap = JournalingHashMap.create(new File(directory));
+		if (filePointerMap.get(filename) == null) {
+			filePointerMap.put(filename, 0L);
+		}
+		new MonitorThread(filename);
+	}
+
+	@Override
+	public String toString() {
+		return "FileTail [format=" + format + ", directory=" + directory + ", checkFrequency="
+				+ checkFrequency + ", checkNewFrequency=" + checkNewFrequency + "]";
+	}
+
+	private class MonitorThread implements Runnable {
+		public MonitorThread(String name) {
+			Thread t = new Thread(this, "filetail: " + name);
+			t.start();
+		}
+
+		@Override
+		public void run() {
+			long numIntervalsPerNewFileCheck = checkNewFrequency / checkFrequency;
+			long i = 0;
+			while (true) {
+				if ((i++ % numIntervalsPerNewFileCheck) == 0)
+					scanForNewFiles();
+				scanForNewLines();
+				try {
+					Thread.sleep(checkFrequency);
+				} catch (InterruptedException e) {
+					Thread.currentThread().interrupt();
+					return;
+				}
+			}
+		}
+	}
+
+	private void scanForNewFiles() {
+		if (format == null) return;
+		FindFiles ff = new FindFiles(format);
+		Date now = new Date();
+		HashMap<String, Long> map = new HashMap<String, Long>();
+		synchronized (this) {
+			for (ParameterizedFile f : ff.findFiles()) {
+				File f1 = new File(f.getFile());
+				if (f1.lastModified() + scanDuration < now.getTime()) continue;
+				if (filePointerMap.get(f.getFile()) == null) {
+					map.put(f.getFile(), 0L);
+					filePointerMap.put(f.getFile(), 0L);
+					logger.info("Found new file: " + f.getFile());
+				} else {
+					map.put(f.getFile(), filePointerMap.get(f.getFile()));
+				}
+			}
+			List<String> remove = new ArrayList<String>();
+			for (String f : filePointerMap.keySet()) {
+				if (map.get(f) == null) {
+					logger.info("Deleted file: " + f);
+					remove.add(f);
+				}
+			}
+			for (String f : remove) filePointerMap.remove(f);
+			filePointerMap.save();
+		}
+	}
+
+	private void scanForNewLines() {
+		logger.info("old status: " + getStatus().toString(2));
+		Set<String> files = filePointerMap.keySet();
+		for (String f : files) {
+			try {
+				long p;
+				synchronized (this) {
+					p = filePointerMap.get(f);
+				}
+				File file = new File(f);
+				long len = file.length();
+				if (len == p)
+					continue;
+				if (len < p) {
+					logger.info("Logfile reset. Restarted at start");
+					p = 0;
+				}
+				if (len > p) {
+					if (!file.canRead()) {
+						handler.fixFilePermissions(file);
+					}
+					if (!file.canRead()) {
+						LoggingUtils.dampingLogger(logger, Level.WARN, "Unable to read: " + file.getAbsolutePath());
+						continue;
+					}
+					RandomAccessFile rf = new RandomAccessFile(file, "r");
+					rf.seek(p);
+					String line = null;
+					while ((line = rf.readLine()) != null) {
+						if (line.length() == 0)
+							continue;
+						if (logger.isDebugEnabled())
+							logger.debug("New line from file: " + f + " " + line);
+						p = rf.getFilePointer();
+						try {
+							handler.newLine(file.getAbsolutePath(),line,context);
+						} catch (Exception e) {
+							logger.warn("Handler error: " + f + " " + e + " line=" + line);
+							e.printStackTrace();
+						}
+					}
+					rf.close();
+				}
+				synchronized (this) {
+					filePointerMap.put(f, p);
+					filePointerMap.save();
+				}
+			} catch (Exception e) {
+				e.printStackTrace(System.out);
+				logger.error("Scaning error: " + f + " " + e);
+			}
+		}
+		logger.info("new status: " + getStatus().toString(2));
+	}
+
+	public JSONObject getStatus() {
+		JSONObject o = new JSONObject();
+		synchronized (this) {
+			for (String f : filePointerMap.keySet()) {
+				o.put(f, filePointerMap.get(f));
+			}
+		}
+		return o;
+	}
+
+}