diff options
Diffstat (limited to 'LogParser')
| -rw-r--r-- | LogParser/.gitignore | 2 | ||||
| -rw-r--r-- | LogParser/LineTest.txt | 9 | ||||
| -rw-r--r-- | LogParser/LineTest2.txt | 7 | ||||
| -rw-r--r-- | LogParser/parserlog.properties | 38 | ||||
| -rw-r--r-- | LogParser/policyLogger.properties | 44 | ||||
| -rw-r--r-- | LogParser/pom.xml | 113 | ||||
| -rw-r--r-- | LogParser/src/META-INF/MANIFEST.MF | 3 | ||||
| -rw-r--r-- | LogParser/src/main/java/org/openecomp/xacml/parser/LogEntryObject.java | 74 | ||||
| -rw-r--r-- | LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java | 437 | ||||
| -rw-r--r-- | LogParser/src/main/scripts/parserlog.sh | 130 | ||||
| -rw-r--r-- | LogParser/src/test/java/org/openecomp/xacml/parser/ParseLogTest.java | 382 | ||||
| -rw-r--r-- | LogParser/test_config.properties | 31 |
12 files changed, 1270 insertions, 0 deletions
diff --git a/LogParser/.gitignore b/LogParser/.gitignore new file mode 100644 index 000000000..09e3bc9b2 --- /dev/null +++ b/LogParser/.gitignore @@ -0,0 +1,2 @@ +/bin/ +/target/ diff --git a/LogParser/LineTest.txt b/LogParser/LineTest.txt new file mode 100644 index 000000000..449644219 --- /dev/null +++ b/LogParser/LineTest.txt @@ -0,0 +1,9 @@ +2016-04-26T00:00:00{GMT+0}+00:00|||Thread-5||||ERROR|||localhost.com||org.openecomp.policy.im.IntegrityMonitor.writeFpc(IntegrityMonitor.java:377)||ERROR_VALUE +2016-04-27T11:22:08{GMT+0}+00:00|||main||||INFO|||||org.openecomp.xacml.parser.ParseLog.main(ParseLog.java:86)||File Line Count: 18409 value read in: 0 +2016-04-27T11:22:48{GMT+0}+00:00|||Thread-0||||INFO|||||org.openecomp.xacml.parser.ParseLog$1.lambda$0(ParseLog.java:96)||Last line Read: 12 +2015_10_02_15_11_19_006 [http-nio-8081-exec-10] INFO corg.openecomp.research.xacml.rest.XACMLPdpServlet.doPutConfig(XACMLPdpServlet.java:409)- Success +2015-04-01 09:13:44.947 INFO 17482 --- [nio-8480-exec-7] c.a.l.ecomp.policy.std.StdPolicyConfig : config Retrieved +08-Apr-2015 10:31:26.503 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [PyPDPServer] appears to have started a thread named [Grizzly(1) SelectorRunner] but has failed to stop it. This is very likely to create a memory leak. service Catalina08-Apr-2015 10:31:26.503 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [PyPDPServer] appears to have started a thread named [Grizzly(1) SelectorRunner] but has failed to stop it. This is very likely to create a memory leak. +06-Mar-2015 11:50:06.243 SEVERE [main] org.apache.coyote.AbstractProtocol.init Failed to initialize end point associated with ProtocolHandler ["ajp-nio-8009"] java.net.BindException: Address already in use +2015_09_30_10_39_31_675 [http-nio-8081-exec-1] ERROR org.openecomp.research.xacml.rest.XACMLPdpServlet.doPost(XACMLPdpServlet.java:644)- PE100 - Permissions Error: PEP not Authorized for making this Request!! +2015_03_17_15_01_08_348 [qtp1688376486-32] WARN org.openecomp.research.xacml.admin.components.PolicyManagement$1.accept(PolicyManagement.java:184)- Filtering out: C:\git\D2PE-take2\policy-engine-prototype\XACML-PAP-ADMIN\workspace\admin\repository\com\.svnignore
\ No newline at end of file diff --git a/LogParser/LineTest2.txt b/LogParser/LineTest2.txt new file mode 100644 index 000000000..2243960df --- /dev/null +++ b/LogParser/LineTest2.txt @@ -0,0 +1,7 @@ +2016-04-26T00:00:00{GMT+0}+00:00|||Thread-5||||ERROR|||localhost.com||org.openecomp.ecomp.policy.im.IntegrityMonitor.writeFpc(IntegrityMonitor.java:377)||ERROR_VALUE +2015_10_02_15_11_19_006 [http-nio-8081-exec-10] INFO org.openecomp.research.xacml.rest.XACMLPdpServlet.doPutConfig(XACMLPdpServlet.java:409)- Success +2015-04-01 09:13:44.947 INFO 17482 --- [nio-8480-exec-7] c.a.l.ecomp.policy.std.StdPolicyConfig : config Retrieved +08-Apr-2015 10:31:26.503 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [PyPDPServer] appears to have started a thread named [Grizzly(1) SelectorRunner] but has failed to stop it. This is very likely to create a memory leak. service Catalina08-Apr-2015 10:31:26.503 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [PyPDPServer] appears to have started a thread named [Grizzly(1) SelectorRunner] but has failed to stop it. This is very likely to create a memory leak. +06-Mar-2015 11:50:06.243 SEVERE [main] org.apache.coyote.AbstractProtocol.init Failed to initialize end point associated with ProtocolHandler ["ajp-nio-8009"] java.net.BindException: Address already in use +2015_09_30_10_39_31_675 [http-nio-8081-exec-1] ERROR org.openecomp.research.xacml.rest.XACMLPdpServlet.doPost(XACMLPdpServlet.java:644)- PE100 - Permissions Error: PEP not Authorized for making this Request!! +2015_03_17_15_01_08_348 [qtp1688376486-32] WARN org.openecomp.research.xacml.admin.components.PolicyManagement$1.accept(PolicyManagement.java:184)- Filtering out: C:\git\D2PE-take2\policy-engine-prototype\XACML-PAP-ADMIN\workspace\admin\repository\com\.svnignore
\ No newline at end of file diff --git a/LogParser/parserlog.properties b/LogParser/parserlog.properties new file mode 100644 index 000000000..8b0ac753f --- /dev/null +++ b/LogParser/parserlog.properties @@ -0,0 +1,38 @@ +### +# ============LICENSE_START======================================================= +# LogParser +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + +#Health Check Values +RESOURCE_NAME=logparser_pap01 +javax.persistence.jdbc.driver=com.mysql.jdbc.Driver +javax.persistence.jdbc.url=jdbc:mysql://localhost:3306/xacml +javax.persistence.jdbc.user=policy_user +javax.persistence.jdbc.password=password + +#Log Parser application values +JDBC_DRIVER=com.mysql.jdbc.Driver +JDBC_URL=jdbc:mysql://localhost:3306/log +JDBC_USER=policy_user +JDBC_PASSWORD=password +SERVER=https://localhost:9091/pap/ +LOGTYPE=PAP +LOGPATH=C:\\Workspaces\\HealthCheck\\pap-rest.log +PARSERLOGPATH=parserlog.log +node_type=logparser +site_name=site_1 diff --git a/LogParser/policyLogger.properties b/LogParser/policyLogger.properties new file mode 100644 index 000000000..131267dcf --- /dev/null +++ b/LogParser/policyLogger.properties @@ -0,0 +1,44 @@ +### +# ============LICENSE_START======================================================= +# LogParser +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + +################################### Set concurrentHashMap and timer info ####################### +#Timer initial delay and the delay between in milliseconds before task is to be execute. +timer.delay.time=1000 +#Timer scheduleAtFixedRate period - time in milliseconds between successive task executions. +check.interval= 30000 +#Longest time an event info can be stored in the concurrentHashMap for logging - in seconds. +event.expired.time=86400 +#Size of the concurrentHashMap which stores the event starting time, etc - when its size reaches this limit, the Timer gets executed +#to remove all expired records from this concurrentHashMap. +concurrentHashMap.limit=5000 +#Size of the concurrentHashMap - when its size drops to this point, stop the Timer +stop.check.point=2500 +################################### Set logging format ############################################# +# set EELF for EELF logging format, set LOG4J for using log4j, set SYSTEMOUT for using system.out.println +logger.type=EELF +#################################### Set level for EELF or SYSTEMOUT logging ################################## +# Set level for debug file. Set DEBUG to enable .info, .warn and .debug; set INFO for enable .info and .warn; set OFF to disable all +debugLogger.level=INFO +# Set level for metrics file. Set OFF to disable; set ON to enable +metricsLogger.level=ON +# Set level for error file. Set OFF to disable; set ON to enable +error.level=ON +# Set level for audit file. Set OFF to disable; set ON to enable +audit.level=ON diff --git a/LogParser/pom.xml b/LogParser/pom.xml new file mode 100644 index 000000000..ae67d0d1a --- /dev/null +++ b/LogParser/pom.xml @@ -0,0 +1,113 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ============LICENSE_START======================================================= + ECOMP Policy Engine + ================================================================================ + Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + ================================================================================ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============LICENSE_END========================================================= + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.openecomp.policy.engine</groupId> + <artifactId>PolicyEngineSuite</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + <groupId>org.openecomp.policy.engine</groupId> + <artifactId>LogParser</artifactId> + + <dependencies> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <version>1.4.193</version> + </dependency> + <dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + <version>5.1.30</version> + </dependency> + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <version>1.2.3</version> + </dependency> + <dependency> + <groupId>org.openecomp.policy.common</groupId> + <artifactId>integrity-monitor</artifactId> + <version>${common-modules.version}</version> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.17</version> + </dependency> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>javax.persistence</artifactId> + <version>2.1.0</version> + </dependency> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>eclipselink</artifactId> + <version>2.6.0</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>1.9.5</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <!-- Build an executable JAR --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.2</version> + <configuration> + <archive> + <manifest> + <addClasspath>true</addClasspath> + <classpathPrefix>lib/</classpathPrefix> + <mainClass>org.openecomp.xacml.parser.ParseLog</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/LogParser/src/META-INF/MANIFEST.MF b/LogParser/src/META-INF/MANIFEST.MF new file mode 100644 index 000000000..254272e1c --- /dev/null +++ b/LogParser/src/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Class-Path: + diff --git a/LogParser/src/main/java/org/openecomp/xacml/parser/LogEntryObject.java b/LogParser/src/main/java/org/openecomp/xacml/parser/LogEntryObject.java new file mode 100644 index 000000000..4b03d2aaa --- /dev/null +++ b/LogParser/src/main/java/org/openecomp/xacml/parser/LogEntryObject.java @@ -0,0 +1,74 @@ +/*- + * ============LICENSE_START======================================================= + * LogParser + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.xacml.parser; + +import java.util.Date; + +public class LogEntryObject { + + private String system; + private String description; + private Date date; + private String remote; + private String systemType; + private LOGTYPE logType; + + public enum LOGTYPE { + INFO, ERROR, SEVERE, WARN; + } + + public String getSystem() { + return system; + } + public void setSystem(String system) { + this.system = system; + } + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + public Date getDate() { + return date; + } + public void setDate(Date date) { + this.date = date; + } + public String getRemote() { + return remote; + } + public void setRemote(String remote) { + this.remote = remote; + } + public String getSystemType() { + return systemType; + } + public void setSystemType(String systemType) { + this.systemType = systemType; + } + public LOGTYPE getLogType() { + return logType; + } + public void setLogType(LOGTYPE logType) { + this.logType = logType; + } +} diff --git a/LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java b/LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java new file mode 100644 index 000000000..5303bacc2 --- /dev/null +++ b/LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java @@ -0,0 +1,437 @@ +/*- + * ============LICENSE_START======================================================= + * LogParser + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.xacml.parser; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.LineNumberReader; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.Format; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.openecomp.policy.common.im.AdministrativeStateException; +import org.openecomp.policy.common.im.IntegrityMonitor; +import org.openecomp.policy.common.im.StandbyStatusException; + +import org.apache.log4j.Logger; +import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE; + +/** + * Parse log files and store the information in a H2 database. + * + * + */ +public class ParseLog { + + private static final Logger logger = Logger.getLogger(ParseLog.class.getName()); + + private static String system; + private static int lastNumberRead = 0; + private static String type; + private static long startFileSize; + private static String systemLogFile; + private static String logFile; + + private static String JDBC_URL; + private static String JDBC_USER; + private static String JDBC_PASSWORD = ""; + private static String JDBC_DRIVER; + private static int maxLength = 255; //Max length that is allowed in the DB table + private static String resourceName; + private static long sleepTimer = 50000; + static IntegrityMonitor im; + + public static void main(String[] args) throws Exception { + + Properties logProperties = getPropertiesValue("parserlog.properties"); + Path filePath = Paths.get(logFile); + File file = new File(logFile); + File fileLog = new File(systemLogFile); + startFileSize = file.length(); + + im = IntegrityMonitor.getInstance(resourceName,logProperties ); + + logger.info("System: " + system ); + logger.info("System type: " + type ); + logger.info("Logging File: " + systemLogFile ); + logger.info("log file: " + logFile); + logger.info("JDBC_URL: " + JDBC_URL); + logger.info("JDBC_DRIVER: " + JDBC_DRIVER); + + String filesRead = PullLastLineRead(fileLog); + if (filesRead!= null){ + filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />"); + lastNumberRead= Integer.parseInt(filesRead.trim()); + }else{ + lastNumberRead = 0; + } + startFileSize = countLines(logFile); + logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead); + if (startFileSize < lastNumberRead ){ + logger.error("Filed Rolled: set Last number read to 0"); + lastNumberRead = 0; + } + Runnable runnable = new Runnable (){ + public void run(){ + while (true){ + + if (file.isFile()){ + try (Stream<String> lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> logger.info("Last line Read: " + lastNumberRead)).skip(lastNumberRead)) { + + lines.forEachOrdered(line -> process(line, type)); + + } catch (IOException e) { + logger.error("Error processing line in log file: " + e); + } + } + try { + Thread.sleep(sleepTimer); + startFileSize = countLines(logFile); + } catch (InterruptedException | IOException e) { + logger.error("Error: " + e); + } + + logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead); + if (startFileSize < lastNumberRead ){ + logger.info("Failed Rolled: set Last number read to 0"); + lastNumberRead = 0; + } + } + } + }; + + Thread thread = new Thread(runnable); + thread.start(); + + } + + public static int countLines(String filename) throws IOException { + LineNumberReader reader = new LineNumberReader(new FileReader(filename)); + int cnt = 0; + String lineRead = ""; + while ((lineRead = reader.readLine()) != null) {} + + cnt = reader.getLineNumber(); + reader.close(); + return cnt; + } + + public static String PullLastLineRead(File file) throws IOException { + if(!file.exists()){ + file.createNewFile(); + return null; + } + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r"); + int lines = 0; + StringBuilder builder = new StringBuilder(); + long length = file.length(); + length--; + randomAccessFile.seek(length); + for(long seek = length; seek >= 0; --seek){ + randomAccessFile.seek(seek); + char c = (char)randomAccessFile.read(); + builder.append(c); + if(c == '\n'){ + builder = builder.reverse(); + if (builder.toString().contains("Last line Read:")){ + String[] parseString = builder.toString().split("Last line Read:"); + String returnValue = parseString[1].replace("\r", ""); + return returnValue.trim(); + //return parseString[2].replace("\r", ""); + } + lines++; + builder = null; + builder = new StringBuilder(); + } + + } + return null; + } + + public static LogEntryObject pullOutLogValues(String line, String type){ + Date date; + LogEntryObject logEntry = new LogEntryObject(); + logEntry.setSystemType(type); + String description = null; + + logEntry.setSystem(system); + + //Values for PDP/PAP log file + if(line.contains("||INFO||") || line.contains("||ERROR||")){ + String[] splitString = line.split("[||]"); + String dateString = splitString[0].substring(0, 19); + logEntry.setDescription(splitString[splitString.length-1]); + + //parse out date + date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false); + logEntry.setDate(date); + + logEntry.setRemote(parseRemoteSystem(line)); + if (line.contains("||INFO||")){ + logEntry.setLogType(LOGTYPE.INFO); + }else{ + logEntry.setLogType(LOGTYPE.ERROR); + } + }else if (line.contains("INFO") && line.contains(")-")){ + //parse out description + logEntry.setDescription(line.substring(line.indexOf(")-")+3)); + + date = parseDate(line, "yy_MM_dd_HH_mm_ss", true); + logEntry.setDate(date); + + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.INFO); + } else if (line.contains("INFO") && line.contains("--- [")){ + //parse out description + String temp = line.substring(line.indexOf("---")+1); + String[] split = temp.split(":"); + + logEntry.setDescription(split[1]); + + //parse out date + date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false); + logEntry.setDate(date); + + //remote system + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.INFO); + }else if (line.contains("SEVERE") && line.contains("[main]")){ + String[] splitString = line.split(" "); + + for (int i = 5; i < splitString.length; i++){ + description = description + " " + splitString[i]; + } + + logEntry.setDescription(description); + //parse out date + date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false); + logEntry.setDate(date); + logEntry.setLogType(LOGTYPE.SEVERE); + } else if (line.contains("WARN") && line.contains(")-")){ + //parse out description + + logEntry.setDescription(line.substring(line.indexOf(")-")+3)); + + //parse out date + date = parseDate(line, "yy_MM_dd_HH_mm_ss", true); + logEntry.setDate(date); + + //remote system + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.WARN); + }else if (line.contains("WARNING") && type =="PyPDP"){ + String[] splitString = line.split(" "); + for (int i = 5; i < splitString.length; i++){ + description = description + " " + splitString[i]; + } + + //parse out date + date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false); + logEntry.setDate(date); + logEntry.setLogType(LOGTYPE.WARN); + }else if (line.contains("ERROR") && line.contains(")-")){ + //parse out description + description = line.substring(line.indexOf(")-")+3); + + //parse out date + date = parseDate(line, "yy_MM_dd_HH_mm_ss", true); + logEntry.setDate(date); + //remote system + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.ERROR); + }else { + return null; + } + + + return logEntry; + } + + private static void DBClose(Connection conn) { + try { + conn.close(); + } catch (SQLException e) { + logger.error("Error closing DB Connection: " + e); + + } + } + + public static void process(String line, String type) { + LogEntryObject returnLogValue = null; + if (im!=null){ + try { + im.startTransaction(); + } catch (AdministrativeStateException e) { + logger.error("Error received" + e); + + } catch (StandbyStatusException e) { + logger.error("Error received" + e); + } + } + returnLogValue = pullOutLogValues(line, type); + lastNumberRead++; + if (returnLogValue!=null){ + writeDB(returnLogValue); + } + if (im!=null){ + im.endTransaction(); + } + } + + private static void writeDB(LogEntryObject returnLogValue) { + Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD); + DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(), + returnLogValue.getDate(), returnLogValue.getRemote(), + returnLogValue.getSystemType(), returnLogValue.getLogType().toString()); + DBClose(conn); + } + + private static Connection DBConnection(String driver, String jdbc, String user, String pass){ + + try { + Class.forName(driver); + Connection conn = DriverManager.getConnection(jdbc, user, pass); + return conn; + } catch ( Exception e) { + logger.error("Error connecting to DB: " + e); + } + return null; + } + private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) { + + String sdate = null; + + if (date!=null){ + Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + sdate = formatter.format(date); + } + + //ensure the length of description is less than the maximumm db char length + if (description.length() > maxLength) { + description = description.substring(0, maxLength); + } + + try { + PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);"); + prep.setString(1, system); + prep.setString(2, description); + prep.setString(3, remote); + prep.setString(4, type); + prep.setString(5, sdate); + prep.setString(6, logType); + + prep.executeUpdate(); + prep.close(); + + } catch (SQLException e1) { + logger.error("Error trying to excute SQL Statment: " + e1); + } + } + + public static Date parseDate(String dateline, String pattern, boolean singleSplit) { + + Date returnDate; + String[] splitString = dateline.split(" "); + SimpleDateFormat formatter = new SimpleDateFormat(pattern); + if (singleSplit){ + try { + returnDate = formatter.parse(splitString[0]); + } catch (ParseException e) { + logger.error("Unable to parse date for line: " + dateline); + returnDate = null; + } + }else{ + String tmpString = splitString[0] + " " + splitString[1]; + try { + returnDate = formatter.parse(tmpString); + } catch (ParseException e) { + logger.error("Unable to parse date for line: " + dateline); + returnDate = null; + } + } + + return returnDate; + } + + + public static String parseRemoteSystem(String line) { + + if (line.contains("http") && !(line.contains("www.w3.org"))){ + + Pattern pattern = Pattern.compile("://(.+?)/"); + Matcher remote = pattern.matcher(line); + if (remote.find()) + { + return remote.group(1); + } + } + return null; + } + + public static Properties getPropertiesValue(String fileName) { + Properties config = new Properties(); + Path file = Paths.get(fileName); + if (Files.notExists(file)) { + logger.info("File doesn't exist in the specified Path " + file.toString()); + }else{ + if (file.toString().endsWith(".properties")) { + InputStream in; + try { + in = new FileInputStream(file.toFile()); + config.load(in); + + resourceName = config.getProperty("RESOURCE_NAME"); + system = config.getProperty("SERVER"); + type = config.getProperty("LOGTYPE"); + systemLogFile = config.getProperty("PARSERLOGPATH"); + logFile = config.getProperty("LOGPATH"); + JDBC_URL = config.getProperty("JDBC_URL").replace("'", ""); + JDBC_USER = config.getProperty("JDBC_USER"); + JDBC_DRIVER = config.getProperty("JDBC_DRIVER"); + JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD"); + return config; + + } catch (IOException e) { + logger.info("Error porcessing Cofnig file will be unable to create Health Check"); + } + + } + } + return null; + } +} diff --git a/LogParser/src/main/scripts/parserlog.sh b/LogParser/src/main/scripts/parserlog.sh new file mode 100644 index 000000000..79ea8a3b3 --- /dev/null +++ b/LogParser/src/main/scripts/parserlog.sh @@ -0,0 +1,130 @@ +### +# ============LICENSE_START======================================================= +# LogParser +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + +#!/bin/sh +# +# init script for a Java application +# + +#Arguments for application +SERVER=http://localhost:8070/pap/ +LOGTYPE=PAP +LOGPATH="/var/lib/servers/pap/logs/catalina.out" +PARSERLOGPATH="$POLICY_HOME/logs/parserlog.log" +JDBC_URL="jdbc:h2:tcp://localhost:9092/log" +JDBC_USER="sa" +JDBC_DRIVER="org.h2.Driver" +JDBC_PASSWORD="" +SERVICE="LogParser.jar" + +# Check the application status +# +# This function checks if the application is running +check_status() { + + # Running pgrep with some arguments to check if the PID exists + if pgrep -f "$SERVICE $SERVER $LOGTYPE" ; then + RESULT=$(pgrep -f ${SERVICE}) + return $RESULT + fi + return 0 + # In any another case, return 0 + +} + +# Starts the application +start() { + + # At first checks if the application is already started calling the check_status + # function + check_status + + # $? is a special variable that hold the "exit status of the most recently executed + # foreground pipeline" + pid=$? + + if [ $pid -ne 0 ] ; then + echo "The application is already started" + exit 1 + fi + + # If the application isn't running, starts it + echo -n "Starting application: " + + # Redirects default and error output to a log file + java -jar LogParser.jar $SERVER $LOGTYPE $LOGPATH $PARSERLOGPATH $JDBC_URL $JDBC_USER $JDBC_DRIVER $JDBC_PASSWORD>> $POLICY_HOME/logs/parserlog.log 2>&1 & + echo "OK" +} + +# Stops the application +stop() { + + # Like as the start function, checks the application status + check_status + + pid=$? + + if [ $pid -eq 0 ] ; then + echo "Application is already stopped" + exit 1 + fi + + # Kills the application process + echo -n "Stopping application: " + kill -9 $pid & + echo "OK" +} + +# Show the application status +status() { + + # The check_status function, again... + check_status + + # If the PID was returned means the application is running + if [ $? -ne 0 ] ; then + echo "Application is started" + else + echo "Application is stopped" + fi + +} + +# Main logic, a simple case to call functions +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status + ;; + restart|reload) + stop + start + ;; + *) + echo "Usage: $0 {start|stop|restart|reload|status}" + exit 1 +esac + +exit 0 diff --git a/LogParser/src/test/java/org/openecomp/xacml/parser/ParseLogTest.java b/LogParser/src/test/java/org/openecomp/xacml/parser/ParseLogTest.java new file mode 100644 index 000000000..08df57a9b --- /dev/null +++ b/LogParser/src/test/java/org/openecomp/xacml/parser/ParseLogTest.java @@ -0,0 +1,382 @@ +/*- + * ============LICENSE_START======================================================= + * LogParser + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.xacml.parser; + +import static org.junit.Assert.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.Properties; + +import org.openecomp.policy.common.im.AdministrativeStateException; +import org.openecomp.policy.common.im.IntegrityMonitor; +import org.openecomp.policy.common.im.StandbyStatusException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.openecomp.xacml.parser.LogEntryObject; +import org.openecomp.xacml.parser.ParseLog; +import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE; + + +public class ParseLogTest { + + private ParseLog logParser = new ParseLog(); + private Properties config = new Properties(); + private String configFile = "test_config.properties"; + private static Properties myProp; + private static Properties systemProps; + private static String resourceName; + private IntegrityMonitor im; + + + @Before + public void setUp() throws Exception { + System.setProperty("com.sun.management.jmxremote.port", "9998"); + im = Mockito.mock(IntegrityMonitor.class); + + // Need PowerMockito for mocking static method getInstance(...) + // PowerMockito.mockStatic(IntegrityMonitor.class); + + try { + Mockito.doNothing().when(im).startTransaction(); + } catch (StandbyStatusException | AdministrativeStateException e) { + fail(); + } + Mockito.doNothing().when(im).endTransaction(); + + } + + @After + public void tearDown() { + File file = new File("nonExistFile.txt"); + file.delete(); +// systemProps.remove("com.sun.management.jmxremote.port"); + } + + @Test + public void testMain() { + try { + + logParser.main(null); + } catch (Exception e) { + //fail(); + } + } + + @Test + public void testCountLines() throws IOException { + String fileName = "LineTest.txt"; + int returnValue = logParser.countLines(fileName); + + assertEquals(9, returnValue); + } + + @Test + public void testParseRemoteSystem() { + String line = "||org.openecomp.policy.pap.xacml.rest.XACMLPapServlet$Heartbeat.run(XACMLPapServlet.java:2801)||Heartbeat 'https://localhost:8081/pdp/' status='UP_TO_DATE'"; + String returnValue = ParseLog.parseRemoteSystem(line); + assertEquals("localhost:8081", returnValue); + } + + @Test + public void testGetPropertiesValue() { + config = new Properties(); + config.put("RESOURCE_NAME", "logparser_pap01"); + config.put("JDBC_DRIVER" ,"com.mysql.jdbc.Driver"); + config.put("JDBC_URL", "jdbc:mysql://localhost:3306/"); + config.put("JDBC_USER", "root"); + config.put("JDBC_PASSWORD", "password"); + config.put("JMX_URL", "service:jmx:rmi:///jndi/rmi://localhost:9998/jmxrmi"); + config.put("SERVER", "password"); + config.put("JDBC_PASSWORD", "https://localhost:9091/pap/"); + config.put("LOGTYPE", "PAP"); + config.put("LOGPATH", "C:\\Workspaces\\HealthCheck\\pap-rest.log"); + config.put("PARSERLOGPATH", "IntegrityMonitor.log"); + + Properties returnConfig = logParser.getPropertiesValue(configFile); + + + assertEquals(config.get("RESOURCE_NAME"), returnConfig.get("RESOURCE_NAME")); + } + + @Test + public void testGetPropertiesFail() { + Properties returnValue = ParseLog.getPropertiesValue("nonExistFile"); + + assertEquals(null, returnValue); + } + + @Test + public void testParseDate(){ + String line = "2016-02-23 08:07:30"; + Date returnValue = ParseLog.parseDate(line, "yyyy-MM-dd HH:mm:ss", false); + + assertEquals("Tue Feb 23 08:07:30 CST 2016", returnValue.toString()); + } + + @Test + public void testParseDateFail(){ + String line = "2016-02-23 08:07:30"; + Date returnValue = ParseLog.parseDate(line, "yyyy-MM-dd HH:mm:ss", true); + + assertEquals(null, returnValue); + } + + @Test + public void testPullLastLineRead(){ + + File file = new File("LineTest.txt"); + String returnValue = null; + try { + returnValue = ParseLog.PullLastLineRead(file).trim(); + } catch (IOException e) { + fail(); + } + assertEquals("12", returnValue); + + } + + @Test + public void testPullLastLineReadNoFile(){ + + File file = new File("nonExistFile.txt"); + try { + assertEquals(null, ParseLog.PullLastLineRead(file)); + } catch (IOException e) { + fail(); + } + } + @Test + public void testPullLastLineReadFail(){ + + File file = new File("LineTest2.txt"); + try { + assertEquals(null, ParseLog.PullLastLineRead(file)); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValues(){ + //ERROR_VALUE + // Open the file + FileInputStream fstream; + try { + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "ERROR"); + assertEquals("ERROR_VALUE", retrunObject.getDescription()); + br.close(); + } catch (IOException e) { + fail(); + } +// assert(true); + } + @Test + public void testPullOutLogValuesSecond(){ + //ERROR_VALUE + // Open the file + FileInputStream fstream; + try { + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + strLine = br.readLine(); + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "INFO"); + assertEquals(LOGTYPE.INFO, retrunObject.getLogType()); + br.close(); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValuesThird(){ + //ERROR_VALUE + // Open the file + FileInputStream fstream; + try { + int number = 3; + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + for (int i =0; i < number; i++){ + strLine = br.readLine(); + } + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "PAP"); + assertEquals(LOGTYPE.INFO, retrunObject.getLogType()); + br.close(); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValuesFourth(){ + // Open the file + FileInputStream fstream; + try { + int number = 4; + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + for (int i =0; i < number; i++){ + strLine = br.readLine(); + } + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "PAP"); + assertEquals(LOGTYPE.INFO, retrunObject.getLogType()); + br.close(); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValuesFith(){ + // Open the file + FileInputStream fstream; + try { + int number = 5; + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + for (int i =0; i < number; i++){ + strLine = br.readLine(); + } + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "PyPDP"); + assertEquals(LOGTYPE.WARN, retrunObject.getLogType()); + br.close(); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValuesSixth(){ + // Open the file + FileInputStream fstream; + try { + int number = 6; + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + for (int i =0; i < number; i++){ + strLine = br.readLine(); + } + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "PyPDP"); + assertEquals(LOGTYPE.SEVERE, retrunObject.getLogType()); + br.close(); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValuesSeven(){ + // Open the file + FileInputStream fstream; + try { + int number = 7; + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + for (int i =0; i < number; i++){ + strLine = br.readLine(); + } + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "Console"); + assertEquals(LOGTYPE.ERROR, retrunObject.getLogType()); + br.close(); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValuesEight(){ + // Open the file + FileInputStream fstream; + try { + int number = 8; + fstream = new FileInputStream("LineTest.txt"); + BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + String strLine = br.readLine(); + for (int i =0; i < number; i++){ + strLine = br.readLine(); + } + LogEntryObject retrunObject = ParseLog.pullOutLogValues(strLine, "pap"); + assertEquals(LOGTYPE.WARN, retrunObject.getLogType()); + br.close(); + } catch (IOException e) { + fail(); + } + } + + @Test + public void testPullOutLogValuesNull(){ + // Open the file + FileInputStream fstream; + LogEntryObject retrunObject = ParseLog.pullOutLogValues("", "Console"); + assertEquals(null, retrunObject); + } + + @Test + public void testLogEntryObject(){ + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + Date date = new Date(); + + // Open the file + LogEntryObject logObject = new LogEntryObject(); + logObject.setSystem("vm02"); + logObject.setSystemType("pap"); + logObject.setDate(date); + logObject.setRemote("remote"); + + assertEquals("vm02", logObject.getSystem()); + assertEquals("pap", logObject.getSystemType()); + assertEquals(date, logObject.getDate()); + assertEquals("remote", logObject.getRemote()); + } + + @Test + public void testProcess(){ + String line = "2015-04-01 09:13:44.947 DEBUG 17482 --- [nio-8480-exec-7] c.a.l.ecomp.policy.std.StdPolicyConfig : config Retrieved "; + + im = Mockito.mock(IntegrityMonitor.class); + try { + Mockito.doNothing().when(im).startTransaction(); + } catch (StandbyStatusException | AdministrativeStateException e) { + fail(); + } + Mockito.doNothing().when(im).endTransaction(); + ParseLog.process(line, "pap"); + } +} diff --git a/LogParser/test_config.properties b/LogParser/test_config.properties new file mode 100644 index 000000000..5a6a460ba --- /dev/null +++ b/LogParser/test_config.properties @@ -0,0 +1,31 @@ +### +# ============LICENSE_START======================================================= +# LogParser +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + + +RESOURCE_NAME=logparser_pap01 +JDBC_DRIVER=com.mysql.jdbc.Driver +JDBC_URL=jdbc:mysql://localhost:3306/ +JDBC_USER=root +JDBC_PASSWORD=password +jmx_url=service:jmx:rmi:///jndi/rmi://localhost:9996/jmxrmi +SERVER=https://localhost:9091/pap/ +LOGTYPE=PAP +LOGPATH=C:\\Workspaces\\HealthCheck\\pap-rest.log +PARSERLOGPATH=IntegrityMonitor.log |