diff options
Diffstat (limited to 'LogParser/src/main')
3 files changed, 641 insertions, 0 deletions
diff --git a/LogParser/src/main/java/org/openecomp/xacml/parser/LogEntryObject.java b/LogParser/src/main/java/org/openecomp/xacml/parser/LogEntryObject.java new file mode 100644 index 000000000..4b03d2aaa --- /dev/null +++ b/LogParser/src/main/java/org/openecomp/xacml/parser/LogEntryObject.java @@ -0,0 +1,74 @@ +/*- + * ============LICENSE_START======================================================= + * LogParser + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.xacml.parser; + +import java.util.Date; + +public class LogEntryObject { + + private String system; + private String description; + private Date date; + private String remote; + private String systemType; + private LOGTYPE logType; + + public enum LOGTYPE { + INFO, ERROR, SEVERE, WARN; + } + + public String getSystem() { + return system; + } + public void setSystem(String system) { + this.system = system; + } + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + public Date getDate() { + return date; + } + public void setDate(Date date) { + this.date = date; + } + public String getRemote() { + return remote; + } + public void setRemote(String remote) { + this.remote = remote; + } + public String getSystemType() { + return systemType; + } + public void setSystemType(String systemType) { + this.systemType = systemType; + } + public LOGTYPE getLogType() { + return logType; + } + public void setLogType(LOGTYPE logType) { + this.logType = logType; + } +} diff --git a/LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java b/LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java new file mode 100644 index 000000000..5303bacc2 --- /dev/null +++ b/LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java @@ -0,0 +1,437 @@ +/*- + * ============LICENSE_START======================================================= + * LogParser + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.xacml.parser; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.LineNumberReader; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.text.Format; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.openecomp.policy.common.im.AdministrativeStateException; +import org.openecomp.policy.common.im.IntegrityMonitor; +import org.openecomp.policy.common.im.StandbyStatusException; + +import org.apache.log4j.Logger; +import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE; + +/** + * Parse log files and store the information in a H2 database. + * + * + */ +public class ParseLog { + + private static final Logger logger = Logger.getLogger(ParseLog.class.getName()); + + private static String system; + private static int lastNumberRead = 0; + private static String type; + private static long startFileSize; + private static String systemLogFile; + private static String logFile; + + private static String JDBC_URL; + private static String JDBC_USER; + private static String JDBC_PASSWORD = ""; + private static String JDBC_DRIVER; + private static int maxLength = 255; //Max length that is allowed in the DB table + private static String resourceName; + private static long sleepTimer = 50000; + static IntegrityMonitor im; + + public static void main(String[] args) throws Exception { + + Properties logProperties = getPropertiesValue("parserlog.properties"); + Path filePath = Paths.get(logFile); + File file = new File(logFile); + File fileLog = new File(systemLogFile); + startFileSize = file.length(); + + im = IntegrityMonitor.getInstance(resourceName,logProperties ); + + logger.info("System: " + system ); + logger.info("System type: " + type ); + logger.info("Logging File: " + systemLogFile ); + logger.info("log file: " + logFile); + logger.info("JDBC_URL: " + JDBC_URL); + logger.info("JDBC_DRIVER: " + JDBC_DRIVER); + + String filesRead = PullLastLineRead(fileLog); + if (filesRead!= null){ + filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />"); + lastNumberRead= Integer.parseInt(filesRead.trim()); + }else{ + lastNumberRead = 0; + } + startFileSize = countLines(logFile); + logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead); + if (startFileSize < lastNumberRead ){ + logger.error("Filed Rolled: set Last number read to 0"); + lastNumberRead = 0; + } + Runnable runnable = new Runnable (){ + public void run(){ + while (true){ + + if (file.isFile()){ + try (Stream<String> lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> logger.info("Last line Read: " + lastNumberRead)).skip(lastNumberRead)) { + + lines.forEachOrdered(line -> process(line, type)); + + } catch (IOException e) { + logger.error("Error processing line in log file: " + e); + } + } + try { + Thread.sleep(sleepTimer); + startFileSize = countLines(logFile); + } catch (InterruptedException | IOException e) { + logger.error("Error: " + e); + } + + logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead); + if (startFileSize < lastNumberRead ){ + logger.info("Failed Rolled: set Last number read to 0"); + lastNumberRead = 0; + } + } + } + }; + + Thread thread = new Thread(runnable); + thread.start(); + + } + + public static int countLines(String filename) throws IOException { + LineNumberReader reader = new LineNumberReader(new FileReader(filename)); + int cnt = 0; + String lineRead = ""; + while ((lineRead = reader.readLine()) != null) {} + + cnt = reader.getLineNumber(); + reader.close(); + return cnt; + } + + public static String PullLastLineRead(File file) throws IOException { + if(!file.exists()){ + file.createNewFile(); + return null; + } + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r"); + int lines = 0; + StringBuilder builder = new StringBuilder(); + long length = file.length(); + length--; + randomAccessFile.seek(length); + for(long seek = length; seek >= 0; --seek){ + randomAccessFile.seek(seek); + char c = (char)randomAccessFile.read(); + builder.append(c); + if(c == '\n'){ + builder = builder.reverse(); + if (builder.toString().contains("Last line Read:")){ + String[] parseString = builder.toString().split("Last line Read:"); + String returnValue = parseString[1].replace("\r", ""); + return returnValue.trim(); + //return parseString[2].replace("\r", ""); + } + lines++; + builder = null; + builder = new StringBuilder(); + } + + } + return null; + } + + public static LogEntryObject pullOutLogValues(String line, String type){ + Date date; + LogEntryObject logEntry = new LogEntryObject(); + logEntry.setSystemType(type); + String description = null; + + logEntry.setSystem(system); + + //Values for PDP/PAP log file + if(line.contains("||INFO||") || line.contains("||ERROR||")){ + String[] splitString = line.split("[||]"); + String dateString = splitString[0].substring(0, 19); + logEntry.setDescription(splitString[splitString.length-1]); + + //parse out date + date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false); + logEntry.setDate(date); + + logEntry.setRemote(parseRemoteSystem(line)); + if (line.contains("||INFO||")){ + logEntry.setLogType(LOGTYPE.INFO); + }else{ + logEntry.setLogType(LOGTYPE.ERROR); + } + }else if (line.contains("INFO") && line.contains(")-")){ + //parse out description + logEntry.setDescription(line.substring(line.indexOf(")-")+3)); + + date = parseDate(line, "yy_MM_dd_HH_mm_ss", true); + logEntry.setDate(date); + + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.INFO); + } else if (line.contains("INFO") && line.contains("--- [")){ + //parse out description + String temp = line.substring(line.indexOf("---")+1); + String[] split = temp.split(":"); + + logEntry.setDescription(split[1]); + + //parse out date + date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false); + logEntry.setDate(date); + + //remote system + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.INFO); + }else if (line.contains("SEVERE") && line.contains("[main]")){ + String[] splitString = line.split(" "); + + for (int i = 5; i < splitString.length; i++){ + description = description + " " + splitString[i]; + } + + logEntry.setDescription(description); + //parse out date + date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false); + logEntry.setDate(date); + logEntry.setLogType(LOGTYPE.SEVERE); + } else if (line.contains("WARN") && line.contains(")-")){ + //parse out description + + logEntry.setDescription(line.substring(line.indexOf(")-")+3)); + + //parse out date + date = parseDate(line, "yy_MM_dd_HH_mm_ss", true); + logEntry.setDate(date); + + //remote system + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.WARN); + }else if (line.contains("WARNING") && type =="PyPDP"){ + String[] splitString = line.split(" "); + for (int i = 5; i < splitString.length; i++){ + description = description + " " + splitString[i]; + } + + //parse out date + date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false); + logEntry.setDate(date); + logEntry.setLogType(LOGTYPE.WARN); + }else if (line.contains("ERROR") && line.contains(")-")){ + //parse out description + description = line.substring(line.indexOf(")-")+3); + + //parse out date + date = parseDate(line, "yy_MM_dd_HH_mm_ss", true); + logEntry.setDate(date); + //remote system + logEntry.setRemote(parseRemoteSystem(line)); + logEntry.setLogType(LOGTYPE.ERROR); + }else { + return null; + } + + + return logEntry; + } + + private static void DBClose(Connection conn) { + try { + conn.close(); + } catch (SQLException e) { + logger.error("Error closing DB Connection: " + e); + + } + } + + public static void process(String line, String type) { + LogEntryObject returnLogValue = null; + if (im!=null){ + try { + im.startTransaction(); + } catch (AdministrativeStateException e) { + logger.error("Error received" + e); + + } catch (StandbyStatusException e) { + logger.error("Error received" + e); + } + } + returnLogValue = pullOutLogValues(line, type); + lastNumberRead++; + if (returnLogValue!=null){ + writeDB(returnLogValue); + } + if (im!=null){ + im.endTransaction(); + } + } + + private static void writeDB(LogEntryObject returnLogValue) { + Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD); + DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(), + returnLogValue.getDate(), returnLogValue.getRemote(), + returnLogValue.getSystemType(), returnLogValue.getLogType().toString()); + DBClose(conn); + } + + private static Connection DBConnection(String driver, String jdbc, String user, String pass){ + + try { + Class.forName(driver); + Connection conn = DriverManager.getConnection(jdbc, user, pass); + return conn; + } catch ( Exception e) { + logger.error("Error connecting to DB: " + e); + } + return null; + } + private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) { + + String sdate = null; + + if (date!=null){ + Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + sdate = formatter.format(date); + } + + //ensure the length of description is less than the maximumm db char length + if (description.length() > maxLength) { + description = description.substring(0, maxLength); + } + + try { + PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);"); + prep.setString(1, system); + prep.setString(2, description); + prep.setString(3, remote); + prep.setString(4, type); + prep.setString(5, sdate); + prep.setString(6, logType); + + prep.executeUpdate(); + prep.close(); + + } catch (SQLException e1) { + logger.error("Error trying to excute SQL Statment: " + e1); + } + } + + public static Date parseDate(String dateline, String pattern, boolean singleSplit) { + + Date returnDate; + String[] splitString = dateline.split(" "); + SimpleDateFormat formatter = new SimpleDateFormat(pattern); + if (singleSplit){ + try { + returnDate = formatter.parse(splitString[0]); + } catch (ParseException e) { + logger.error("Unable to parse date for line: " + dateline); + returnDate = null; + } + }else{ + String tmpString = splitString[0] + " " + splitString[1]; + try { + returnDate = formatter.parse(tmpString); + } catch (ParseException e) { + logger.error("Unable to parse date for line: " + dateline); + returnDate = null; + } + } + + return returnDate; + } + + + public static String parseRemoteSystem(String line) { + + if (line.contains("http") && !(line.contains("www.w3.org"))){ + + Pattern pattern = Pattern.compile("://(.+?)/"); + Matcher remote = pattern.matcher(line); + if (remote.find()) + { + return remote.group(1); + } + } + return null; + } + + public static Properties getPropertiesValue(String fileName) { + Properties config = new Properties(); + Path file = Paths.get(fileName); + if (Files.notExists(file)) { + logger.info("File doesn't exist in the specified Path " + file.toString()); + }else{ + if (file.toString().endsWith(".properties")) { + InputStream in; + try { + in = new FileInputStream(file.toFile()); + config.load(in); + + resourceName = config.getProperty("RESOURCE_NAME"); + system = config.getProperty("SERVER"); + type = config.getProperty("LOGTYPE"); + systemLogFile = config.getProperty("PARSERLOGPATH"); + logFile = config.getProperty("LOGPATH"); + JDBC_URL = config.getProperty("JDBC_URL").replace("'", ""); + JDBC_USER = config.getProperty("JDBC_USER"); + JDBC_DRIVER = config.getProperty("JDBC_DRIVER"); + JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD"); + return config; + + } catch (IOException e) { + logger.info("Error porcessing Cofnig file will be unable to create Health Check"); + } + + } + } + return null; + } +} diff --git a/LogParser/src/main/scripts/parserlog.sh b/LogParser/src/main/scripts/parserlog.sh new file mode 100644 index 000000000..79ea8a3b3 --- /dev/null +++ b/LogParser/src/main/scripts/parserlog.sh @@ -0,0 +1,130 @@ +### +# ============LICENSE_START======================================================= +# LogParser +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + +#!/bin/sh +# +# init script for a Java application +# + +#Arguments for application +SERVER=http://localhost:8070/pap/ +LOGTYPE=PAP +LOGPATH="/var/lib/servers/pap/logs/catalina.out" +PARSERLOGPATH="$POLICY_HOME/logs/parserlog.log" +JDBC_URL="jdbc:h2:tcp://localhost:9092/log" +JDBC_USER="sa" +JDBC_DRIVER="org.h2.Driver" +JDBC_PASSWORD="" +SERVICE="LogParser.jar" + +# Check the application status +# +# This function checks if the application is running +check_status() { + + # Running pgrep with some arguments to check if the PID exists + if pgrep -f "$SERVICE $SERVER $LOGTYPE" ; then + RESULT=$(pgrep -f ${SERVICE}) + return $RESULT + fi + return 0 + # In any another case, return 0 + +} + +# Starts the application +start() { + + # At first checks if the application is already started calling the check_status + # function + check_status + + # $? is a special variable that hold the "exit status of the most recently executed + # foreground pipeline" + pid=$? + + if [ $pid -ne 0 ] ; then + echo "The application is already started" + exit 1 + fi + + # If the application isn't running, starts it + echo -n "Starting application: " + + # Redirects default and error output to a log file + java -jar LogParser.jar $SERVER $LOGTYPE $LOGPATH $PARSERLOGPATH $JDBC_URL $JDBC_USER $JDBC_DRIVER $JDBC_PASSWORD>> $POLICY_HOME/logs/parserlog.log 2>&1 & + echo "OK" +} + +# Stops the application +stop() { + + # Like as the start function, checks the application status + check_status + + pid=$? + + if [ $pid -eq 0 ] ; then + echo "Application is already stopped" + exit 1 + fi + + # Kills the application process + echo -n "Stopping application: " + kill -9 $pid & + echo "OK" +} + +# Show the application status +status() { + + # The check_status function, again... + check_status + + # If the PID was returned means the application is running + if [ $? -ne 0 ] ; then + echo "Application is started" + else + echo "Application is stopped" + fi + +} + +# Main logic, a simple case to call functions +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status + ;; + restart|reload) + stop + start + ;; + *) + echo "Usage: $0 {start|stop|restart|reload|status}" + exit 1 +esac + +exit 0 |