1 files changed, 317 insertions, 0 deletions
diff --git a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java
new file mode 100644
index 0000000..a3dc708
--- /dev/null
+++ b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java
@@ -0,0 +1,317 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datacleanup;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.onap.aai.exceptions.AAIException;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+
+import com.att.eelf.configuration.EELFLogger;
+
+@Component
+@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
+public class DataCleanupTasks {
+
+	private static final EELFLogger logger = EELFManager.getInstance().getLogger(DataCleanupTasks.class);
+	private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
+	
+	/**The function archives/deletes files that end in .out (Ie. dataGrooming.201511111305.out) that sit in our log/data directory structure.
+		logDir is the {project_home}/logs
+		archiveDir is the ARCHIVE directory where the files will be stored after 5 days.
+		ageZip is the number of days after which the file will be moved to the ARCHIVE folder.
+		ageDelete is the number of days after which the data files will be deleted i.e after 30 days.
+	*/
+	@Scheduled(cron = "${datagroomingcleanup.cron}" )
+	public void dataGroomingCleanup() throws AAIException, Exception {
+		
+		logger.info("Started cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date()));
+		
+		try {
+			String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs";
+			String dataGroomingDir = logDir + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataGrooming";
+			String archiveDir = dataGroomingDir + AAIConstants.AAI_FILESEP + "ARCHIVE";
+			String dataGroomingArcDir = archiveDir + AAIConstants.AAI_FILESEP + "dataGrooming";		
+			File path = new File(dataGroomingDir);
+			File archivepath = new File(archiveDir);
+			File dataGroomingPath = new File(dataGroomingArcDir);
+		
+			logger.info("The logDir is " + logDir);
+			logger.info("The dataGroomingDir is " + dataGroomingDir);
+			logger.info("The archiveDir is " + archiveDir );
+			logger.info("The dataGroomingArcDir is " + dataGroomingArcDir );
+		
+			boolean exists = directoryExists(logDir);
+			logger.info("Directory" + logDir + "exists: " + exists);
+			if(exists == false)
+				logger.error("The directory" + logDir +"does not exists");
+		
+			Integer ageZip = AAIConfig.getInt("aai.datagrooming.agezip");
+			Integer ageDelete = AAIConfig.getInt("aai.datagrooming.agedelete");
+							
+			Date newAgeZip = getZipDate(ageZip);
+						         	
+			//Iterate through the dataGroomingDir
+			File[] listFiles = path.listFiles();  
+			if(listFiles != null) {
+				for(File listFile : listFiles) {
+					if (listFile.toString().contains("ARCHIVE")){
+						continue;
+					}
+					if(listFile.isFile()){
+						logger.info("The file name in dataGrooming: " +listFile.getName()); 
+						Date fileCreateDate = fileCreationMonthDate(listFile);
+						logger.info("The fileCreateDate in dataGrooming is " + fileCreateDate);
+						if( fileCreateDate.compareTo(newAgeZip) < 0) {
+						archive(listFile,archiveDir,dataGroomingArcDir);						
+						}
+					}
+				}
+			}
+		
+			Date newAgeDelete = getZipDate(ageDelete);
+			//Iterate through the archive/dataGrooming dir
+			File[] listFilesArchive = dataGroomingPath.listFiles(); 
+			if(listFilesArchive != null) {
+				for(File listFileArchive : listFilesArchive) { 
+					if(listFileArchive.isFile()) {
+				logger.info("The file name in ARCHIVE/dataGrooming: " +listFileArchive.getName()); 
+				Date fileCreateDate = fileCreationMonthDate(listFileArchive);
+				logger.info("The fileCreateDate in ARCHIVE/dataGrooming is " + fileCreateDate);
+				if(fileCreateDate.compareTo(newAgeDelete) < 0) {
+					delete(listFileArchive);
+					}
+				}	
+			}
+			}
+		}
+		catch (Exception e) {
+			ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+			logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+			throw e;
+		}
+	}
+	
+    /**
+     * This method checks if the directory exists
+     * @param DIR
+     * 
+     */
+    public boolean directoryExists(String dir) {
+    	File path = new File(dir);
+		boolean exists = path.exists();
+		return exists;	
+    }
+    
+    public Date getZipDate(Integer days) throws Exception {
+    	return getZipDate(days, new Date());
+    }
+    
+    public Date getZipDate(Integer days, Date date) throws Exception{
+    	
+    	Calendar cal = Calendar.getInstance();
+    	logger.info("The current date is " + date );
+    	cal.setTime(date);	
+    	cal.add(Calendar.DATE, -days);
+    	Date newAgeZip = cal.getTime();
+		logger.info("The newAgeDate is " +newAgeZip);
+		return newAgeZip;		
+    }
+    
+    
+    public Date fileCreationMonthDate (File file) throws Exception {
+
+        BasicFileAttributes attr = Files.readAttributes(file.toPath(),
+                                                        BasicFileAttributes.class);
+        FileTime time = attr.creationTime();
+	    String formatted = simpleDateFormat.format( new Date( time.toMillis() ) );
+	    Date d = simpleDateFormat.parse(formatted);
+	    return d;
+    }
+    
+    /**
+     * This method will zip the files and add it to the archive folder
+     * Checks if the archive folder exists, if not then creates one
+     * After adding the file to archive folder it deletes the file from the filepath
+     * @throws AAIException
+     * @throws Exception
+     */
+    public void archive(File file, String archiveDir, String afterArchiveDir) throws AAIException, Exception {
+		
+    	logger.info("Inside the archive folder");  
+    	String filename = file.getName();
+    	logger.info("file name is " +filename);
+		File archivepath = new File(archiveDir);
+		
+		String zipFile = afterArchiveDir + AAIConstants.AAI_FILESEP + filename;
+		
+		File dataGroomingPath = new File(afterArchiveDir);
+	
+		boolean exists = directoryExists(archiveDir);
+		logger.info("Directory" + archiveDir + "exists: " + exists);		
+		if(exists == false) {
+			logger.error("The directory" + archiveDir +"does not exists so will create a new archive folder");
+			//Create an archive folder if does not exists		
+			boolean flag = dataGroomingPath.mkdirs();
+			if(flag == false)
+				logger.error("Failed to create ARCHIVE folder");		
+		}
+		try {		
+			FileOutputStream outputstream = new FileOutputStream(zipFile + ".gz");
+			ZipOutputStream zoutputstream = new ZipOutputStream(outputstream);
+			ZipEntry ze = new ZipEntry(file.getName());
+			zoutputstream.putNextEntry(ze);
+			//read the file and write to the zipOutputStream
+			FileInputStream inputstream = new FileInputStream(file);
+			byte[] buffer = new byte[1024];
+			int len;
+			while ((len = inputstream.read(buffer)) > 0) {
+				zoutputstream.write(buffer,0,len);
+			}			
+			//close all the sources
+			zoutputstream.closeEntry();
+			zoutputstream.close();
+			inputstream.close();
+			outputstream.close();
+			//Delete the file after been added to archive folder
+			delete(file);
+			logger.info("The file archived is " + file + " at " + afterArchiveDir );
+		}	
+	 catch (IOException e) {
+		 ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup " + e.getStackTrace());
+		 logger.info("AAI_4000", "Exception running cron job for DataCleanup", e);
+		 throw e;
+	 	}
+    }
+    
+    /**
+     * This method will delete all the files from the archive folder that are older than 60 days
+     * @param file
+     */
+    public static void delete(File file) {
+    	
+    	logger.info("Deleting the file " + file);
+    	boolean deleteStatus = file.delete();
+		if(deleteStatus == false){
+			logger.error("Failed to delete the file" +file);			
+		}
+    }
+    
+    /**The function archives/deletes files that end in .out (Ie. dataGrooming.201511111305.out) that sit in our log/data directory structure.
+	logDir is the {project_home}/logs
+	archiveDir is the ARCHIVE directory where the files will be stored after 5 days.
+	ageZip is the number of days after which the file will be moved to the ARCHIVE folder.
+	ageDelete is the number of days after which the data files will be deleted i.e after 30 days.
+*/
+    @Scheduled(cron = "${datasnapshotcleanup.cron}" )
+    public void dataSnapshotCleanup() throws AAIException, Exception {
+	
+    	logger.info("Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date()));
+	
+    	try {
+    		String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs";
+    		String dataSnapshotDir = logDir + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataSnapshots";
+    		String archiveDir = dataSnapshotDir + AAIConstants.AAI_FILESEP + "ARCHIVE";
+    		String dataSnapshotArcDir = archiveDir + AAIConstants.AAI_FILESEP + "dataSnapshots";		
+    		File path = new File(dataSnapshotDir);
+    		File archivepath = new File(archiveDir);
+    		File dataSnapshotPath = new File(dataSnapshotArcDir);
+	
+    		logger.info("The logDir is " + logDir);
+    		logger.info("The dataSnapshotDir is " + dataSnapshotDir);
+    		logger.info("The archiveDir is " + archiveDir );
+    		logger.info("The dataSnapshotArcDir is " + dataSnapshotArcDir );
+	
+    		boolean exists = directoryExists(logDir);
+    		logger.info("Directory" + logDir + "exists: " + exists);
+    		if(exists == false)
+    			logger.error("The directory" + logDir +"does not exists");
+	
+    		Integer ageZipSnapshot = AAIConfig.getInt("aai.datasnapshot.agezip");
+    		Integer ageDeleteSnapshot = AAIConfig.getInt("aai.datasnapshot.agedelete");
+    		
+    		Date newAgeZip = getZipDate(ageZipSnapshot);
+			         	
+    		//Iterate through the dataGroomingDir
+    		File[] listFiles = path.listFiles();  
+    		if(listFiles != null) {
+    			for(File listFile : listFiles) {
+    				if (listFile.toString().contains("ARCHIVE")){
+					continue;
+    				}
+    				if(listFile.isFile()){
+    					logger.info("The file name in dataSnapshot: " +listFile.getName()); 
+    					Date fileCreateDate = fileCreationMonthDate(listFile);
+    					logger.info("The fileCreateDate in dataSnapshot is " + fileCreateDate);
+    					if( fileCreateDate.compareTo(newAgeZip) < 0) {
+    						archive(listFile,archiveDir,dataSnapshotArcDir);						
+    					}
+    				}
+    			}
+    		}
+	
+    		Date newAgeDelete = getZipDate(ageDeleteSnapshot);
+    		//Iterate through the archive/dataSnapshots dir
+    		File[] listFilesArchive = dataSnapshotPath.listFiles(); 
+    		if(listFilesArchive != null) {
+    			for(File listFileArchive : listFilesArchive) { 
+    				if(listFileArchive.isFile()) {
+    					logger.info("The file name in ARCHIVE/dataSnapshot: " +listFileArchive.getName()); 
+    					Date fileCreateDate = fileCreationMonthDate(listFileArchive);
+    					logger.info("The fileCreateDate in ARCHIVE/dataSnapshot is " + fileCreateDate);
+    					if(fileCreateDate.compareTo(newAgeDelete) < 0) {
+    						delete(listFileArchive);
+    					}
+    				}	
+    			}
+    		}
+	}
+	catch (Exception e) {
+		ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+		logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+		throw e;
+	}
+  }   
+}