Expire, Remove Batch, restore

Issue-ID: AAF-641
Change-Id: I3d36005d59eb466141154fb729d48d5e2763fa9d
Signed-off-by: Instrumental <jonathan.gathman@att.com>

7 files changed, 441 insertions, 325 deletions

diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
index 483f70f4..ec609488 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
@@ -32,7 +32,6 @@ import java.util.TreeMap;
 
 import org.onap.aaf.auth.dao.cass.CredDAO;
 import org.onap.aaf.auth.dao.hl.Question;
-import org.onap.aaf.auth.helpers.Cred.Instance;
 import org.onap.aaf.cadi.util.CSV;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
@@ -72,7 +71,7 @@ public class Cred  {
         }
         
         public String toString() {
-        	return expires.toString() + type;
+        	return expires.toString() + ": " + type;
         }
     }
     
@@ -132,20 +131,16 @@ public class Cred  {
         try {
             Iterator<Row> iter = results.iterator();
             Row row;
-            int type; // for filtering
-            String id;
             tt = trans.start("Load Credentials", Env.SUB);
             try {
                 while (iter.hasNext()) {
                     ++count;
                     row = iter.next();
-                    id = row.getString(0);
-                    type = row.getInt(1);
+                    int type = row.getInt(1);
                     if (types.length>0) { // filter by types, if requested
                         boolean quit = true;
                         for (int t : types) {
                             if (t==type) {
-                                quit=false;
                                 break;
                             }
                         }
@@ -153,27 +148,7 @@ public class Cred  {
                             continue;
                         }
                     }
-                    Cred cred = data.get(id);
-                    if (cred==null) {
-                        cred = new Cred(id);
-                        data.put(id, cred);
-                    }
-                    cred.instances.add(new Instance(type, row.getTimestamp(2), row.getInt(3), row.getLong(4)/1000));
-                    
-                    List<Cred> lscd = byNS.get(cred.ns);
-                    if (lscd==null) {
-                        byNS.put(cred.ns, (lscd=new ArrayList<>()));
-                    }
-                    boolean found = false;
-                    for (Cred c : lscd) {
-                        if (c.id.equals(cred.id)) {
-                            found=true;
-                            break;
-                        }
-                    }
-                    if (!found) {
-                        lscd.add(cred);
-                    }
+                    add(row.getString(0), row.getInt(1),row.getTimestamp(2),row.getInt(3),row.getLong(4));
                 }
             } finally {
                 tt.done();
@@ -183,7 +158,38 @@ public class Cred  {
         }
     }
 
-    /** 
+    public static void add(
+    		final String id, 
+    		final int type,
+    		final Date timestamp,
+    		final int other,
+    		final long written
+    		) {
+        Cred cred = data.get(id);
+        if (cred==null) {
+            cred = new Cred(id);
+            data.put(id, cred);
+        }
+        cred.instances.add(new Instance(type, timestamp, other, written/1000));
+        
+        List<Cred> lscd = byNS.get(cred.ns);
+        if (lscd==null) {
+            byNS.put(cred.ns, (lscd=new ArrayList<>()));
+        }
+        boolean found = false;
+        for (Cred c : lscd) {
+            if (c.id.equals(cred.id)) {
+                found=true;
+                break;
+            }
+        }
+        if (!found) {
+            lscd.add(cred);
+        }
+	}
+
+
+	/** 
      * Count entries in Cred data.
      * Note, as opposed to other methods, need to load the whole cred table for the Types.
      * @param numbuckets 
@@ -269,8 +275,8 @@ public class Cred  {
 
     }
     
-    public void row(CSV.Writer csvw, Instance inst) {
-    	csvw.row("cred",id,ns,Integer.toString(inst.type),Chrono.dateOnlyStamp(inst.expires),Long.toString(inst.expires.getTime()));
+    public void row(final CSV.Writer csvw, final Instance inst) {
+    	csvw.row("cred",id,ns,Integer.toString(inst.type),Chrono.dateOnlyStamp(inst.expires),inst.expires.getTime());
     }
 
 
@@ -279,6 +285,7 @@ public class Cred  {
     	sb.append(row.get(1));
     	sb.append("' AND type=");
     	sb.append(Integer.parseInt(row.get(3)));
+    	// Note: We have to work with long, because Expires is part of Key... can't easily do date.
     	sb.append(" AND expires=dateof(maxtimeuuid(");
     	sb.append(row.get(5));
     	sb.append("));\n");
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
index eb1f821c..d9e6bed3 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
@@ -304,8 +304,8 @@ public class UserRole implements Cloneable, CacheChange.Data  {
         cache.resetLocalData();
     }
 
-    public void row(CSV.Writer csvw) {
-    	csvw.row("ur",user(),role(),Chrono.dateOnlyStamp(expires()));
+    public void row(final CSV.Writer csvw) {
+    	csvw.row("ur",user(),ns(),rname(),Chrono.dateOnlyStamp(expires()));
     }
     
     public static void row(StringBuilder sb, List<String> row) {
@@ -313,6 +313,8 @@ public class UserRole implements Cloneable, CacheChange.Data  {
     	sb.append(row.get(1));
     	sb.append("' AND role='");
     	sb.append(row.get(2));
+    	sb.append('.');
+    	sb.append(row.get(3));
     	sb.append("';\n");
     }
     
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java
index bb75e110..95fe3c0e 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java
@@ -21,7 +21,6 @@
 
 package org.onap.aaf.auth.helpers;
 
-import java.awt.HeadlessException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.cert.X509Certificate;
@@ -29,7 +28,6 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.onap.aaf.cadi.Hash;
-import org.onap.aaf.cadi.Symm;
 import org.onap.aaf.cadi.util.CSV;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java
index d283f414..6974a5db 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java
@@ -27,9 +27,8 @@ import java.io.IOException;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
-import java.util.ArrayList;
 import java.util.Date;
-import java.util.GregorianCalendar;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -44,10 +43,11 @@ import org.onap.aaf.auth.helpers.Cred.Instance;
 import org.onap.aaf.auth.helpers.UserRole;
 import org.onap.aaf.auth.helpers.Visitor;
 import org.onap.aaf.auth.helpers.X509;
+import org.onap.aaf.auth.org.ExpireRange;
+import org.onap.aaf.auth.org.ExpireRange.Range;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.cadi.configure.Factory;
 import org.onap.aaf.cadi.util.CSV;
-import org.onap.aaf.cadi.util.CSV.Writer;
 import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
@@ -56,18 +56,14 @@ import org.onap.aaf.misc.env.util.Chrono;
 
 public class Expiring extends Batch {
     
-    private int minOwners;
-	private ArrayList<Writer> writerList;
+	private static final String CSV = ".csv";
+	private static final String INFO = "info";
+	private static final String EXPIRED_OWNERS = "ExpiredOwners";
+	private int minOwners;
+	private Map<String, CSV.Writer> writerList;
 	private File logDir;
-	private Date now;
-	private Date twoWeeksPast;
-	private Writer twoWeeksPastCSV;
-	private Date twoWeeksAway;
-	private Writer twoWeeksAwayCSV;
-	private Date oneMonthAway;
-	private Writer oneMonthAwayCSV;
-	private Date twoMonthsAway;
-	private Writer twoMonthsAwayCSV;
+	private ExpireRange expireRange;
+	private Date deleteDate;
 	
 	public Expiring(AuthzTrans trans) throws APIException, IOException, OrganizationException {
         super(trans.env());
@@ -84,40 +80,32 @@ public class Expiring extends Batch {
             
             // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
             Cred.load(trans, session);
+            UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor());
 
             minOwners=1;
 
             // Create Intermediate Output 
-            writerList = new ArrayList<CSV.Writer>();
+            writerList = new HashMap<>();
             logDir = new File(logDir());
             logDir.mkdirs();
             
-            GregorianCalendar gc = new GregorianCalendar();
-            now = gc.getTime();
-            gc.add(GregorianCalendar.WEEK_OF_MONTH, -2);
-            twoWeeksPast = gc.getTime();
-            File file = new File(logDir,"Expired"+Chrono.dateOnlyStamp(now)+".csv");
-            twoWeeksPastCSV = new CSV(file).writer();
-            writerList.add(twoWeeksPastCSV);
-            
-            gc.add(GregorianCalendar.WEEK_OF_MONTH, 2+2);
-            twoWeeksAway = gc.getTime();
-            file = new File(logDir,"TwoWeeksAway"+Chrono.dateOnlyStamp(now)+".csv");
-            twoWeeksAwayCSV = new CSV(file).writer();
-            writerList.add(twoWeeksAwayCSV);
-
-            gc.add(GregorianCalendar.WEEK_OF_MONTH, -2);
-            gc.add(GregorianCalendar.MONTH, 1);
-            oneMonthAway = gc.getTime();
-            file = new File(logDir,"OneMonthAway"+Chrono.dateOnlyStamp(now)+".csv");
-            oneMonthAwayCSV = new CSV(file).writer();
-            writerList.add(oneMonthAwayCSV);
+            expireRange = new ExpireRange(trans.env().access());
+            String sdate = Chrono.dateOnlyStamp(expireRange.now);
+            for( List<Range> lr : expireRange.ranges.values()) {
+            	for(Range r : lr ) {
+            		if(writerList.get(r.name())==null) {
+                    	File file = new File(logDir,r.name() + sdate +CSV);
+                    	CSV csv = new CSV(file);
+                    	CSV.Writer cw = csv.writer(false);
+                    	cw.row(INFO,r.name(),Chrono.dateOnlyStamp(expireRange.now),r.reportingLevel());
+                    	writerList.put(r.name(),cw);
+                    	if("Delete".equals(r.name())) {
+                    		deleteDate = r.getStart();
+                    	}
+            		}
+            	}
+            }
             
-            gc.add(GregorianCalendar.MONTH, 1);
-            twoMonthsAway = gc.getTime();
-            file = new File(logDir,"TwoMonthsAway"+Chrono.dateOnlyStamp(now)+".csv");
-            twoMonthsAwayCSV = new CSV(file).writer();
-            writerList.add(twoMonthsAwayCSV);
         } finally {
             tt0.done();
         }
@@ -126,7 +114,7 @@ public class Expiring extends Batch {
     @Override
     protected void run(AuthzTrans trans) {
 		try {
-			File file = new File(logDir, "AllOwnersExpired" + Chrono.dateOnlyStamp(now) + ".csv");
+			File file = new File(logDir, EXPIRED_OWNERS + Chrono.dateOnlyStamp(expireRange.now) + CSV);
 			final CSV ownerCSV = new CSV(file);
 
 			Map<String, Set<UserRole>> owners = new TreeMap<String, Set<UserRole>>();
@@ -143,7 +131,7 @@ public class Expiring extends Batch {
 						}
 						urs.add(ur);
 					} else {
-						writeAnalysis(ur);
+						writeAnalysis(trans,ur);
 					}
 				}
 			});
@@ -160,19 +148,20 @@ public class Expiring extends Batch {
 					for (Set<UserRole> sur : owners.values()) {
 						int goodOwners = 0;
 						for (UserRole ur : sur) {
-							if (ur.expires().after(now)) {
+							if (ur.expires().after(expireRange.now)) {
 								++goodOwners;
 							}
 						}
 
 						for (UserRole ur : sur) {
 							if (goodOwners >= minOwners) {
-								writeAnalysis(ur);
+								writeAnalysis(trans, ur);
 							} else {
 								if (expOwner == null) {
 									expOwner = ownerCSV.writer();
+									expOwner.row(INFO,EXPIRED_OWNERS,Chrono.dateOnlyStamp(expireRange.now),2);
 								}
-								expOwner.row(ur.role(), ur.user(), ur.expires());
+								expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
 							}
 						}
 					}
@@ -182,24 +171,26 @@ public class Expiring extends Batch {
 			}
 			
 			trans.info().log("Checking for Expired Credentials");
+			
 			for (Cred cred : Cred.data.values()) {
 		    	List<Instance> linst = cred.instances;
 		    	if(linst!=null) {
 			    	Instance lastBath = null;
 			    	for(Instance inst : linst) {
-			        	if(inst.expires.before(twoWeeksPast)) {
-			        		cred.row(twoWeeksPastCSV,inst);
-			    		} else if(inst.expires.after(now)){
-			    			if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
-			    				if(lastBath==null || lastBath.expires.before(inst.expires)) {
-			    					lastBath = inst;
-			    				}
-			    			} else if(inst.type==CredDAO.CERT_SHA256_RSA) {
-			    				writeAnalysis(cred, inst);
-			    			}
-			    		}
+			    		// Special Behavior: only eval the LAST Instance
+		    			if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
+				        	if(deleteDate!=null && inst.expires.before(deleteDate)) {
+				        		writeAnalysis(trans, cred, inst); // will go to Delete
+				    		} else if(lastBath==null || lastBath.expires.before(inst.expires)) {
+		    					lastBath = inst;
+		    				}
+		    			} else {
+		    				writeAnalysis(trans, cred, inst);
+		    			}
+			    	}
+			    	if(lastBath!=null) {
+			    		writeAnalysis(trans, cred, lastBath);
 			    	}
-			    	writeAnalysis(cred, lastBath);
 		    	}
 			}
 			
@@ -209,7 +200,7 @@ public class Expiring extends Batch {
 				public void visit(X509 x509) {
 					try {
 						for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
-							writeAnalysis(x509, (X509Certificate)cert);
+							writeAnalysis(trans, x509, (X509Certificate)cert);
 						}
 					} catch (CertificateException | IOException e) {
 						trans.error().log(e, "Error Decrypting X509");
@@ -223,52 +214,84 @@ public class Expiring extends Batch {
 	}
     
  
-	protected void writeAnalysis(UserRole ur) {
-    	if(ur.expires().before(twoWeeksPast)) {
-    		ur.row(twoWeeksPastCSV);
-		} else {
-			if(ur.expires().after(now) && ur.expires().before(twoWeeksAway)) {
-	    		ur.row(twoWeeksAwayCSV);
-			} else {
-				if(ur.expires().before(oneMonthAway)) {
-		    		ur.row(oneMonthAwayCSV);
-				} else {
-					if(ur.expires().before(twoMonthsAway)) {
-			    		ur.row(twoMonthsAwayCSV);
-					}
-				}
+	private void writeAnalysis(AuthzTrans trans, UserRole ur) {
+		Range r = expireRange.getRange("ur", ur.expires());
+		if(r!=null) {
+			CSV.Writer cw = writerList.get(r.name());
+			if(cw!=null) {
+				ur.row(cw);
 			}
 		}
 	}
     
-    protected void writeAnalysis(Cred cred, Instance inst) {
-    	if(inst!=null) {
-			if(inst.expires.after(now) && inst.expires.before(twoWeeksAway)) {
-	    		cred.row(twoWeeksAwayCSV, inst);
-			} else {
-				if(inst.expires.before(oneMonthAway)) {
-		    		cred.row(oneMonthAwayCSV, inst);
-				} else {
-					if(inst.expires.before(twoMonthsAway)) {
-			    		cred.row(twoMonthsAwayCSV, inst);
-					}
+    private void writeAnalysis(AuthzTrans trans, Cred cred, Instance inst) {
+    	if(cred!=null && inst!=null) {
+			Range r = expireRange.getRange("cred", inst.expires);
+			if(r!=null) {
+				CSV.Writer cw = writerList.get(r.name());
+				if(cw!=null) {
+					cred.row(cw,inst);
 				}
 			}
-		}
+    	}
 	}
 
-    protected void writeAnalysis(X509 x509, X509Certificate x509Cert) throws IOException {
-    	if(x509Cert!=null) {
-	    	if(twoWeeksPast.after(x509Cert.getNotAfter())) {
-				x509.row(twoWeeksPastCSV,x509Cert);
+    private void writeAnalysis(AuthzTrans trans, X509 x509, X509Certificate x509Cert) throws IOException {
+		Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
+		if(r!=null) {
+			CSV.Writer cw = writerList.get(r.name());
+			if(cw!=null) {
+				x509.row(cw,x509Cert);
 			}
-    	}
+		}
 	}
 
+    /*
+    private String[] contacts(final AuthzTrans trans, final String ns, final int levels) {
+    	List<UserRole> owners = UserRole.getByRole().get(ns+".owner");
+    	List<UserRole> current = new ArrayList<>();
+    	for(UserRole ur : owners) {
+    		if(expireRange.now.before(ur.expires())) {
+    			current.add(ur);
+    		}
+    	}
+    	if(current.isEmpty()) {
+    		trans.warn().log(ns,"has no current owners");
+    		current = owners;
+    	}
+    	
+    	List<String> email = new ArrayList<>();
+    	for(UserRole ur : current) {
+    		Identity id;
+    		int i=0;
+    		boolean go = true;
+    		try {
+    			id = org.getIdentity(trans, ur.user());
+        		do {
+	    			if(id!=null) {
+						email.add(id.email());
+						if(i<levels) {
+							id = id.responsibleTo();
+						} else {
+							go = false;
+						}
+	    			} else {
+	    				go = false;
+	    			}
+        		} while(go);
+			} catch (OrganizationException e) {
+				trans.error().log(e);
+			}
+    	}
+    	
+    	return email.toArray(new String[email.size()]);
+    }
+*/
+    
 	@Override
     protected void _close(AuthzTrans trans) {
         session.close();
-    	for(CSV.Writer cw : writerList) {
+    	for(CSV.Writer cw : writerList.values()) {
     		cw.close();
     	}
     }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
index cbc3ef38..848b54e8 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
@@ -21,17 +21,21 @@
 
 package org.onap.aaf.auth.update;
 
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.TreeMap;
 
 import org.onap.aaf.auth.Batch;
@@ -43,6 +47,7 @@ import org.onap.aaf.auth.dao.cass.CredDAO;
 import org.onap.aaf.auth.dao.hl.Question;
 import org.onap.aaf.auth.env.AuthzTrans;
 import org.onap.aaf.auth.helpers.Cred;
+import org.onap.aaf.auth.helpers.NS;
 import org.onap.aaf.auth.helpers.Notification;
 import org.onap.aaf.auth.helpers.Notification.TYPE;
 import org.onap.aaf.auth.helpers.UserRole;
@@ -52,6 +57,8 @@ import org.onap.aaf.auth.org.Organization;
 import org.onap.aaf.auth.org.Organization.Identity;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.auth.org.OrganizationFactory;
+import org.onap.aaf.cadi.CadiException;
+import org.onap.aaf.cadi.util.CSV;
 import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
@@ -73,6 +80,8 @@ public class NotifyCredExpiring extends Batch {
     private final PrintStream ps;
     private final AuthzTrans noAvg;
     private String supportEmailAddr;
+	private CSV csv;
+	private CSVInfo csvInfo;
 
     public NotifyCredExpiring(AuthzTrans trans) throws APIException, IOException, OrganizationException {
         super(trans.env());
@@ -86,9 +95,6 @@ public class NotifyCredExpiring extends Batch {
         noAvg = env.newTransNoAvg();
         noAvg.setUser(new BatchPrincipal("batch:NotifyCredExpiring"));
         
-        if ((supportEmailAddr = env.getProperty("mailFromUserId"))==null) {
-            throw new APIException("mailFromUserId property must be set");
-        }
         if (isDryRun()) {
             email = new EmailPrint();
             maxEmails=3;
@@ -105,10 +111,33 @@ public class NotifyCredExpiring extends Batch {
         email.signature("Sincerely,\nAAF Team (Our MOTS# 22830)\n"
                 + "https://wiki.web.att.com/display/aaf/Contact+Us\n"
                 + "(Use 'Other Misc Requests (TOPS)')");
-
-        Cred.load(trans, session,CredDAO.BASIC_AUTH, CredDAO.BASIC_AUTH_SHA256);
+        
+        boolean quit = false;
+        if(args().length<1) {
+        	System.err.println("Need CSV formatted Expiring Report");
+        	quit = true;
+        } else {
+        	File f = new File(logDir(),args()[0]);
+        	System.out.println("Reading " + f.getCanonicalPath());
+        	csv = new CSV(f);
+        }
+        
+        if(args().length<2) {
+        	System.err.println("Need Email Template");
+        	//quit = true;
+        }
+        if(quit) {
+        	System.exit(2);
+        }
+        
+        csvInfo = new CSVInfo(System.err);
+        try {
+			csv.visit(csvInfo);
+		} catch (CadiException e) {
+			throw new APIException(e);
+		}
+        
         Notification.load(trans, session, Notification.v2_0_18);
-        UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor());
         
         ps = new PrintStream(new FileOutputStream(logDir() + "/email"+Chrono.dateOnlyStamp()+".log",true));
         ps.printf("### Approval Notify %s for %s%s\n",Chrono.dateTime(),batchEnv,dryRun?", DryRun":"");
@@ -117,199 +146,244 @@ public class NotifyCredExpiring extends Batch {
     @Override
     protected void run(AuthzTrans trans) {
         
-        EmailWarnings ewp = org.emailWarningPolicy();
-        long now = System.currentTimeMillis();
-        Date early = new Date(now+(ewp.credExpirationWarning()*2)); // 2 months back
-        Date must = new Date(now+ewp.credExpirationWarning()); // 1 months back
-        Date critical = new Date(now+ewp.emailUrgentWarning()); // 1 week
-        Date within2Weeks = new Date(now+604800000 * 2);
-        Date withinLastWeek = new Date(now-604800000);
-        Date tooLate = new Date(now);
-        
         // Temp structures
-        Map<String,Cred> lastCred = new HashMap<>();
         Map<String,List<LastCred>> ownerCreds = new TreeMap<>();
-        Date last;
         
 
         List<LastCred> noOwner = new ArrayList<>();
         ownerCreds.put(UNKNOWN_ID,noOwner);
-
-        // Get a list of ONLY the ones needing email by Owner
-        for (Entry<String, List<Cred>> es : Cred.byNS.entrySet()) {
-            lastCred.clear();
-            for (Cred c : es.getValue()) {
-                last = c.last(CredDAO.BASIC_AUTH,CredDAO.BASIC_AUTH_SHA256);
-                if (last!=null && last.after(tooLate) && last.before(early)) {
-                    List<UserRole> ownerURList = UserRole.getByRole().get(es.getKey()+".owner");
-                    if (ownerURList!=null) {
-                        for (UserRole ur:ownerURList) {
-                            String owner = ur.user();
-                            List<LastCred> llc = ownerCreds.get(owner);
-                            if (llc==null) {
-                                ownerCreds.put(owner, (llc=new ArrayList<>()));
-                            }
-                            llc.add(new LastCred(c,last));
-                        }
-                    } else {
-                        noOwner.add(new LastCred(c,last));
-                    }
-                }
-            }
-        }
-        
-        boolean bCritical,bNormal,bEarly;
         int emailCount=0;
-        Message msg = new Message();
-        Notification ownNotf;
-        StringBuilder logMessage = new StringBuilder();
-        for (Entry<String,List<LastCred>> es : ownerCreds.entrySet()) {
-            String owner = es.getKey();
-            boolean header = true;
-            try {
-                Organization org = OrganizationFactory.obtain(env, owner);
-                Identity user = org.getIdentity(noAvg, owner);
-                if (!UNKNOWN_ID.equals(owner) && user==null) {
-                    ps.printf("Invalid Identity: %s\n", owner);
-                } else {
-                    logMessage.setLength(0);
-                    if (maxEmails>emailCount) {
-                        bCritical=bNormal=bEarly = false;
-                        email.clear();
-                        msg.clear();
-                        email.addTo(user==null?supportEmailAddr:user.email());
-
-                        ownNotf = Notification.get(es.getKey(),TYPE.CN);
-                        if (ownNotf==null) {
-                            ownNotf = Notification.create(user==null?UNKNOWN_ID:user.fullID(), TYPE.CN);
-                        }
-                        last = ownNotf.last;
-                        // Get Max ID size for formatting purposes
-                        int length = AAF_INSTANTIATED_MECHID.length();
-                        for (LastCred lc : es.getValue()) {
-                            length = Math.max(length, lc.cred.id.length());
-                        }
-                        String id_exp_fmt = "\t%-"+length+"s  %15s  %s";
 
-                        Collections.sort(es.getValue(),LastCred.COMPARE);
-                        for (LastCred lc : es.getValue()) {
-                            if (lc.last.after(must) && lc.last.before(early) && 
-                                (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
-                                if (!bEarly && header) {
-                                    msg.line("\tThe following are friendly 2 month reminders, just in case you need to schedule your updates early.  "
-                                            + "You will be reminded next month\n");
-                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
-                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
-                                    header = false;
-                                }
-                                bEarly = true;
-                            } else if (lc.last.after(critical) && lc.last.before(must) && 
-                                    (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
-                                if (!bNormal) {
-                                    boolean last2wks = lc.last.before(within2Weeks);
-                                    if (last2wks) {
-                                        try {
-                                            Identity supvsr = user.responsibleTo();
-                                            email.addCC(supvsr.email());
-                                        } catch (OrganizationException e) {
-                                            trans.error().log(e, "Supervisor cannot be looked up");
-                                        }
-                                    }
-                                    if (header) {
-                                        msg.line("\tIt is now important for you to update Passwords all all configurations using them for the following.\n" +
-                                                (last2wks?"\tNote: Your Supervisor is CCd\n":"\tNote: Your Supervisor will be notified if this is not being done before the last 2 weeks\n"));
-                                        msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
-                                        msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
-                                    }
-                                    header = false;
-                                }
-                                bNormal=true;
-                            } else if (lc.last.after(tooLate) && lc.last.before(critical)) { // Email Every Day, with Supervisor
-                                if (!bCritical && header) {
-                                    msg.line("\t!!! WARNING: These Credentials will expire in LESS THAN ONE WEEK !!!!\n" +
-                                             "\tYour supervisor is added to this Email\n");
-                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
-                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
-                                    header = false;
-                                }
-                                bCritical = true;
-                                try {
-                                    if (user!=null) {
-                                        Identity supvsr = user.responsibleTo();
-                                        if (supvsr!=null) {
-                                            email.addCC(supvsr.email());
-                                            supvsr = supvsr.responsibleTo();
-                                            if (supvsr!=null) {
-                                                email.addCC(supvsr.email());
-                                            }
-                                        }
-                                    }
-                                } catch (OrganizationException e) {
-                                    trans.error().log(e, "Supervisor cannot be looked up");
-                                }
-                            }
-                            if (bEarly || bNormal || bCritical) {
-                                if (logMessage.length()==0) {
-                                    logMessage.append("NotifyCredExpiring");
-                                }
-                                logMessage.append("\n\t");
-                                logMessage.append(lc.cred.id);
-                                logMessage.append('\t');
-                                logMessage.append(Chrono.dateOnlyStamp(lc.last));
-                                msg.line(id_exp_fmt, lc.cred.id, Chrono.dateOnlyStamp(lc.last)+"     ",env.getProperty(GUI_URL)+"/creddetail?ns="+Question.domain2ns(lc.cred.id));
-                            }
-                        }
-                        
-                        if (bEarly || bNormal || bCritical) {
-                            msg.line(LINE);
-                            msg.line("Why are you receiving this Notification?\n");
-                                msg.line("You are the listed owner of one or more AAF Namespaces. ASPR requires that those responsible for "
-                                        + "applications and their access review them regularly for accuracy.  The AAF WIKI page for AT&T is https://wiki.web.att.com/display/aaf.  "
-                                        + "You might like https://wiki.web.att.com/display/aaf/AAF+in+a+Nutshell.  More detailed info regarding questions of being a Namespace Owner is available at https://wiki.web.att.com/pages/viewpage.action?pageId=594741363\n");
-                                msg.line("You may view the Namespaces you listed as Owner for in this AAF Env by viewing the following webpage:\n");
-                                msg.line("   %s/ns\n\n",env.getProperty(GUI_URL));
-                            email.msg(msg);
-                            Result<Void> rv = email.exec(trans, org,"");
-                            if (rv.isOK()) {
-                                ++emailCount;
-                                if (!isDryRun()) {
-                                    ownNotf.update(noAvg, session, false);
-                                    // SET LastNotification
-                                }
-                                email.log(ps,logMessage.toString());
-                            } else {
-                                trans.error().log(rv.errorString());
-                            }
-                        }
-                    }
-                }
-            } catch (OrganizationException e) {
-                trans.info().log(e);
-            }
-        }
+//        // Get a list of ONLY the ones needing email by Owner
+//        for (Entry<String, List<Cred>> es : Cred.byNS.entrySet()) {
+//            for (Cred c : es.getValue()) {
+//                List<UserRole> ownerURList = UserRole.getByRole().get(es.getKey()+".owner");
+//                if (ownerURList!=null) {
+//                    for (UserRole ur:ownerURList) {
+//                        String owner = ur.user();
+//                        List<LastCred> llc = ownerCreds.get(owner);
+//                        if (llc==null) {
+//                            ownerCreds.put(owner, (llc=new ArrayList<>()));
+//                        }
+//                        llc.add(new LastCred(c,last));
+//                    }
+//                } else {
+//                    noOwner.add(new LastCred(c,last));
+//                }
+//            }
+//        }
+//        
+//        boolean bCritical,bNormal,bEarly;
+//        Message msg = new Message();
+//        Notification ownNotf;
+//        StringBuilder logMessage = new StringBuilder();
+//        for (Entry<String,List<LastCred>> es : ownerCreds.entrySet()) {
+//            String owner = es.getKey();
+//            boolean header = true;
+//            try {
+//                Organization org = OrganizationFactory.obtain(env, owner);
+//                Identity user = org.getIdentity(noAvg, owner);
+//                if (!UNKNOWN_ID.equals(owner) && user==null) {
+//                    ps.printf("Invalid Identity: %s\n", owner);
+//                } else {
+//                    logMessage.setLength(0);
+//                    if (maxEmails>emailCount) {
+//                        bCritical=bNormal=bEarly = false;
+//                        email.clear();
+//                        msg.clear();
+//                        email.addTo(user==null?supportEmailAddr:user.email());
+//
+//                        ownNotf = Notification.get(es.getKey(),TYPE.CN);
+//                        if (ownNotf==null) {
+//                            ownNotf = Notification.create(user==null?UNKNOWN_ID:user.fullID(), TYPE.CN);
+//                        }
+//                        last = ownNotf.last;
+//                        // Get Max ID size for formatting purposes
+//                        int length = AAF_INSTANTIATED_MECHID.length();
+//                        for (LastCred lc : es.getValue()) {
+//                            length = Math.max(length, lc.cred.id.length());
+//                        }
+//                        String id_exp_fmt = "\t%-"+length+"s  %15s  %s";
+//
+//                        Collections.sort(es.getValue(),LastCred.COMPARE);
+//                        for (LastCred lc : es.getValue()) {
+//                            if (lc.last.after(must) && lc.last.before(early) && 
+//                                (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
+//                                if (!bEarly && header) {
+//                                    msg.line("\tThe following are friendly 2 month reminders, just in case you need to schedule your updates early.  "
+//                                            + "You will be reminded next month\n");
+//                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
+//                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
+//                                    header = false;
+//                                }
+//                                bEarly = true;
+//                            } else if (lc.last.after(critical) && lc.last.before(must) && 
+//                                    (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
+//                                if (!bNormal) {
+//                                    boolean last2wks = lc.last.before(within2Weeks);
+//                                    if (last2wks) {
+//                                        try {
+//                                            Identity supvsr = user.responsibleTo();
+//                                            email.addCC(supvsr.email());
+//                                        } catch (OrganizationException e) {
+//                                            trans.error().log(e, "Supervisor cannot be looked up");
+//                                        }
+//                                    }
+//                                    if (header) {
+//                                        msg.line("\tIt is now important for you to update Passwords all all configurations using them for the following.\n" +
+//                                                (last2wks?"\tNote: Your Supervisor is CCd\n":"\tNote: Your Supervisor will be notified if this is not being done before the last 2 weeks\n"));
+//                                        msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
+//                                        msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
+//                                    }
+//                                    header = false;
+//                                }
+//                                bNormal=true;
+//                            } else if (lc.last.after(tooLate) && lc.last.before(critical)) { // Email Every Day, with Supervisor
+//                                if (!bCritical && header) {
+//                                    msg.line("\t!!! WARNING: These Credentials will expire in LESS THAN ONE WEEK !!!!\n" +
+//                                             "\tYour supervisor is added to this Email\n");
+//                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
+//                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
+//                                    header = false;
+//                                }
+//                                bCritical = true;
+//                                try {
+//                                    if (user!=null) {
+//                                        Identity supvsr = user.responsibleTo();
+//                                        if (supvsr!=null) {
+//                                            email.addCC(supvsr.email());
+//                                            supvsr = supvsr.responsibleTo();
+//                                            if (supvsr!=null) {
+//                                                email.addCC(supvsr.email());
+//                                            }
+//                                        }
+//                                    }
+//                                } catch (OrganizationException e) {
+//                                    trans.error().log(e, "Supervisor cannot be looked up");
+//                                }
+//                            }
+//                            if (bEarly || bNormal || bCritical) {
+//                                if (logMessage.length()==0) {
+//                                    logMessage.append("NotifyCredExpiring");
+//                                }
+//                                logMessage.append("\n\t");
+//                                logMessage.append(lc.cred.id);
+//                                logMessage.append('\t');
+//                                logMessage.append(Chrono.dateOnlyStamp(lc.last));
+//                                msg.line(id_exp_fmt, lc.cred.id, Chrono.dateOnlyStamp(lc.last)+"     ",env.getProperty(GUI_URL)+"/creddetail?ns="+Question.domain2ns(lc.cred.id));
+//                            }
+//                        }
+//                        
+//                        if (bEarly || bNormal || bCritical) {
+//                            msg.line(LINE);
+//                            msg.line("Why are you receiving this Notification?\n");
+//                                msg.line("You are the listed owner of one or more AAF Namespaces. ASPR requires that those responsible for "
+//                                        + "applications and their access review them regularly for accuracy.  The AAF WIKI page for AT&T is https://wiki.web.att.com/display/aaf.  "
+//                                        + "You might like https://wiki.web.att.com/display/aaf/AAF+in+a+Nutshell.  More detailed info regarding questions of being a Namespace Owner is available at https://wiki.web.att.com/pages/viewpage.action?pageId=594741363\n");
+//                                msg.line("You may view the Namespaces you listed as Owner for in this AAF Env by viewing the following webpage:\n");
+//                                msg.line("   %s/ns\n\n",env.getProperty(GUI_URL));
+//                            email.msg(msg);
+//                            Result<Void> rv = email.exec(trans, org,"");
+//                            if (rv.isOK()) {
+//                                ++emailCount;
+//                                if (!isDryRun()) {
+//                                    ownNotf.update(noAvg, session, false);
+//                                    // SET LastNotification
+//                                }
+//                                email.log(ps,logMessage.toString());
+//                            } else {
+//                                trans.error().log(rv.errorString());
+//                            }
+//                        }
+//                    }
+//                }
+//            } catch (OrganizationException e) {
+//                trans.info().log(e);
+//            }
+//        }
         trans.info().printf("%d emails sent for %s", emailCount,batchEnv);
     }
     
-    private static class LastCred {
-        public Cred cred; 
-        public Date last;
-        
-        public LastCred(Cred cred, Date last) {
-            this.cred = cred;
-            this.last = last;
-        }
-        
-        // Reverse Sort (Oldest on top)
-        public static Comparator<LastCred> COMPARE = new Comparator<LastCred>() {
-            @Override
-            public int compare(LastCred o1, LastCred o2) {
-                return o2.last.compareTo(o1.last);
-            }
-        };
+    
+    private static class CSVInfo implements CSV.Visitor {
+    	private PrintStream out;
+    	private Set<String> unsupported;
+    	private NotifyCredVisitor credv;
+    	private List<LastCred> llc;
+    	
+    	public CSVInfo(PrintStream out) {
+    		this.out = out;
+    		credv = new NotifyCredVisitor(llc = new ArrayList<>());
+    	}
+    	
+		@Override
+		public void visit(List<String> row) throws IOException, CadiException {
+			
+			switch(row.get(0)) {
+			   case NotifyCredVisitor.SUPPORTS:
+				   credv.visit(row);
+				   break;
+			   default:
+				   if(unsupported==null) {
+					   unsupported = new HashSet<String>();
+				   }
+				   if(!unsupported.contains(row.get(0))) {
+					   unsupported.add(row.get(0));
+					   out.println("Unsupported Type: " + row.get(0));
+				   }
+			}
+		}
+    }
+    
+    private static class Contact {
+    	public List<String> contacts;
+		private List<UserRole> owners;
+    	
+    	public Contact(final String ns) {
+    		contacts = new ArrayList<>();
+    		loadFromNS(ns);
+    	}
+    	
+    	public void loadFromNS(final String ns) {
+    		owners = UserRole.getByRole().get(ns+".owner");
+    	}
+    }
+    
+    private static class LastCred extends Contact {
+    	public final String id;
+    	public final int type;
+        public final Date expires;
         
-        public String toString() {
-            return Chrono.dateTime(last) + cred.toString();
-        }
+        public LastCred(final String id, final String ns, final int type, final Date expires) {
+			super(ns);
+			this.id = id;
+			this.type = type;
+			this.expires = expires;
+		}
+    }
+    
+    private static class NotifyCredVisitor implements CSV.Visitor {
+    	public static final String SUPPORTS = "cred";
+		private final List<LastCred> lastCred;
+    	
+    	public NotifyCredVisitor(final List<LastCred> lastCred) {
+    		this.lastCred = lastCred;
+    	}
+    	
+		@Override
+		public void visit(List<String> row) throws IOException, CadiException {
+			 try {
+				lastCred.add(new LastCred(
+					row.get(1), 
+					row.get(2),
+					Integer.parseInt(row.get(3)), 
+					Chrono.dateOnlyFmt.parse(row.get(4))
+					)
+				);
+			} catch (NumberFormatException | ParseException e) {
+				throw new CadiException(e);
+			}
+		}
     }
     
     @Override
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java
index 22961f61..d35cfd34 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java
@@ -52,7 +52,7 @@ public class Remove extends Batch {
         trans.info().log("Starting Connection Process");
         
         noAvg = env.newTransNoAvg();
-        noAvg.setUser(new BatchPrincipal("batch:ExpiringP2"));
+        noAvg.setUser(new BatchPrincipal("batch:RemoveExpired"));
 
         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
         try {
@@ -77,7 +77,8 @@ public class Remove extends Batch {
 
         // Create Intermediate Output 
         File logDir = new File(logDir());
-        File expired = new File(logDir,"Expired"+Chrono.dateOnlyStamp()+".csv");
+        
+        File expired = new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv");
         CSV expiredCSV = new CSV(expired);
         try {
         	final StringBuilder sb = cqlBatch.begin();
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java
index f308e226..a670f37d 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java
@@ -73,10 +73,17 @@ public class Upload extends Batch {
 
 				if(file.exists()) {
 					count=batchCnt=0;
+					boolean justOne = false;
 					try {
 						BufferedReader br = new BufferedReader(new FileReader(file));
 						try {
 							while((line=br.readLine())!=null) {
+								if(line.length()>5000) {
+									if(query.length()>0) {
+										applyBatch(query);
+										justOne=true;
+									}
+								}
 								if(query.length()==0) {
 									query.append("BEGIN BATCH\n");
 								}
@@ -104,11 +111,12 @@ public class Upload extends Batch {
 								addField(feed,fldcnt,array,sb);
 								query.append(build(feed, array));
 								
-								if((++count % BATCH_LENGTH)==0) {
+								if((++count % BATCH_LENGTH)==0 || justOne) {
 									applyBatch(query);
+									justOne=false;
 								}
 							}
-							if((count % BATCH_LENGTH)!=0) {
+							if(query.length()>0) {
 								applyBatch(query);
 							}
 							
@@ -279,17 +287,20 @@ public class Upload extends Batch {
 	}
 
 	private void applyBatch(StringBuilder query) {
-		query.append("APPLY BATCH;");
-		ResultSet rv = session.execute(query.toString());
-		if(rv.wasApplied()) {
-			System.out.print('.');
-			if((++batchCnt % 60)==0) {
-				System.out.println();
+		try {
+			query.append("APPLY BATCH;");
+			ResultSet rv = session.execute(query.toString());
+			if(rv.wasApplied()) {
+				System.out.print('.');
+				if((++batchCnt % 60)==0) {
+					System.out.println();
+				}
+			} else {
+				System.out.print("Data NOT APPLIED");
 			}
-		} else {
-			System.out.print("Data NOT APPLIED");
+		} finally {
+			query.setLength(0);
 		}
-		query.setLength(0);
 	}