Expire, Remove Batch, restore

Issue-ID: AAF-641
Change-Id: I06560c8252ed27bd474ea140c6d9cacef88ef1b9
Signed-off-by: Instrumental <jonathan.gathman@att.com>

8 files changed, 1217 insertions, 212 deletions

diff --git a/auth/auth-batch/.gitignore b/auth/auth-batch/.gitignore
index 9f0fc218..6e368115 100644
--- a/auth/auth-batch/.gitignore
+++ b/auth/auth-batch/.gitignore
@@ -2,3 +2,6 @@
 /target/
 /.classpath
 /.project
+/logs
+NotifyCredExpiringOrig.java
+/*.dat
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
index 2f6ed415..fa49c290 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
@@ -32,6 +32,10 @@ import java.util.TreeMap;
 
 import org.onap.aaf.auth.dao.cass.CredDAO;
 import org.onap.aaf.auth.dao.hl.Question;
+<<<<<<< HEAD
+=======
+import org.onap.aaf.cadi.util.CSV;
+>>>>>>> a6baa197... Expire, Remove Batch, restore
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
 import org.onap.aaf.misc.env.Trans;
@@ -67,6 +71,13 @@ public class Cred  {
             this.other = other;
             this.written = new Date(written);
         }
+<<<<<<< HEAD
+=======
+        
+        public String toString() {
+        	return expires.toString() + ": " + type;
+        }
+>>>>>>> a6baa197... Expire, Remove Batch, restore
     }
     
     public Date last(final int ... types) {
@@ -125,20 +136,16 @@ public class Cred  {
         try {
             Iterator<Row> iter = results.iterator();
             Row row;
-            int type; // for filtering
-            String id;
             tt = trans.start("Load Credentials", Env.SUB);
             try {
                 while (iter.hasNext()) {
                     ++count;
                     row = iter.next();
-                    id = row.getString(0);
-                    type = row.getInt(1);
+                    int type = row.getInt(1);
                     if (types.length>0) { // filter by types, if requested
                         boolean quit = true;
                         for (int t : types) {
                             if (t==type) {
-                                quit=false;
                                 break;
                             }
                         }
@@ -146,27 +153,7 @@ public class Cred  {
                             continue;
                         }
                     }
-                    Cred cred = data.get(id);
-                    if (cred==null) {
-                        cred = new Cred(id);
-                        data.put(id, cred);
-                    }
-                    cred.instances.add(new Instance(type, row.getTimestamp(2), row.getInt(3), row.getLong(4)/1000));
-                    
-                    List<Cred> lscd = byNS.get(cred.ns);
-                    if (lscd==null) {
-                        byNS.put(cred.ns, (lscd=new ArrayList<>()));
-                    }
-                    boolean found = false;
-                    for (Cred c : lscd) {
-                        if (c.id.equals(cred.id)) {
-                            found=true;
-                            break;
-                        }
-                    }
-                    if (!found) {
-                        lscd.add(cred);
-                    }
+                    add(row.getString(0), row.getInt(1),row.getTimestamp(2),row.getInt(3),row.getLong(4));
                 }
             } finally {
                 tt.done();
@@ -175,8 +162,44 @@ public class Cred  {
             trans.info().log("Found",count,"creds");
         }
     }
+<<<<<<< HEAD
     
     /** 
+=======
+
+    public static void add(
+    		final String id, 
+    		final int type,
+    		final Date timestamp,
+    		final int other,
+    		final long written
+    		) {
+        Cred cred = data.get(id);
+        if (cred==null) {
+            cred = new Cred(id);
+            data.put(id, cred);
+        }
+        cred.instances.add(new Instance(type, timestamp, other, written/1000));
+        
+        List<Cred> lscd = byNS.get(cred.ns);
+        if (lscd==null) {
+            byNS.put(cred.ns, (lscd=new ArrayList<>()));
+        }
+        boolean found = false;
+        for (Cred c : lscd) {
+            if (c.id.equals(cred.id)) {
+                found=true;
+                break;
+            }
+        }
+        if (!found) {
+            lscd.add(cred);
+        }
+	}
+
+
+	/** 
+>>>>>>> a6baa197... Expire, Remove Batch, restore
      * Count entries in Cred data.
      * Note, as opposed to other methods, need to load the whole cred table for the Types.
      * @param numbuckets 
@@ -273,7 +296,28 @@ public class Cred  {
 
     }
     
+<<<<<<< HEAD
     public String toString() {
+=======
+    public void row(final CSV.Writer csvw, final Instance inst) {
+    	csvw.row("cred",id,ns,Integer.toString(inst.type),Chrono.dateOnlyStamp(inst.expires),inst.expires.getTime());
+    }
+
+
+    public static void row(StringBuilder sb, List<String> row) {
+    	sb.append("DELETE from authz.cred WHERE id='");
+    	sb.append(row.get(1));
+    	sb.append("' AND type=");
+    	sb.append(Integer.parseInt(row.get(3)));
+    	// Note: We have to work with long, because Expires is part of Key... can't easily do date.
+    	sb.append(" AND expires=dateof(maxtimeuuid(");
+    	sb.append(row.get(5));
+    	sb.append("));\n");
+	}
+
+
+	public String toString() {
+>>>>>>> a6baa197... Expire, Remove Batch, restore
         StringBuilder sb = new StringBuilder(id);
         sb.append('[');
         for (Instance i : instances) {
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
index 762f6c66..c9fc658a 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
@@ -296,4 +296,22 @@ public class UserRole implements Cloneable, CacheChange.Data  {
         cache.resetLocalData();
     }
 
+<<<<<<< HEAD
+=======
+    public void row(final CSV.Writer csvw) {
+    	csvw.row("ur",user(),ns(),rname(),Chrono.dateOnlyStamp(expires()));
+    }
+    
+    public static void row(StringBuilder sb, List<String> row) {
+    	sb.append("DELETE from authz.user_role WHERE user='");
+    	sb.append(row.get(1));
+    	sb.append("' AND role='");
+    	sb.append(row.get(2));
+    	sb.append('.');
+    	sb.append(row.get(3));
+    	sb.append("';\n");
+    }
+    
+
+>>>>>>> a6baa197... Expire, Remove Batch, restore
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java
new file mode 100644
index 00000000..95fe3c0e
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java
@@ -0,0 +1,119 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+
+package org.onap.aaf.auth.helpers;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.cert.X509Certificate;
+import java.util.Iterator;
+import java.util.List;
+
+import org.onap.aaf.cadi.Hash;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.Trans;
+import org.onap.aaf.misc.env.util.Chrono;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
+
+public class X509 {
+    public final String ca,id,x500,x509;
+    public ByteBuffer serial;
+    
+    public X509(String ca, String id, String x500, String x509, ByteBuffer serial) {
+    	this.ca = ca;
+    	this.id = id;
+    	this.x500 = x500;
+    	this.x509 = x509;
+    	this.serial = serial;
+    }
+    
+
+    public static void load(Trans trans, Session session, Visitor<X509> visitor) {
+        load(trans,session,"select ca, id, x500, x509, serial from authz.x509;", visitor);
+    }
+
+    private static void load(Trans trans, Session session, String query, Visitor<X509> visitor) {
+        trans.info().log( "query: " + query );
+        TimeTaken tt = trans.start("Read Roles", Env.REMOTE);
+       
+        ResultSet results;
+        try {
+            Statement stmt = new SimpleStatement( query );
+            results = session.execute(stmt);
+        } finally {
+            tt.done();
+        }
+
+        int count = 0;
+        try {
+            Iterator<Row> iter = results.iterator();
+            Row row;
+            tt = trans.start("Load X509s", Env.SUB);
+            try {
+                while (iter.hasNext()) {
+                	++count;
+                    row = iter.next();
+                    visitor.visit(new X509(row.getString(0),row.getString(1), row.getString(2),row.getString(3),row.getBytes(4)));
+                }
+            } finally {
+                tt.done();
+            }
+        } finally {
+            trans.info().log("Found",count,"X509 Certificates");
+        }
+    }
+    
+    public static long count(Trans trans, Session session) {
+        String query = "select count(*) from authz.x509 LIMIT 1000000;";
+        trans.info().log( "query: " + query );
+        TimeTaken tt = trans.start("Count x509s", Env.REMOTE);
+        ResultSet results;
+        try {
+            Statement stmt = new SimpleStatement(query).setReadTimeoutMillis(12000);
+            results = session.execute(stmt);
+            return results.one().getLong(0);
+        } finally {
+            tt.done();
+        }
+    }
+    
+
+	public void row(CSV.Writer cw, X509Certificate x509Cert) throws IOException {
+		cw.row("x509",ca,Hash.toHex(serial.array()),Chrono.dateOnlyStamp(x509Cert.getNotAfter()),x500);
+	}
+
+
+	public static void row(StringBuilder sb, List<String> row) throws IOException {
+    	sb.append("DELETE from authz.x509 WHERE ca='");
+    	sb.append(row.get(1));
+    	sb.append("' AND serial=");
+    	sb.append(row.get(2));
+    	sb.append(";\n");
+	}
+
+}
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java
new file mode 100644
index 00000000..6974a5db
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java
@@ -0,0 +1,299 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+
+package org.onap.aaf.auth.reports;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.onap.aaf.auth.Batch;
+import org.onap.aaf.auth.dao.cass.CredDAO;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.helpers.Cred;
+import org.onap.aaf.auth.helpers.Cred.Instance;
+import org.onap.aaf.auth.helpers.UserRole;
+import org.onap.aaf.auth.helpers.Visitor;
+import org.onap.aaf.auth.helpers.X509;
+import org.onap.aaf.auth.org.ExpireRange;
+import org.onap.aaf.auth.org.ExpireRange.Range;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.configure.Factory;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+
+
+public class Expiring extends Batch {
+    
+	private static final String CSV = ".csv";
+	private static final String INFO = "info";
+	private static final String EXPIRED_OWNERS = "ExpiredOwners";
+	private int minOwners;
+	private Map<String, CSV.Writer> writerList;
+	private File logDir;
+	private ExpireRange expireRange;
+	private Date deleteDate;
+	
+	public Expiring(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+        super(trans.env());
+        trans.info().log("Starting Connection Process");
+        
+        TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+        try {
+            TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
+            try {
+                session = cluster.connect();
+            } finally {
+                tt.done();
+            }
+            
+            // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
+            Cred.load(trans, session);
+            UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor());
+
+            minOwners=1;
+
+            // Create Intermediate Output 
+            writerList = new HashMap<>();
+            logDir = new File(logDir());
+            logDir.mkdirs();
+            
+            expireRange = new ExpireRange(trans.env().access());
+            String sdate = Chrono.dateOnlyStamp(expireRange.now);
+            for( List<Range> lr : expireRange.ranges.values()) {
+            	for(Range r : lr ) {
+            		if(writerList.get(r.name())==null) {
+                    	File file = new File(logDir,r.name() + sdate +CSV);
+                    	CSV csv = new CSV(file);
+                    	CSV.Writer cw = csv.writer(false);
+                    	cw.row(INFO,r.name(),Chrono.dateOnlyStamp(expireRange.now),r.reportingLevel());
+                    	writerList.put(r.name(),cw);
+                    	if("Delete".equals(r.name())) {
+                    		deleteDate = r.getStart();
+                    	}
+            		}
+            	}
+            }
+            
+        } finally {
+            tt0.done();
+        }
+    }
+
+    @Override
+    protected void run(AuthzTrans trans) {
+		try {
+			File file = new File(logDir, EXPIRED_OWNERS + Chrono.dateOnlyStamp(expireRange.now) + CSV);
+			final CSV ownerCSV = new CSV(file);
+
+			Map<String, Set<UserRole>> owners = new TreeMap<String, Set<UserRole>>();
+			trans.info().log("Process UserRoles");
+			UserRole.load(trans, session, UserRole.v2_0_11, new Visitor<UserRole>() {
+				@Override
+				public void visit(UserRole ur) {
+					// Cannot just delete owners, unless there is at least one left. Process later
+					if ("owner".equals(ur.rname())) {
+						Set<UserRole> urs = owners.get(ur.role());
+						if (urs == null) {
+							urs = new HashSet<UserRole>();
+							owners.put(ur.role(), urs);
+						}
+						urs.add(ur);
+					} else {
+						writeAnalysis(trans,ur);
+					}
+				}
+			});
+
+			// Now Process Owners, one owner Role at a time, ensuring one is left,
+			// preferably
+			// a good one. If so, process the others as normal. Otherwise, write
+			// ExpiredOwners
+			// report
+			if (!owners.values().isEmpty()) {
+				// Lazy Create file
+				CSV.Writer expOwner = null;
+				try {
+					for (Set<UserRole> sur : owners.values()) {
+						int goodOwners = 0;
+						for (UserRole ur : sur) {
+							if (ur.expires().after(expireRange.now)) {
+								++goodOwners;
+							}
+						}
+
+						for (UserRole ur : sur) {
+							if (goodOwners >= minOwners) {
+								writeAnalysis(trans, ur);
+							} else {
+								if (expOwner == null) {
+									expOwner = ownerCSV.writer();
+									expOwner.row(INFO,EXPIRED_OWNERS,Chrono.dateOnlyStamp(expireRange.now),2);
+								}
+								expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
+							}
+						}
+					}
+				} finally {
+					expOwner.close();
+				}
+			}
+			
+			trans.info().log("Checking for Expired Credentials");
+			
+			for (Cred cred : Cred.data.values()) {
+		    	List<Instance> linst = cred.instances;
+		    	if(linst!=null) {
+			    	Instance lastBath = null;
+			    	for(Instance inst : linst) {
+			    		// Special Behavior: only eval the LAST Instance
+		    			if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
+				        	if(deleteDate!=null && inst.expires.before(deleteDate)) {
+				        		writeAnalysis(trans, cred, inst); // will go to Delete
+				    		} else if(lastBath==null || lastBath.expires.before(inst.expires)) {
+		    					lastBath = inst;
+		    				}
+		    			} else {
+		    				writeAnalysis(trans, cred, inst);
+		    			}
+			    	}
+			    	if(lastBath!=null) {
+			    		writeAnalysis(trans, cred, lastBath);
+			    	}
+		    	}
+			}
+			
+			trans.info().log("Checking for Expired X509s");
+			X509.load(trans, session, new Visitor<X509>() {
+				@Override
+				public void visit(X509 x509) {
+					try {
+						for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
+							writeAnalysis(trans, x509, (X509Certificate)cert);
+						}
+					} catch (CertificateException | IOException e) {
+						trans.error().log(e, "Error Decrypting X509");
+					}
+					
+				}
+			});
+		} catch (FileNotFoundException e) {
+			trans.info().log(e);
+		}
+	}
+    
+ 
+	private void writeAnalysis(AuthzTrans trans, UserRole ur) {
+		Range r = expireRange.getRange("ur", ur.expires());
+		if(r!=null) {
+			CSV.Writer cw = writerList.get(r.name());
+			if(cw!=null) {
+				ur.row(cw);
+			}
+		}
+	}
+    
+    private void writeAnalysis(AuthzTrans trans, Cred cred, Instance inst) {
+    	if(cred!=null && inst!=null) {
+			Range r = expireRange.getRange("cred", inst.expires);
+			if(r!=null) {
+				CSV.Writer cw = writerList.get(r.name());
+				if(cw!=null) {
+					cred.row(cw,inst);
+				}
+			}
+    	}
+	}
+
+    private void writeAnalysis(AuthzTrans trans, X509 x509, X509Certificate x509Cert) throws IOException {
+		Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
+		if(r!=null) {
+			CSV.Writer cw = writerList.get(r.name());
+			if(cw!=null) {
+				x509.row(cw,x509Cert);
+			}
+		}
+	}
+
+    /*
+    private String[] contacts(final AuthzTrans trans, final String ns, final int levels) {
+    	List<UserRole> owners = UserRole.getByRole().get(ns+".owner");
+    	List<UserRole> current = new ArrayList<>();
+    	for(UserRole ur : owners) {
+    		if(expireRange.now.before(ur.expires())) {
+    			current.add(ur);
+    		}
+    	}
+    	if(current.isEmpty()) {
+    		trans.warn().log(ns,"has no current owners");
+    		current = owners;
+    	}
+    	
+    	List<String> email = new ArrayList<>();
+    	for(UserRole ur : current) {
+    		Identity id;
+    		int i=0;
+    		boolean go = true;
+    		try {
+    			id = org.getIdentity(trans, ur.user());
+        		do {
+	    			if(id!=null) {
+						email.add(id.email());
+						if(i<levels) {
+							id = id.responsibleTo();
+						} else {
+							go = false;
+						}
+	    			} else {
+	    				go = false;
+	    			}
+        		} while(go);
+			} catch (OrganizationException e) {
+				trans.error().log(e);
+			}
+    	}
+    	
+    	return email.toArray(new String[email.size()]);
+    }
+*/
+    
+	@Override
+    protected void _close(AuthzTrans trans) {
+        session.close();
+    	for(CSV.Writer cw : writerList.values()) {
+    		cw.close();
+    	}
+    }
+
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
index b4631f66..f202f817 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
@@ -21,17 +21,25 @@
 
 package org.onap.aaf.auth.update;
 
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+<<<<<<< HEAD
+=======
+import java.util.Set;
+import java.util.TreeMap;
+>>>>>>> a6baa197... Expire, Remove Batch, restore
 
 import org.onap.aaf.auth.Batch;
 import org.onap.aaf.auth.BatchPrincipal;
@@ -42,6 +50,7 @@ import org.onap.aaf.auth.dao.cass.CredDAO;
 import org.onap.aaf.auth.dao.hl.Question;
 import org.onap.aaf.auth.env.AuthzTrans;
 import org.onap.aaf.auth.helpers.Cred;
+import org.onap.aaf.auth.helpers.NS;
 import org.onap.aaf.auth.helpers.Notification;
 import org.onap.aaf.auth.helpers.UserRole;
 import org.onap.aaf.auth.helpers.Notification.TYPE;
@@ -50,7 +59,12 @@ import org.onap.aaf.auth.org.EmailWarnings;
 import org.onap.aaf.auth.org.Organization;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.auth.org.OrganizationFactory;
+<<<<<<< HEAD
 import org.onap.aaf.auth.org.Organization.Identity;
+=======
+import org.onap.aaf.cadi.CadiException;
+import org.onap.aaf.cadi.util.CSV;
+>>>>>>> a6baa197... Expire, Remove Batch, restore
 import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
@@ -74,6 +88,8 @@ public class NotifyCredExpiring extends Batch {
     private final PrintStream ps;
     private final AuthzTrans noAvg;
     private String supportEmailAddr;
+	private CSV csv;
+	private CSVInfo csvInfo;
 
     public NotifyCredExpiring(AuthzTrans trans) throws APIException, IOException, OrganizationException {
         super(trans.env());
@@ -87,9 +103,6 @@ public class NotifyCredExpiring extends Batch {
         noAvg = env.newTransNoAvg();
         noAvg.setUser(new BatchPrincipal("batch:NotifyCredExpiring"));
         
-        if ((supportEmailAddr = env.getProperty("mailFromUserId"))==null) {
-            throw new APIException("mailFromUserId property must be set");
-        }
         if (isDryRun()) {
             email = new EmailPrint();
             maxEmails=3;
@@ -106,10 +119,37 @@ public class NotifyCredExpiring extends Batch {
         email.signature("Sincerely,\nAAF Team (Our MOTS# 22830)\n"
                 + "https://wiki.web.att.com/display/aaf/Contact+Us\n"
                 + "(Use 'Other Misc Requests (TOPS)')");
-
-        Cred.load(trans, session,CredDAO.BASIC_AUTH, CredDAO.BASIC_AUTH_SHA256);
+        
+        boolean quit = false;
+        if(args().length<1) {
+        	System.err.println("Need CSV formatted Expiring Report");
+        	quit = true;
+        } else {
+        	File f = new File(logDir(),args()[0]);
+        	System.out.println("Reading " + f.getCanonicalPath());
+        	csv = new CSV(f);
+        }
+        
+        if(args().length<2) {
+        	System.err.println("Need Email Template");
+        	//quit = true;
+        }
+        if(quit) {
+        	System.exit(2);
+        }
+        
+        csvInfo = new CSVInfo(System.err);
+        try {
+			csv.visit(csvInfo);
+		} catch (CadiException e) {
+			throw new APIException(e);
+		}
+        
         Notification.load(trans, session, Notification.v2_0_18);
+<<<<<<< HEAD
         UserRole.load(trans, session, UserRole.v2_0_11);
+=======
+>>>>>>> a6baa197... Expire, Remove Batch, restore
         
         ps = new PrintStream(new FileOutputStream(logDir() + "/email"+Chrono.dateOnlyStamp()+".log",true));
         ps.printf("### Approval Notify %s for %s%s\n",Chrono.dateTime(),batchEnv,dryRun?", DryRun":"");
@@ -118,199 +158,244 @@ public class NotifyCredExpiring extends Batch {
     @Override
     protected void run(AuthzTrans trans) {
         
-        EmailWarnings ewp = org.emailWarningPolicy();
-        long now = System.currentTimeMillis();
-        Date early = new Date(now+(ewp.credExpirationWarning()*2)); // 2 months back
-        Date must = new Date(now+ewp.credExpirationWarning()); // 1 months back
-        Date critical = new Date(now+ewp.emailUrgentWarning()); // 1 week
-        Date within2Weeks = new Date(now+604800000 * 2);
-        Date withinLastWeek = new Date(now-604800000);
-        Date tooLate = new Date(now);
-        
         // Temp structures
-        Map<String,Cred> lastCred = new HashMap<>();
         Map<String,List<LastCred>> ownerCreds = new TreeMap<>();
-        Date last;
         
 
         List<LastCred> noOwner = new ArrayList<>();
         ownerCreds.put(UNKNOWN_ID,noOwner);
-
-        // Get a list of ONLY the ones needing email by Owner
-        for (Entry<String, List<Cred>> es : Cred.byNS.entrySet()) {
-            lastCred.clear();
-            for (Cred c : es.getValue()) {
-                last = c.last(CredDAO.BASIC_AUTH,CredDAO.BASIC_AUTH_SHA256);
-                if (last!=null && last.after(tooLate) && last.before(early)) {
-                    List<UserRole> ownerURList = UserRole.getByRole().get(es.getKey()+".owner");
-                    if (ownerURList!=null) {
-                        for (UserRole ur:ownerURList) {
-                            String owner = ur.user();
-                            List<LastCred> llc = ownerCreds.get(owner);
-                            if (llc==null) {
-                                ownerCreds.put(owner, (llc=new ArrayList<>()));
-                            }
-                            llc.add(new LastCred(c,last));
-                        }
-                    } else {
-                        noOwner.add(new LastCred(c,last));
-                    }
-                }
-            }
-        }
-        
-        boolean bCritical,bNormal,bEarly;
         int emailCount=0;
-        Message msg = new Message();
-        Notification ownNotf;
-        StringBuilder logMessage = new StringBuilder();
-        for (Entry<String,List<LastCred>> es : ownerCreds.entrySet()) {
-            String owner = es.getKey();
-            boolean header = true;
-            try {
-                Organization org = OrganizationFactory.obtain(env, owner);
-                Identity user = org.getIdentity(noAvg, owner);
-                if (!UNKNOWN_ID.equals(owner) && user==null) {
-                    ps.printf("Invalid Identity: %s\n", owner);
-                } else {
-                    logMessage.setLength(0);
-                    if (maxEmails>emailCount) {
-                        bCritical=bNormal=bEarly = false;
-                        email.clear();
-                        msg.clear();
-                        email.addTo(user==null?supportEmailAddr:user.email());
-
-                        ownNotf = Notification.get(es.getKey(),TYPE.CN);
-                        if (ownNotf==null) {
-                            ownNotf = Notification.create(user==null?UNKNOWN_ID:user.fullID(), TYPE.CN);
-                        }
-                        last = ownNotf.last;
-                        // Get Max ID size for formatting purposes
-                        int length = AAF_INSTANTIATED_MECHID.length();
-                        for (LastCred lc : es.getValue()) {
-                            length = Math.max(length, lc.cred.id.length());
-                        }
-                        String id_exp_fmt = "\t%-"+length+"s  %15s  %s";
 
-                        Collections.sort(es.getValue(),LastCred.COMPARE);
-                        for (LastCred lc : es.getValue()) {
-                            if (lc.last.after(must) && lc.last.before(early) && 
-                                (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
-                                if (!bEarly && header) {
-                                    msg.line("\tThe following are friendly 2 month reminders, just in case you need to schedule your updates early.  "
-                                            + "You will be reminded next month\n");
-                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
-                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
-                                    header = false;
-                                }
-                                bEarly = true;
-                            } else if (lc.last.after(critical) && lc.last.before(must) && 
-                                    (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
-                                if (!bNormal) {
-                                    boolean last2wks = lc.last.before(within2Weeks);
-                                    if (last2wks) {
-                                        try {
-                                            Identity supvsr = user.responsibleTo();
-                                            email.addCC(supvsr.email());
-                                        } catch (OrganizationException e) {
-                                            trans.error().log(e, "Supervisor cannot be looked up");
-                                        }
-                                    }
-                                    if (header) {
-                                        msg.line("\tIt is now important for you to update Passwords all all configurations using them for the following.\n" +
-                                                (last2wks?"\tNote: Your Supervisor is CCd\n":"\tNote: Your Supervisor will be notified if this is not being done before the last 2 weeks\n"));
-                                        msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
-                                        msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
-                                    }
-                                    header = false;
-                                }
-                                bNormal=true;
-                            } else if (lc.last.after(tooLate) && lc.last.before(critical)) { // Email Every Day, with Supervisor
-                                if (!bCritical && header) {
-                                    msg.line("\t!!! WARNING: These Credentials will expire in LESS THAN ONE WEEK !!!!\n" +
-                                             "\tYour supervisor is added to this Email\n");
-                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
-                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
-                                    header = false;
-                                }
-                                bCritical = true;
-                                try {
-                                    if (user!=null) {
-                                        Identity supvsr = user.responsibleTo();
-                                        if (supvsr!=null) {
-                                            email.addCC(supvsr.email());
-                                            supvsr = supvsr.responsibleTo();
-                                            if (supvsr!=null) {
-                                                email.addCC(supvsr.email());
-                                            }
-                                        }
-                                    }
-                                } catch (OrganizationException e) {
-                                    trans.error().log(e, "Supervisor cannot be looked up");
-                                }
-                            }
-                            if (bEarly || bNormal || bCritical) {
-                                if (logMessage.length()==0) {
-                                    logMessage.append("NotifyCredExpiring");
-                                }
-                                logMessage.append("\n\t");
-                                logMessage.append(lc.cred.id);
-                                logMessage.append('\t');
-                                logMessage.append(Chrono.dateOnlyStamp(lc.last));
-                                msg.line(id_exp_fmt, lc.cred.id, Chrono.dateOnlyStamp(lc.last)+"     ",env.getProperty(GUI_URL)+"/creddetail?ns="+Question.domain2ns(lc.cred.id));
-                            }
-                        }
-                        
-                        if (bEarly || bNormal || bCritical) {
-                            msg.line(LINE);
-                            msg.line("Why are you receiving this Notification?\n");
-                                msg.line("You are the listed owner of one or more AAF Namespaces. ASPR requires that those responsible for "
-                                        + "applications and their access review them regularly for accuracy.  The AAF WIKI page for AT&T is https://wiki.web.att.com/display/aaf.  "
-                                        + "You might like https://wiki.web.att.com/display/aaf/AAF+in+a+Nutshell.  More detailed info regarding questions of being a Namespace Owner is available at https://wiki.web.att.com/pages/viewpage.action?pageId=594741363\n");
-                                msg.line("You may view the Namespaces you listed as Owner for in this AAF Env by viewing the following webpage:\n");
-                                msg.line("   %s/ns\n\n",env.getProperty(GUI_URL));
-                            email.msg(msg);
-                            Result<Void> rv = email.exec(trans, org,"");
-                            if (rv.isOK()) {
-                                ++emailCount;
-                                if (!isDryRun()) {
-                                    ownNotf.update(noAvg, session, false);
-                                    // SET LastNotification
-                                }
-                                email.log(ps,logMessage.toString());
-                            } else {
-                                trans.error().log(rv.errorString());
-                            }
-                        }
-                    }
-                }
-            } catch (OrganizationException e) {
-                trans.info().log(e);
-            }
-        }
+//        // Get a list of ONLY the ones needing email by Owner
+//        for (Entry<String, List<Cred>> es : Cred.byNS.entrySet()) {
+//            for (Cred c : es.getValue()) {
+//                List<UserRole> ownerURList = UserRole.getByRole().get(es.getKey()+".owner");
+//                if (ownerURList!=null) {
+//                    for (UserRole ur:ownerURList) {
+//                        String owner = ur.user();
+//                        List<LastCred> llc = ownerCreds.get(owner);
+//                        if (llc==null) {
+//                            ownerCreds.put(owner, (llc=new ArrayList<>()));
+//                        }
+//                        llc.add(new LastCred(c,last));
+//                    }
+//                } else {
+//                    noOwner.add(new LastCred(c,last));
+//                }
+//            }
+//        }
+//        
+//        boolean bCritical,bNormal,bEarly;
+//        Message msg = new Message();
+//        Notification ownNotf;
+//        StringBuilder logMessage = new StringBuilder();
+//        for (Entry<String,List<LastCred>> es : ownerCreds.entrySet()) {
+//            String owner = es.getKey();
+//            boolean header = true;
+//            try {
+//                Organization org = OrganizationFactory.obtain(env, owner);
+//                Identity user = org.getIdentity(noAvg, owner);
+//                if (!UNKNOWN_ID.equals(owner) && user==null) {
+//                    ps.printf("Invalid Identity: %s\n", owner);
+//                } else {
+//                    logMessage.setLength(0);
+//                    if (maxEmails>emailCount) {
+//                        bCritical=bNormal=bEarly = false;
+//                        email.clear();
+//                        msg.clear();
+//                        email.addTo(user==null?supportEmailAddr:user.email());
+//
+//                        ownNotf = Notification.get(es.getKey(),TYPE.CN);
+//                        if (ownNotf==null) {
+//                            ownNotf = Notification.create(user==null?UNKNOWN_ID:user.fullID(), TYPE.CN);
+//                        }
+//                        last = ownNotf.last;
+//                        // Get Max ID size for formatting purposes
+//                        int length = AAF_INSTANTIATED_MECHID.length();
+//                        for (LastCred lc : es.getValue()) {
+//                            length = Math.max(length, lc.cred.id.length());
+//                        }
+//                        String id_exp_fmt = "\t%-"+length+"s  %15s  %s";
+//
+//                        Collections.sort(es.getValue(),LastCred.COMPARE);
+//                        for (LastCred lc : es.getValue()) {
+//                            if (lc.last.after(must) && lc.last.before(early) && 
+//                                (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
+//                                if (!bEarly && header) {
+//                                    msg.line("\tThe following are friendly 2 month reminders, just in case you need to schedule your updates early.  "
+//                                            + "You will be reminded next month\n");
+//                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
+//                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
+//                                    header = false;
+//                                }
+//                                bEarly = true;
+//                            } else if (lc.last.after(critical) && lc.last.before(must) && 
+//                                    (ownNotf.last==null || ownNotf.last.before(withinLastWeek))) {
+//                                if (!bNormal) {
+//                                    boolean last2wks = lc.last.before(within2Weeks);
+//                                    if (last2wks) {
+//                                        try {
+//                                            Identity supvsr = user.responsibleTo();
+//                                            email.addCC(supvsr.email());
+//                                        } catch (OrganizationException e) {
+//                                            trans.error().log(e, "Supervisor cannot be looked up");
+//                                        }
+//                                    }
+//                                    if (header) {
+//                                        msg.line("\tIt is now important for you to update Passwords all all configurations using them for the following.\n" +
+//                                                (last2wks?"\tNote: Your Supervisor is CCd\n":"\tNote: Your Supervisor will be notified if this is not being done before the last 2 weeks\n"));
+//                                        msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
+//                                        msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
+//                                    }
+//                                    header = false;
+//                                }
+//                                bNormal=true;
+//                            } else if (lc.last.after(tooLate) && lc.last.before(critical)) { // Email Every Day, with Supervisor
+//                                if (!bCritical && header) {
+//                                    msg.line("\t!!! WARNING: These Credentials will expire in LESS THAN ONE WEEK !!!!\n" +
+//                                             "\tYour supervisor is added to this Email\n");
+//                                    msg.line(id_exp_fmt, AAF_INSTANTIATED_MECHID,EXPIRATION_DATE, QUICK_LINK);
+//                                    msg.line(id_exp_fmt, DASH_1, DASH_2, DASH_3);
+//                                    header = false;
+//                                }
+//                                bCritical = true;
+//                                try {
+//                                    if (user!=null) {
+//                                        Identity supvsr = user.responsibleTo();
+//                                        if (supvsr!=null) {
+//                                            email.addCC(supvsr.email());
+//                                            supvsr = supvsr.responsibleTo();
+//                                            if (supvsr!=null) {
+//                                                email.addCC(supvsr.email());
+//                                            }
+//                                        }
+//                                    }
+//                                } catch (OrganizationException e) {
+//                                    trans.error().log(e, "Supervisor cannot be looked up");
+//                                }
+//                            }
+//                            if (bEarly || bNormal || bCritical) {
+//                                if (logMessage.length()==0) {
+//                                    logMessage.append("NotifyCredExpiring");
+//                                }
+//                                logMessage.append("\n\t");
+//                                logMessage.append(lc.cred.id);
+//                                logMessage.append('\t');
+//                                logMessage.append(Chrono.dateOnlyStamp(lc.last));
+//                                msg.line(id_exp_fmt, lc.cred.id, Chrono.dateOnlyStamp(lc.last)+"     ",env.getProperty(GUI_URL)+"/creddetail?ns="+Question.domain2ns(lc.cred.id));
+//                            }
+//                        }
+//                        
+//                        if (bEarly || bNormal || bCritical) {
+//                            msg.line(LINE);
+//                            msg.line("Why are you receiving this Notification?\n");
+//                                msg.line("You are the listed owner of one or more AAF Namespaces. ASPR requires that those responsible for "
+//                                        + "applications and their access review them regularly for accuracy.  The AAF WIKI page for AT&T is https://wiki.web.att.com/display/aaf.  "
+//                                        + "You might like https://wiki.web.att.com/display/aaf/AAF+in+a+Nutshell.  More detailed info regarding questions of being a Namespace Owner is available at https://wiki.web.att.com/pages/viewpage.action?pageId=594741363\n");
+//                                msg.line("You may view the Namespaces you listed as Owner for in this AAF Env by viewing the following webpage:\n");
+//                                msg.line("   %s/ns\n\n",env.getProperty(GUI_URL));
+//                            email.msg(msg);
+//                            Result<Void> rv = email.exec(trans, org,"");
+//                            if (rv.isOK()) {
+//                                ++emailCount;
+//                                if (!isDryRun()) {
+//                                    ownNotf.update(noAvg, session, false);
+//                                    // SET LastNotification
+//                                }
+//                                email.log(ps,logMessage.toString());
+//                            } else {
+//                                trans.error().log(rv.errorString());
+//                            }
+//                        }
+//                    }
+//                }
+//            } catch (OrganizationException e) {
+//                trans.info().log(e);
+//            }
+//        }
         trans.info().printf("%d emails sent for %s", emailCount,batchEnv);
     }
     
-    private static class LastCred {
-        public Cred cred; 
-        public Date last;
-        
-        public LastCred(Cred cred, Date last) {
-            this.cred = cred;
-            this.last = last;
-        }
-        
-        // Reverse Sort (Oldest on top)
-        public static Comparator<LastCred> COMPARE = new Comparator<LastCred>() {
-            @Override
-            public int compare(LastCred o1, LastCred o2) {
-                return o2.last.compareTo(o1.last);
-            }
-        };
+    
+    private static class CSVInfo implements CSV.Visitor {
+    	private PrintStream out;
+    	private Set<String> unsupported;
+    	private NotifyCredVisitor credv;
+    	private List<LastCred> llc;
+    	
+    	public CSVInfo(PrintStream out) {
+    		this.out = out;
+    		credv = new NotifyCredVisitor(llc = new ArrayList<>());
+    	}
+    	
+		@Override
+		public void visit(List<String> row) throws IOException, CadiException {
+			
+			switch(row.get(0)) {
+			   case NotifyCredVisitor.SUPPORTS:
+				   credv.visit(row);
+				   break;
+			   default:
+				   if(unsupported==null) {
+					   unsupported = new HashSet<String>();
+				   }
+				   if(!unsupported.contains(row.get(0))) {
+					   unsupported.add(row.get(0));
+					   out.println("Unsupported Type: " + row.get(0));
+				   }
+			}
+		}
+    }
+    
+    private static class Contact {
+    	public List<String> contacts;
+		private List<UserRole> owners;
+    	
+    	public Contact(final String ns) {
+    		contacts = new ArrayList<>();
+    		loadFromNS(ns);
+    	}
+    	
+    	public void loadFromNS(final String ns) {
+    		owners = UserRole.getByRole().get(ns+".owner");
+    	}
+    }
+    
+    private static class LastCred extends Contact {
+    	public final String id;
+    	public final int type;
+        public final Date expires;
         
-        public String toString() {
-            return Chrono.dateTime(last) + cred.toString();
-        }
+        public LastCred(final String id, final String ns, final int type, final Date expires) {
+			super(ns);
+			this.id = id;
+			this.type = type;
+			this.expires = expires;
+		}
+    }
+    
+    private static class NotifyCredVisitor implements CSV.Visitor {
+    	public static final String SUPPORTS = "cred";
+		private final List<LastCred> lastCred;
+    	
+    	public NotifyCredVisitor(final List<LastCred> lastCred) {
+    		this.lastCred = lastCred;
+    	}
+    	
+		@Override
+		public void visit(List<String> row) throws IOException, CadiException {
+			 try {
+				lastCred.add(new LastCred(
+					row.get(1), 
+					row.get(2),
+					Integer.parseInt(row.get(3)), 
+					Chrono.dateOnlyFmt.parse(row.get(4))
+					)
+				);
+			} catch (NumberFormatException | ParseException e) {
+				throw new CadiException(e);
+			}
+		}
     }
     
     @Override
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java
new file mode 100644
index 00000000..d35cfd34
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java
@@ -0,0 +1,124 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+
+package org.onap.aaf.auth.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.onap.aaf.auth.Batch;
+import org.onap.aaf.auth.BatchPrincipal;
+import org.onap.aaf.auth.actions.CacheTouch;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.helpers.CQLBatch;
+import org.onap.aaf.auth.helpers.Cred;
+import org.onap.aaf.auth.helpers.UserRole;
+import org.onap.aaf.auth.helpers.X509;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.CadiException;
+import org.onap.aaf.cadi.client.Holder;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+
+public class Remove extends Batch {
+    private final AuthzTrans noAvg;
+	private CacheTouch cacheTouch;
+	private CQLBatch cqlBatch;
+
+    public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+        super(trans.env());
+        trans.info().log("Starting Connection Process");
+        
+        noAvg = env.newTransNoAvg();
+        noAvg.setUser(new BatchPrincipal("batch:RemoveExpired"));
+
+        TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+        try {
+        	cacheTouch = new CacheTouch(trans, cluster, dryRun);
+            TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE);
+            try {
+                session = cacheTouch.getSession(trans);
+            } finally {
+                tt2.done();
+            }
+            cqlBatch = new CQLBatch(session); 
+            
+
+        } finally {
+            tt0.done();
+        }
+    }
+
+    @Override
+    protected void run(AuthzTrans trans) {
+        final int maxBatch = 50;
+
+        // Create Intermediate Output 
+        File logDir = new File(logDir());
+        
+        File expired = new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv");
+        CSV expiredCSV = new CSV(expired);
+        try {
+        	final StringBuilder sb = cqlBatch.begin();
+            final Holder<Integer> hi = new Holder<Integer>(0);
+			expiredCSV.visit(new CSV.Visitor() {
+				@Override
+				public void visit(List<String> row) throws IOException, CadiException {
+					int i = hi.get();
+					if(i>=maxBatch) {
+						cqlBatch.execute(dryRun);
+						hi.set(0);
+						cqlBatch.begin();
+						i=0;
+					}
+					switch(row.get(0)) {
+						case "ur":
+							hi.set(++i);
+							UserRole.row(sb,row);
+							break;
+						case "cred":
+							hi.set(++i);
+							Cred.row(sb,row);
+					    	break;
+						case "x509":
+							hi.set(++i);
+							X509.row(sb,row);
+							break;
+					}
+				}
+			});
+			cqlBatch.execute(dryRun);
+		} catch (IOException | CadiException e) {
+			e.printStackTrace();
+		}
+    }
+    
+    @Override
+    protected void _close(AuthzTrans trans) {
+        session.close();
+        cacheTouch.close(trans);
+    }
+
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java
new file mode 100644
index 00000000..a670f37d
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java
@@ -0,0 +1,313 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ */
+
+package org.onap.aaf.auth.update;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.onap.aaf.auth.Batch;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+
+import com.datastax.driver.core.ResultSet;
+
+public class Upload extends Batch {
+	public Upload(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+		super(trans.env());
+	    trans.info().log("Starting Connection Process");
+	    
+	    TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+	    try {
+			TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
+			try {
+				session = cluster.connect();
+			} finally {
+				tt.done();
+			}
+
+	    } finally {
+	    	tt0.done();
+	    }
+	}
+
+	private static final int BATCH_LENGTH = 100;
+
+	int count,batchCnt;
+
+	@Override
+	protected void run(AuthzTrans trans) {
+		String line;
+		StringBuilder sb = new StringBuilder();
+		StringBuilder query = new StringBuilder();
+		List<String> array = new ArrayList<String>();
+		for(String feed : args()) {
+			File file = new File(feed + ".dat");
+			TimeTaken tt = trans.start(file.getAbsolutePath(), Env.SUB);
+			System.out.println("#### Running " + feed + ".dat Feed ####");
+		    try {
+
+				if(file.exists()) {
+					count=batchCnt=0;
+					boolean justOne = false;
+					try {
+						BufferedReader br = new BufferedReader(new FileReader(file));
+						try {
+							while((line=br.readLine())!=null) {
+								if(line.length()>5000) {
+									if(query.length()>0) {
+										applyBatch(query);
+										justOne=true;
+									}
+								}
+								if(query.length()==0) {
+									query.append("BEGIN BATCH\n");
+								}
+								// Split into fields, first turning Escaped values into something we can convert back from
+								char c=0;
+								boolean inQuote = false;
+								int fldcnt = 0;
+								
+								for(int i=0;i<line.length();++i) {
+									switch(c=line.charAt(i)) {
+										case '"':
+											inQuote = !inQuote;
+											break;
+										case '|':
+											if(inQuote) {
+												sb.append(c);
+											} else {
+												addField(feed,fldcnt++,array,sb);
+											}
+											break;
+										default:
+											sb.append(c);
+									}
+								}
+								addField(feed,fldcnt,array,sb);
+								query.append(build(feed, array));
+								
+								if((++count % BATCH_LENGTH)==0 || justOne) {
+									applyBatch(query);
+									justOne=false;
+								}
+							}
+							if(query.length()>0) {
+								applyBatch(query);
+							}
+							
+						} finally {
+							br.close();
+							sb.setLength(0);
+							query.setLength(0);
+						}
+						
+					} catch (IOException e) {
+						trans.error().log(e);
+						e.printStackTrace();
+					}
+
+				} else {
+					trans.error().log("No file found: ", file.getAbsolutePath());
+				}
+			} finally {
+				tt.done();
+				System.err.flush();
+				System.out.printf("\n%d applied in %d batches\n",count,batchCnt);
+			}
+
+		}
+
+	}
+
+	// APPROVALS
+	private static final String APPR_INS_FMT="  INSERT INTO authz.approval "
+			+ "(id,approver,last_notified,memo,operation,status,ticket,type,user) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] APPR_QUOTES = new Boolean[]{false,true,true,true,true,true,false,true,true};
+
+	// ARTIFACTS
+	private static final String ARTI_INS_FMT="  INSERT INTO authz.artifact "
+			+ "(mechid,machine,ca,dir,expires,notify,ns,os_user,renewdays,sans,sponsor,type) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] ARTI_QUOTES = new Boolean[]
+			{true,true,true,true,true,true,true,true,false,false,true,false};
+	
+	// CREDS
+	private static final String CRED_INS_FMT="  INSERT INTO authz.cred "
+			+ "(id,type,expires,cred,notes,ns,other,prev) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] CRED_QUOTES = new Boolean[]
+			{true,false,true,false,true,true,false,false};
+	
+	// NS
+	private static final String NS_INS_FMT="  INSERT INTO authz.ns "
+			+ "(name,description,parent,scope,type) "
+			+ "VALUES (%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] NS_QUOTES = new Boolean[]
+			{true,true,true,false,false};
+
+	// x509
+	private static final String X509_INS_FMT="  INSERT INTO authz.x509 "
+			+ "(ca,serial,id,x500,x509) "
+			+ "VALUES (%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] X509_QUOTES = new Boolean[]
+			{true,false,true,true,true};
+
+	// ROLE
+	private static final String ROLE_INS_FMT="  INSERT INTO authz.role "
+			+ "(ns,name,description,perms) "
+			+ "VALUES (%s,%s,%s,%s);\n";
+	private static final Boolean[] ROLE_QUOTES = new Boolean[]
+			{true,true,true,false};
+	// ROLE
+	private static final String PERM_INS_FMT="  INSERT INTO authz.perm "
+			+ "(ns,type,instance,action,description,roles) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] PERM_QUOTES = new Boolean[]
+			{true,true,true,true,true,false};
+
+
+	private String build(String feed, List<String> array) {
+		String rv;
+		switch(feed) {
+			case "approval":
+				rv = String.format(APPR_INS_FMT,array.toArray());
+				break;
+			case "artifact":
+				rv = String.format(ARTI_INS_FMT,array.toArray());
+				break;
+			case "cred":
+				rv = String.format(CRED_INS_FMT,array.toArray());
+				break;
+			case "ns":
+				rv = String.format(NS_INS_FMT,array.toArray());
+				break;
+			case "role":
+				rv = String.format(ROLE_INS_FMT,array.toArray());
+				break;
+			case "perm":
+				rv = String.format(PERM_INS_FMT,array.toArray());
+				break;
+			case "x509":
+				rv = String.format(X509_INS_FMT,array.toArray());
+				break;
+			default:
+				rv = "";
+		}
+		array.clear();
+		return rv;
+	}
+	
+	private void addField(String feed, int fldcnt, List<String> array, StringBuilder sb) {
+		Boolean[] ba;
+		switch(feed) {
+			case "approval":
+				ba = APPR_QUOTES;
+				break;
+			case "artifact":
+				ba = ARTI_QUOTES;
+				break;
+			case "cred":
+				ba = CRED_QUOTES;
+				break;
+			case "ns":
+				ba = NS_QUOTES;
+				break;
+			case "role":
+				ba = ROLE_QUOTES;
+				break;
+			case "perm":
+				ba = PERM_QUOTES;
+				break;
+			case "x509":
+				ba = X509_QUOTES;
+				break;
+			default:
+				ba = null;
+		}
+		if(ba!=null) {
+			if(sb.toString().length()==0) {
+				array.add("null");
+			} else {
+				if(ba[fldcnt]) {
+					String s = null;
+					if(sb.indexOf("'")>=0) {
+						s = sb.toString().replace("'","''");
+					}
+					if(sb.indexOf("\\n")>=0) {
+						if(s==null) {
+							s = sb.toString().replace("\\n","\n");
+						} else {
+							s = s.replace("\\n","\n");
+						}
+					}
+					if(sb.indexOf("\\t")>=0) {
+						if(s==null) {
+							s = sb.toString().replace("\\t","\t");
+						} else {
+							s = s.replace("\\t","\t");
+						}
+					}
+					if(s==null) {
+						array.add("'" + sb + '\'');
+					} else {
+						array.add("'" + s + '\'');
+					}
+				} else {
+					array.add(sb.toString());
+				}
+			}
+			sb.setLength(0);
+		}
+	}
+
+	private void applyBatch(StringBuilder query) {
+		try {
+			query.append("APPLY BATCH;");
+			ResultSet rv = session.execute(query.toString());
+			if(rv.wasApplied()) {
+				System.out.print('.');
+				if((++batchCnt % 60)==0) {
+					System.out.println();
+				}
+			} else {
+				System.out.print("Data NOT APPLIED");
+			}
+		} finally {
+			query.setLength(0);
+		}
+	}
+
+
+	@Override
+	protected void _close(AuthzTrans trans) {
+        session.close();
+	}
+
+}
+