Batch upgrades for ONAP

Issue-ID: AAF-641
Change-Id: I7b5e8b97d8c6c484418c995b6e51507af78fc765
Signed-off-by: Instrumental <jonathan.gathman@att.com>

13 files changed, 1025 insertions, 226 deletions

diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java
index 5670ac95..5a19ab38 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java
@@ -39,7 +39,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.TimeZone;
 
-import org.apache.log4j.Logger;
 import org.onap.aaf.auth.common.Define;
 import org.onap.aaf.auth.dao.CassAccess;
 import org.onap.aaf.auth.dao.cass.RoleDAO;
@@ -53,12 +52,12 @@ import org.onap.aaf.auth.org.Organization.Identity;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.auth.org.OrganizationFactory;
 import org.onap.aaf.cadi.PropAccess;
+import org.onap.aaf.cadi.Access.Level;
+import org.onap.aaf.cadi.config.Config;
 import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.StaticSlot;
 import org.onap.aaf.misc.env.TimeTaken;
-import org.onap.aaf.misc.env.impl.Log4JLogTarget;
-import org.onap.aaf.misc.env.log4j.LogFileNamer;
 
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.ResultSet;
@@ -77,7 +76,6 @@ public abstract class Batch {
     protected final Cluster cluster; 
     protected static AuthzEnv env;
     protected static Session session;
-    protected static Logger aspr;
     protected static Set<String> specialNames;
     protected static boolean dryRun; 
     protected static String batchEnv;
@@ -166,14 +164,7 @@ public abstract class Batch {
             return (false);
         }
     }
-    
-    public boolean isMechID(String user) {
-        if (user.matches("m[0-9][0-9][0-9][0-9][0-9]")) {
-            return (true);
-        } else {
-            return (false);
-        }
-    }
+
 
     protected PrintStream fallout(PrintStream inFallout, String logType)
             throws IOException {
@@ -391,50 +382,51 @@ public abstract class Batch {
         try {
             Define.set(access);
             rootNs =Define.ROOT_NS();
-            
-            File f = new File("etc/authzBatch.props");
-            try {
-                if (f.exists()) {
-                    filename = f.getAbsolutePath();
-                    is = new FileInputStream(f);
-                    propLoc = f.getPath();
-                } else {
-                    URL rsrc = ClassLoader.getSystemResource("authBatch.props");
-                    filename = rsrc.toString();
-                    is = rsrc.openStream();
-                    propLoc = rsrc.getPath();
-                }
-                access.load(is);
-            } finally {
-                if (is == null) {
-                    System.err.println("authBatch.props must exist in etc dir, or in Classpath");
-                    System.exit(1);
-                }
-                is.close();
-            }
+            if(access.getProperty(Config.CADI_PROP_FILES)==null) {
+	            File f = new File("authBatch.props");
+	            try {
+	                if (f.exists()) {
+	                    filename = f.getAbsolutePath();
+	                    is = new FileInputStream(f);
+	                    propLoc = f.getPath();
+	                } else {
+	                    URL rsrc = ClassLoader.getSystemResource("authBatch.props");
+	                    filename = rsrc.toString();
+	                    is = rsrc.openStream();
+	                    propLoc = rsrc.getPath();
+	                }
+	                access.load(is);
+	            } finally {
+	                if (is == null) {
+	                    System.err.println("authBatch.props must exist in current dir, or in Classpath");
+	                    System.exit(1);
+	                }
+	                is.close();
+	            }
+	            if (filename != null) {
+	                access.log(Level.INFO,"Instantiated properties from", filename);
+	            }
+
+	            // Log where Config found
+	            access.log(Level.INFO,"Configuring from", propLoc);
 
+            }
             env = new AuthzEnv(access);
 
             transferVMProps(env, CASS_ENV, "DRY_RUN", "NS", "Organization");
 
             // Flow all Env Logs to Log4j, with ENV
 
-            LogFileNamer lfn;
-            lfn = new LogFileNamer(logDir(),"").noPID();
-            lfn.setAppender("authz-batch");
-            lfn.setAppender("aspr|ASPR");
-            lfn.setAppender("sync");
-            lfn.setAppender("jobchange");
-            lfn.setAppender("validateuser");
-            aspr = Logger.getLogger("aspr");
-            Log4JLogTarget.setLog4JEnv("authz-batch", env);
-            if (filename != null) {
-                env.init().log("Instantiated properties from", filename);
-            }
-
-            // Log where Config found
-            env.info().log("Configuring from", propLoc);
-            propLoc = null;
+//            LogFileNamer lfn;
+//            lfn = new LogFileNamer(logDir(),"").noPID();
+//            lfn.setAppender("authz-batch");
+//            lfn.setAppender("aspr|ASPR");
+//            lfn.setAppender("sync");
+//            lfn.setAppender("jobchange");
+//            lfn.setAppender("validateuser");
+//            aspr = Logger.getLogger("aspr");
+//            Log4JLogTarget.setLog4JEnv("authz-batch", env);
+//            propLoc = null;
 
             Batch batch = null;
             // setup ATTUser and Organization Slots before starting this:
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/CQLBatch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/CQLBatch.java
new file mode 100644
index 00000000..e4d487d6
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/CQLBatch.java
@@ -0,0 +1,71 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ */
+
+package org.onap.aaf.auth.helpers;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+
+public class CQLBatch {
+	private Session session;
+	private StringBuilder sb;
+	private int hasAdded;
+
+	public CQLBatch(Session session) {
+		this.session = session;
+		sb = new StringBuilder();
+		hasAdded = 0;
+	}
+	public StringBuilder begin() {
+		sb.setLength(0);
+		sb.append("BEGIN BATCH\n");
+		hasAdded = sb.length();
+		return sb;
+	}
+	
+	private boolean end() {
+		if(sb.length()==hasAdded) {
+			System.out.println("Nothing to Process");
+			return false;
+		} else {
+			sb.append("APPLY BATCH;\n");
+			System.out.println(sb);
+			return true;
+		}
+	}
+	
+	public ResultSet execute() {
+		if(end()) {
+			return session.execute(sb.toString());
+		} else {
+			return null;
+		}
+	}
+	
+	public ResultSet execute(boolean dryRun) {
+		if(dryRun) {
+			end();
+			return null;
+		} else {
+			return execute();
+		}
+		
+	}
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
index 2f6ed415..483f70f4 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java
@@ -32,9 +32,12 @@ import java.util.TreeMap;
 
 import org.onap.aaf.auth.dao.cass.CredDAO;
 import org.onap.aaf.auth.dao.hl.Question;
+import org.onap.aaf.auth.helpers.Cred.Instance;
+import org.onap.aaf.cadi.util.CSV;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
 import org.onap.aaf.misc.env.Trans;
+import org.onap.aaf.misc.env.util.Chrono;
 
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
@@ -67,6 +70,10 @@ public class Cred  {
             this.other = other;
             this.written = new Date(written);
         }
+        
+        public String toString() {
+        	return expires.toString() + type;
+        }
     }
     
     public Date last(final int ... types) {
@@ -175,7 +182,7 @@ public class Cred  {
             trans.info().log("Found",count,"creds");
         }
     }
-    
+
     /** 
      * Count entries in Cred data.
      * Note, as opposed to other methods, need to load the whole cred table for the Types.
@@ -190,17 +197,6 @@ public class Cred  {
             }
         }
         return cc;
-//        String query = "select count(*) from authz.cred LIMIT 1000000;";
-//        trans.info().log( "query: " + query );
-//        TimeTaken tt = trans.start("Count Credentials", Env.REMOTE);
-//        ResultSet results;
-//        try {
-//            Statement stmt = new SimpleStatement(query).setReadTimeoutMillis(12000);
-//            results = session.execute(stmt);
-//            return results.one().getLong(0);
-//        } finally {
-//            tt.done();
-//        }
     }
 
     public static class CredCount {
@@ -273,7 +269,23 @@ public class Cred  {
 
     }
     
-    public String toString() {
+    public void row(CSV.Writer csvw, Instance inst) {
+    	csvw.row("cred",id,ns,Integer.toString(inst.type),Chrono.dateOnlyStamp(inst.expires),Long.toString(inst.expires.getTime()));
+    }
+
+
+    public static void row(StringBuilder sb, List<String> row) {
+    	sb.append("DELETE from authz.cred WHERE id='");
+    	sb.append(row.get(1));
+    	sb.append("' AND type=");
+    	sb.append(Integer.parseInt(row.get(3)));
+    	sb.append(" AND expires=dateof(maxtimeuuid(");
+    	sb.append(row.get(5));
+    	sb.append("));\n");
+	}
+
+
+	public String toString() {
         StringBuilder sb = new StringBuilder(id);
         sb.append('[');
         for (Instance i : instances) {
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
index 762f6c66..eb1f821c 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java
@@ -33,6 +33,7 @@ import org.onap.aaf.auth.actions.URDelete;
 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
 import org.onap.aaf.auth.dao.cass.UserRoleDAO.Data;
 import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.cadi.util.CSV;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
 import org.onap.aaf.misc.env.Trans;
@@ -105,19 +106,19 @@ public class UserRole implements Cloneable, CacheChange.Data  {
         return byRole;
     }
 
-    public static void load(Trans trans, Session session, Creator<UserRole> creator ) {
-        load(trans,session,creator,null);
+    public static void load(Trans trans, Session session, Creator<UserRole> creator, Visitor<UserRole> visitor ) {
+        load(trans,session,creator,null,visitor);
     }
 
-    public static void loadOneRole(Trans trans, Session session, Creator<UserRole> creator, String role) {
-        load(trans,session,creator,"role='" + role +"' ALLOW FILTERING;");
+    public static void loadOneRole(Trans trans, Session session, Creator<UserRole> creator, String role, Visitor<UserRole> visitor) {
+        load(trans,session,creator,"role='" + role +"' ALLOW FILTERING;",visitor);
     }
     
-    public static void loadOneUser(Trans trans, Session session, Creator<UserRole> creator, String user ) {
-        load(trans,session,creator,"role='"+ user +"';");
+    public static void loadOneUser(Trans trans, Session session, Creator<UserRole> creator, String user, Visitor<UserRole> visitor ) {
+        load(trans,session,creator,"role='"+ user +"';",visitor);
     }
 
-    private static void load(Trans trans, Session session, Creator<UserRole> creator, String where) {
+    private static void load(Trans trans, Session session, Creator<UserRole> creator, String where, Visitor<UserRole> visitor) {
         String query = creator.query(where);
         trans.info().log( "query: " + query );
         TimeTaken tt = trans.start("Read UserRoles", Env.REMOTE);
@@ -132,7 +133,7 @@ public class UserRole implements Cloneable, CacheChange.Data  {
         try {
             tt = trans.start("Load UserRole", Env.SUB);
             try {
-                        iterateResults(creator, results.iterator());
+                iterateResults(creator, results.iterator(), visitor);
             } finally {
                 tt.done();
             }
@@ -141,12 +142,19 @@ public class UserRole implements Cloneable, CacheChange.Data  {
         }
     }
 
-    private static void iterateResults(Creator<UserRole> creator, Iterator<Row> iter ) {
+    private static void iterateResults(Creator<UserRole> creator, Iterator<Row> iter, Visitor<UserRole> visit ) {
         Row row;
         while (iter.hasNext()) {
             ++totalLoaded;
             row = iter.next();
             UserRole ur = creator.create(row);
+            visit.visit(ur);
+        }
+    }
+
+    public static class DataLoadVisitor implements Visitor<UserRole> {
+		@Override
+		public void visit(UserRole ur) {
             data.add(ur);
 
             List<UserRole> lur = byUser.get(ur.urdd.user);
@@ -162,9 +170,9 @@ public class UserRole implements Cloneable, CacheChange.Data  {
                 byRole.put(ur.urdd.role, lur);
             }
             lur.add(ur);
-        }
+		}
     }
-
+    
     public int totalLoaded() {
         return totalLoaded;
     }
@@ -296,4 +304,17 @@ public class UserRole implements Cloneable, CacheChange.Data  {
         cache.resetLocalData();
     }
 
+    public void row(CSV.Writer csvw) {
+    	csvw.row("ur",user(),role(),Chrono.dateOnlyStamp(expires()));
+    }
+    
+    public static void row(StringBuilder sb, List<String> row) {
+    	sb.append("DELETE from authz.user_role WHERE user='");
+    	sb.append(row.get(1));
+    	sb.append("' AND role='");
+    	sb.append(row.get(2));
+    	sb.append("';\n");
+    }
+    
+
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Visitor.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Visitor.java
new file mode 100644
index 00000000..1eb9ffec
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Visitor.java
@@ -0,0 +1,25 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ */
+
+package org.onap.aaf.auth.helpers;
+
+public interface Visitor<T> {
+	void visit(T t);
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java
new file mode 100644
index 00000000..bb75e110
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java
@@ -0,0 +1,121 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+
+package org.onap.aaf.auth.helpers;
+
+import java.awt.HeadlessException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.cert.X509Certificate;
+import java.util.Iterator;
+import java.util.List;
+
+import org.onap.aaf.cadi.Hash;
+import org.onap.aaf.cadi.Symm;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.Trans;
+import org.onap.aaf.misc.env.util.Chrono;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.Statement;
+
+public class X509 {
+    public final String ca,id,x500,x509;
+    public ByteBuffer serial;
+    
+    public X509(String ca, String id, String x500, String x509, ByteBuffer serial) {
+    	this.ca = ca;
+    	this.id = id;
+    	this.x500 = x500;
+    	this.x509 = x509;
+    	this.serial = serial;
+    }
+    
+
+    public static void load(Trans trans, Session session, Visitor<X509> visitor) {
+        load(trans,session,"select ca, id, x500, x509, serial from authz.x509;", visitor);
+    }
+
+    private static void load(Trans trans, Session session, String query, Visitor<X509> visitor) {
+        trans.info().log( "query: " + query );
+        TimeTaken tt = trans.start("Read Roles", Env.REMOTE);
+       
+        ResultSet results;
+        try {
+            Statement stmt = new SimpleStatement( query );
+            results = session.execute(stmt);
+        } finally {
+            tt.done();
+        }
+
+        int count = 0;
+        try {
+            Iterator<Row> iter = results.iterator();
+            Row row;
+            tt = trans.start("Load X509s", Env.SUB);
+            try {
+                while (iter.hasNext()) {
+                	++count;
+                    row = iter.next();
+                    visitor.visit(new X509(row.getString(0),row.getString(1), row.getString(2),row.getString(3),row.getBytes(4)));
+                }
+            } finally {
+                tt.done();
+            }
+        } finally {
+            trans.info().log("Found",count,"X509 Certificates");
+        }
+    }
+    
+    public static long count(Trans trans, Session session) {
+        String query = "select count(*) from authz.x509 LIMIT 1000000;";
+        trans.info().log( "query: " + query );
+        TimeTaken tt = trans.start("Count x509s", Env.REMOTE);
+        ResultSet results;
+        try {
+            Statement stmt = new SimpleStatement(query).setReadTimeoutMillis(12000);
+            results = session.execute(stmt);
+            return results.one().getLong(0);
+        } finally {
+            tt.done();
+        }
+    }
+    
+
+	public void row(CSV.Writer cw, X509Certificate x509Cert) throws IOException {
+		cw.row("x509",ca,Hash.toHex(serial.array()),Chrono.dateOnlyStamp(x509Cert.getNotAfter()),x500);
+	}
+
+
+	public static void row(StringBuilder sb, List<String> row) throws IOException {
+    	sb.append("DELETE from authz.x509 WHERE ca='");
+    	sb.append(row.get(1));
+    	sb.append("' AND serial=");
+    	sb.append(row.get(2));
+    	sb.append(";\n");
+	}
+
+}
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java
new file mode 100644
index 00000000..d283f414
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java
@@ -0,0 +1,276 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+
+package org.onap.aaf.auth.reports;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.onap.aaf.auth.Batch;
+import org.onap.aaf.auth.dao.cass.CredDAO;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.helpers.Cred;
+import org.onap.aaf.auth.helpers.Cred.Instance;
+import org.onap.aaf.auth.helpers.UserRole;
+import org.onap.aaf.auth.helpers.Visitor;
+import org.onap.aaf.auth.helpers.X509;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.configure.Factory;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.cadi.util.CSV.Writer;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+
+
+public class Expiring extends Batch {
+    
+    private int minOwners;
+	private ArrayList<Writer> writerList;
+	private File logDir;
+	private Date now;
+	private Date twoWeeksPast;
+	private Writer twoWeeksPastCSV;
+	private Date twoWeeksAway;
+	private Writer twoWeeksAwayCSV;
+	private Date oneMonthAway;
+	private Writer oneMonthAwayCSV;
+	private Date twoMonthsAway;
+	private Writer twoMonthsAwayCSV;
+	
+	public Expiring(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+        super(trans.env());
+        trans.info().log("Starting Connection Process");
+        
+        TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+        try {
+            TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
+            try {
+                session = cluster.connect();
+            } finally {
+                tt.done();
+            }
+            
+            // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
+            Cred.load(trans, session);
+
+            minOwners=1;
+
+            // Create Intermediate Output 
+            writerList = new ArrayList<CSV.Writer>();
+            logDir = new File(logDir());
+            logDir.mkdirs();
+            
+            GregorianCalendar gc = new GregorianCalendar();
+            now = gc.getTime();
+            gc.add(GregorianCalendar.WEEK_OF_MONTH, -2);
+            twoWeeksPast = gc.getTime();
+            File file = new File(logDir,"Expired"+Chrono.dateOnlyStamp(now)+".csv");
+            twoWeeksPastCSV = new CSV(file).writer();
+            writerList.add(twoWeeksPastCSV);
+            
+            gc.add(GregorianCalendar.WEEK_OF_MONTH, 2+2);
+            twoWeeksAway = gc.getTime();
+            file = new File(logDir,"TwoWeeksAway"+Chrono.dateOnlyStamp(now)+".csv");
+            twoWeeksAwayCSV = new CSV(file).writer();
+            writerList.add(twoWeeksAwayCSV);
+
+            gc.add(GregorianCalendar.WEEK_OF_MONTH, -2);
+            gc.add(GregorianCalendar.MONTH, 1);
+            oneMonthAway = gc.getTime();
+            file = new File(logDir,"OneMonthAway"+Chrono.dateOnlyStamp(now)+".csv");
+            oneMonthAwayCSV = new CSV(file).writer();
+            writerList.add(oneMonthAwayCSV);
+            
+            gc.add(GregorianCalendar.MONTH, 1);
+            twoMonthsAway = gc.getTime();
+            file = new File(logDir,"TwoMonthsAway"+Chrono.dateOnlyStamp(now)+".csv");
+            twoMonthsAwayCSV = new CSV(file).writer();
+            writerList.add(twoMonthsAwayCSV);
+        } finally {
+            tt0.done();
+        }
+    }
+
+    @Override
+    protected void run(AuthzTrans trans) {
+		try {
+			File file = new File(logDir, "AllOwnersExpired" + Chrono.dateOnlyStamp(now) + ".csv");
+			final CSV ownerCSV = new CSV(file);
+
+			Map<String, Set<UserRole>> owners = new TreeMap<String, Set<UserRole>>();
+			trans.info().log("Process UserRoles");
+			UserRole.load(trans, session, UserRole.v2_0_11, new Visitor<UserRole>() {
+				@Override
+				public void visit(UserRole ur) {
+					// Cannot just delete owners, unless there is at least one left. Process later
+					if ("owner".equals(ur.rname())) {
+						Set<UserRole> urs = owners.get(ur.role());
+						if (urs == null) {
+							urs = new HashSet<UserRole>();
+							owners.put(ur.role(), urs);
+						}
+						urs.add(ur);
+					} else {
+						writeAnalysis(ur);
+					}
+				}
+			});
+
+			// Now Process Owners, one owner Role at a time, ensuring one is left,
+			// preferably
+			// a good one. If so, process the others as normal. Otherwise, write
+			// ExpiredOwners
+			// report
+			if (!owners.values().isEmpty()) {
+				// Lazy Create file
+				CSV.Writer expOwner = null;
+				try {
+					for (Set<UserRole> sur : owners.values()) {
+						int goodOwners = 0;
+						for (UserRole ur : sur) {
+							if (ur.expires().after(now)) {
+								++goodOwners;
+							}
+						}
+
+						for (UserRole ur : sur) {
+							if (goodOwners >= minOwners) {
+								writeAnalysis(ur);
+							} else {
+								if (expOwner == null) {
+									expOwner = ownerCSV.writer();
+								}
+								expOwner.row(ur.role(), ur.user(), ur.expires());
+							}
+						}
+					}
+				} finally {
+					expOwner.close();
+				}
+			}
+			
+			trans.info().log("Checking for Expired Credentials");
+			for (Cred cred : Cred.data.values()) {
+		    	List<Instance> linst = cred.instances;
+		    	if(linst!=null) {
+			    	Instance lastBath = null;
+			    	for(Instance inst : linst) {
+			        	if(inst.expires.before(twoWeeksPast)) {
+			        		cred.row(twoWeeksPastCSV,inst);
+			    		} else if(inst.expires.after(now)){
+			    			if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
+			    				if(lastBath==null || lastBath.expires.before(inst.expires)) {
+			    					lastBath = inst;
+			    				}
+			    			} else if(inst.type==CredDAO.CERT_SHA256_RSA) {
+			    				writeAnalysis(cred, inst);
+			    			}
+			    		}
+			    	}
+			    	writeAnalysis(cred, lastBath);
+		    	}
+			}
+			
+			trans.info().log("Checking for Expired X509s");
+			X509.load(trans, session, new Visitor<X509>() {
+				@Override
+				public void visit(X509 x509) {
+					try {
+						for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
+							writeAnalysis(x509, (X509Certificate)cert);
+						}
+					} catch (CertificateException | IOException e) {
+						trans.error().log(e, "Error Decrypting X509");
+					}
+					
+				}
+			});
+		} catch (FileNotFoundException e) {
+			trans.info().log(e);
+		}
+	}
+    
+ 
+	protected void writeAnalysis(UserRole ur) {
+    	if(ur.expires().before(twoWeeksPast)) {
+    		ur.row(twoWeeksPastCSV);
+		} else {
+			if(ur.expires().after(now) && ur.expires().before(twoWeeksAway)) {
+	    		ur.row(twoWeeksAwayCSV);
+			} else {
+				if(ur.expires().before(oneMonthAway)) {
+		    		ur.row(oneMonthAwayCSV);
+				} else {
+					if(ur.expires().before(twoMonthsAway)) {
+			    		ur.row(twoMonthsAwayCSV);
+					}
+				}
+			}
+		}
+	}
+    
+    protected void writeAnalysis(Cred cred, Instance inst) {
+    	if(inst!=null) {
+			if(inst.expires.after(now) && inst.expires.before(twoWeeksAway)) {
+	    		cred.row(twoWeeksAwayCSV, inst);
+			} else {
+				if(inst.expires.before(oneMonthAway)) {
+		    		cred.row(oneMonthAwayCSV, inst);
+				} else {
+					if(inst.expires.before(twoMonthsAway)) {
+			    		cred.row(twoMonthsAwayCSV, inst);
+					}
+				}
+			}
+		}
+	}
+
+    protected void writeAnalysis(X509 x509, X509Certificate x509Cert) throws IOException {
+    	if(x509Cert!=null) {
+	    	if(twoWeeksPast.after(x509Cert.getNotAfter())) {
+				x509.row(twoWeeksPastCSV,x509Cert);
+			}
+    	}
+	}
+
+	@Override
+    protected void _close(AuthzTrans trans) {
+        session.close();
+    	for(CSV.Writer cw : writerList) {
+    		cw.close();
+    	}
+    }
+
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/ExpiringNext.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/ExpiringNext.java
deleted file mode 100644
index d34f9248..00000000
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/ExpiringNext.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * ============LICENSE_START====================================================
- * org.onap.aaf
- * ===========================================================================
- * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
- * ===========================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- *      http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END====================================================
- *
- */
-
-package org.onap.aaf.auth.reports;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import java.util.List;
-
-import org.onap.aaf.auth.Batch;
-import org.onap.aaf.auth.dao.cass.CredDAO;
-import org.onap.aaf.auth.env.AuthzTrans;
-import org.onap.aaf.auth.helpers.Cred;
-import org.onap.aaf.auth.helpers.UserRole;
-import org.onap.aaf.auth.helpers.Cred.Instance;
-import org.onap.aaf.auth.org.OrganizationException;
-import org.onap.aaf.misc.env.APIException;
-import org.onap.aaf.misc.env.Env;
-import org.onap.aaf.misc.env.TimeTaken;
-import org.onap.aaf.misc.env.util.Chrono;
-
-public class ExpiringNext extends Batch {
-    
-    public ExpiringNext(AuthzTrans trans) throws APIException, IOException, OrganizationException {
-        super(trans.env());
-        trans.info().log("Starting Connection Process");
-        
-        TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
-        try {
-            TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
-            try {
-                session = cluster.connect();
-            } finally {
-                tt.done();
-            }
-
-            UserRole.load(trans, session, UserRole.v2_0_11);
-            Cred.load(trans, session);
-        } finally {
-            tt0.done();
-        }
-    }
-
-    @Override
-    protected void run(AuthzTrans trans) {
-        GregorianCalendar gc = new GregorianCalendar();
-        Date now = gc.getTime();
-        gc.add(GregorianCalendar.WEEK_OF_MONTH, 2);
-        Date twoWeeks = gc.getTime();
-        // Set time way off
-        gc.set(GregorianCalendar.YEAR, 3000);
-        Date earliestUR = gc.getTime();
-        Date earliestCred = gc.getTime();
-        // Run for Roles
-        List<String> expiring = new ArrayList<>();
-        
-        trans.info().log("Checking for Expired UserRoles");
-        for (UserRole ur : UserRole.getData()) {
-            if (ur.expires().after(now)) {
-                if (ur.expires().before(twoWeeks)) {
-                    expiring.add(Chrono.dateOnlyStamp(ur.expires()) + ":\t" + ur.user() + '\t' + ur.role());
-                }
-                if (ur.expires().before(earliestUR)) {
-                    earliestUR = ur.expires();
-                }
-            }
-        }
-
-        if (expiring.size()>0) {
-            Collections.sort(expiring,Collections.reverseOrder());
-            for (String s : expiring) {
-                System.err.print('\t');
-                System.err.println(s);
-            }
-            trans.info().printf("Earliest Expiring UR is %s\n\n", Chrono.dateOnlyStamp(earliestUR));
-        } else {
-            trans.info().printf("No Expiring UserRoles within 2 weeks");
-        }
-        
-        expiring.clear();
-        
-        trans.info().log("Checking for Expired Credentials");
-        for ( Cred creds : Cred.data.values()) {
-            Instance lastInstance=null;
-            for (Instance inst : creds.instances) {
-                if (inst.type==CredDAO.BASIC_AUTH || inst.type==CredDAO.BASIC_AUTH_SHA256) {
-                    if (lastInstance == null || inst.expires.after(lastInstance.expires)) {
-                        lastInstance = inst;
-                    }
-                }
-            }
-            if (lastInstance!=null) {
-                if (lastInstance.expires.after(now)) {
-                    if (lastInstance.expires.before(twoWeeks)) {
-                        expiring.add(Chrono.dateOnlyStamp(lastInstance.expires) + ": \t" + creds.id);
-                    }
-                }
-                if (lastInstance.expires.before(earliestCred)) {
-                    earliestCred = lastInstance.expires;
-                }
-            }
-        }
-        
-        if (expiring.size()>0) {
-            Collections.sort(expiring,Collections.reverseOrder());
-            for (String s : expiring) {
-                System.err.print('\t');
-                System.err.println(s);
-            }
-            trans.info().printf("Earliest Expiring Cred is %s\n\n", Chrono.dateOnlyStamp(earliestCred));
-        } else {
-            trans.info().printf("No Expiring Creds within 2 weeks");
-        }
-
-    }
-    
-    @Override
-    protected void _close(AuthzTrans trans) {
-        session.close();
-    }
-
-}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringOrig.java
index e9f0e726..36d08b7a 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Expiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringOrig.java
@@ -66,7 +66,7 @@ import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
 import org.onap.aaf.misc.env.util.Chrono;
 
-public class Expiring extends Batch {
+public class ExpiringOrig extends Batch {
     private CredPrint crPrint;
     private URFutureApprove urFutureApprove;
     private URFutureApproveExec urFutureApproveExec;
@@ -81,7 +81,7 @@ public class Expiring extends Batch {
     private Email email;
     private File deletesFile;
 
-    public Expiring(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+    public ExpiringOrig(AuthzTrans trans) throws APIException, IOException, OrganizationException {
         super(trans.env());
         trans.info().log("Starting Connection Process");
         
@@ -124,7 +124,7 @@ public class Expiring extends Batch {
                 urDeleteF = new PrintStream(new FileOutputStream(deletesFile = new File(data_dir,"UserRoleDeletes.dat"),false)));
             UserRole.setRecoverStream(
                 urRecoverF = new PrintStream(new FileOutputStream(new File(data_dir,"UserRoleRecover.dat"),false)));
-            UserRole.load(trans, session, UserRole.v2_0_11);
+            UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor());
             
             Cred.load(trans, session);
             NS.load(trans, session,NS.v2_0_11);
@@ -492,7 +492,7 @@ public class Expiring extends Batch {
     
     @Override
     protected void _close(AuthzTrans trans) {
-        aspr.info("End " + this.getClass().getSimpleName() + " processing" );
+        trans.info().log("End",this.getClass().getSimpleName(),"processing" );
         for (Action<?,?,?> action : new Action<?,?,?>[] {crDelete}) {
             if (action instanceof ActionDAO) {
                 ((ActionDAO<?,?,?>)action).close(trans);
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java
index cab612eb..dae62573 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java
@@ -141,7 +141,7 @@ public class ExpiringP2 extends Batch {
     
     @Override
     protected void _close(AuthzTrans trans) {
-        aspr.info("End " + this.getClass().getSimpleName() + " processing" );
+        trans.info().log("End",this.getClass().getSimpleName(),"processing" );
         for (Action<?,?,?> action : new Action<?,?,?>[] {urDelete,cacheTouch}) {
                 if (action instanceof ActionDAO) {
                     ((ActionDAO<?,?,?>)action).close(trans);
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
index b4631f66..cbc3ef38 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java
@@ -32,6 +32,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.TreeMap;
 
 import org.onap.aaf.auth.Batch;
 import org.onap.aaf.auth.BatchPrincipal;
@@ -43,21 +44,19 @@ import org.onap.aaf.auth.dao.hl.Question;
 import org.onap.aaf.auth.env.AuthzTrans;
 import org.onap.aaf.auth.helpers.Cred;
 import org.onap.aaf.auth.helpers.Notification;
-import org.onap.aaf.auth.helpers.UserRole;
 import org.onap.aaf.auth.helpers.Notification.TYPE;
+import org.onap.aaf.auth.helpers.UserRole;
 import org.onap.aaf.auth.layer.Result;
 import org.onap.aaf.auth.org.EmailWarnings;
 import org.onap.aaf.auth.org.Organization;
+import org.onap.aaf.auth.org.Organization.Identity;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.auth.org.OrganizationFactory;
-import org.onap.aaf.auth.org.Organization.Identity;
 import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
 import org.onap.aaf.misc.env.util.Chrono;
 
-import java.util.TreeMap;
-
 
 public class NotifyCredExpiring extends Batch {
 
@@ -109,7 +108,7 @@ public class NotifyCredExpiring extends Batch {
 
         Cred.load(trans, session,CredDAO.BASIC_AUTH, CredDAO.BASIC_AUTH_SHA256);
         Notification.load(trans, session, Notification.v2_0_18);
-        UserRole.load(trans, session, UserRole.v2_0_11);
+        UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor());
         
         ps = new PrintStream(new FileOutputStream(logDir() + "/email"+Chrono.dateOnlyStamp()+".log",true));
         ps.printf("### Approval Notify %s for %s%s\n",Chrono.dateTime(),batchEnv,dryRun?", DryRun":"");
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java
new file mode 100644
index 00000000..22961f61
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java
@@ -0,0 +1,123 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+
+package org.onap.aaf.auth.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.onap.aaf.auth.Batch;
+import org.onap.aaf.auth.BatchPrincipal;
+import org.onap.aaf.auth.actions.CacheTouch;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.helpers.CQLBatch;
+import org.onap.aaf.auth.helpers.Cred;
+import org.onap.aaf.auth.helpers.UserRole;
+import org.onap.aaf.auth.helpers.X509;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.CadiException;
+import org.onap.aaf.cadi.client.Holder;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+
+public class Remove extends Batch {
+    private final AuthzTrans noAvg;
+	private CacheTouch cacheTouch;
+	private CQLBatch cqlBatch;
+
+    public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+        super(trans.env());
+        trans.info().log("Starting Connection Process");
+        
+        noAvg = env.newTransNoAvg();
+        noAvg.setUser(new BatchPrincipal("batch:ExpiringP2"));
+
+        TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+        try {
+        	cacheTouch = new CacheTouch(trans, cluster, dryRun);
+            TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE);
+            try {
+                session = cacheTouch.getSession(trans);
+            } finally {
+                tt2.done();
+            }
+            cqlBatch = new CQLBatch(session); 
+            
+
+        } finally {
+            tt0.done();
+        }
+    }
+
+    @Override
+    protected void run(AuthzTrans trans) {
+        final int maxBatch = 50;
+
+        // Create Intermediate Output 
+        File logDir = new File(logDir());
+        File expired = new File(logDir,"Expired"+Chrono.dateOnlyStamp()+".csv");
+        CSV expiredCSV = new CSV(expired);
+        try {
+        	final StringBuilder sb = cqlBatch.begin();
+            final Holder<Integer> hi = new Holder<Integer>(0);
+			expiredCSV.visit(new CSV.Visitor() {
+				@Override
+				public void visit(List<String> row) throws IOException, CadiException {
+					int i = hi.get();
+					if(i>=maxBatch) {
+						cqlBatch.execute(dryRun);
+						hi.set(0);
+						cqlBatch.begin();
+						i=0;
+					}
+					switch(row.get(0)) {
+						case "ur":
+							hi.set(++i);
+							UserRole.row(sb,row);
+							break;
+						case "cred":
+							hi.set(++i);
+							Cred.row(sb,row);
+					    	break;
+						case "x509":
+							hi.set(++i);
+							X509.row(sb,row);
+							break;
+					}
+				}
+			});
+			cqlBatch.execute(dryRun);
+		} catch (IOException | CadiException e) {
+			e.printStackTrace();
+		}
+    }
+    
+    @Override
+    protected void _close(AuthzTrans trans) {
+        session.close();
+        cacheTouch.close(trans);
+    }
+
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java
new file mode 100644
index 00000000..f308e226
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java
@@ -0,0 +1,302 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ */
+
+package org.onap.aaf.auth.update;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.onap.aaf.auth.Batch;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+
+import com.datastax.driver.core.ResultSet;
+
+public class Upload extends Batch {
+	public Upload(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+		super(trans.env());
+	    trans.info().log("Starting Connection Process");
+	    
+	    TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+	    try {
+			TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
+			try {
+				session = cluster.connect();
+			} finally {
+				tt.done();
+			}
+
+	    } finally {
+	    	tt0.done();
+	    }
+	}
+
+	private static final int BATCH_LENGTH = 100;
+
+	int count,batchCnt;
+
+	@Override
+	protected void run(AuthzTrans trans) {
+		String line;
+		StringBuilder sb = new StringBuilder();
+		StringBuilder query = new StringBuilder();
+		List<String> array = new ArrayList<String>();
+		for(String feed : args()) {
+			File file = new File(feed + ".dat");
+			TimeTaken tt = trans.start(file.getAbsolutePath(), Env.SUB);
+			System.out.println("#### Running " + feed + ".dat Feed ####");
+		    try {
+
+				if(file.exists()) {
+					count=batchCnt=0;
+					try {
+						BufferedReader br = new BufferedReader(new FileReader(file));
+						try {
+							while((line=br.readLine())!=null) {
+								if(query.length()==0) {
+									query.append("BEGIN BATCH\n");
+								}
+								// Split into fields, first turning Escaped values into something we can convert back from
+								char c=0;
+								boolean inQuote = false;
+								int fldcnt = 0;
+								
+								for(int i=0;i<line.length();++i) {
+									switch(c=line.charAt(i)) {
+										case '"':
+											inQuote = !inQuote;
+											break;
+										case '|':
+											if(inQuote) {
+												sb.append(c);
+											} else {
+												addField(feed,fldcnt++,array,sb);
+											}
+											break;
+										default:
+											sb.append(c);
+									}
+								}
+								addField(feed,fldcnt,array,sb);
+								query.append(build(feed, array));
+								
+								if((++count % BATCH_LENGTH)==0) {
+									applyBatch(query);
+								}
+							}
+							if((count % BATCH_LENGTH)!=0) {
+								applyBatch(query);
+							}
+							
+						} finally {
+							br.close();
+							sb.setLength(0);
+							query.setLength(0);
+						}
+						
+					} catch (IOException e) {
+						trans.error().log(e);
+						e.printStackTrace();
+					}
+
+				} else {
+					trans.error().log("No file found: ", file.getAbsolutePath());
+				}
+			} finally {
+				tt.done();
+				System.err.flush();
+				System.out.printf("\n%d applied in %d batches\n",count,batchCnt);
+			}
+
+		}
+
+	}
+
+	// APPROVALS
+	private static final String APPR_INS_FMT="  INSERT INTO authz.approval "
+			+ "(id,approver,last_notified,memo,operation,status,ticket,type,user) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] APPR_QUOTES = new Boolean[]{false,true,true,true,true,true,false,true,true};
+
+	// ARTIFACTS
+	private static final String ARTI_INS_FMT="  INSERT INTO authz.artifact "
+			+ "(mechid,machine,ca,dir,expires,notify,ns,os_user,renewdays,sans,sponsor,type) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] ARTI_QUOTES = new Boolean[]
+			{true,true,true,true,true,true,true,true,false,false,true,false};
+	
+	// CREDS
+	private static final String CRED_INS_FMT="  INSERT INTO authz.cred "
+			+ "(id,type,expires,cred,notes,ns,other,prev) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] CRED_QUOTES = new Boolean[]
+			{true,false,true,false,true,true,false,false};
+	
+	// NS
+	private static final String NS_INS_FMT="  INSERT INTO authz.ns "
+			+ "(name,description,parent,scope,type) "
+			+ "VALUES (%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] NS_QUOTES = new Boolean[]
+			{true,true,true,false,false};
+
+	// x509
+	private static final String X509_INS_FMT="  INSERT INTO authz.x509 "
+			+ "(ca,serial,id,x500,x509) "
+			+ "VALUES (%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] X509_QUOTES = new Boolean[]
+			{true,false,true,true,true};
+
+	// ROLE
+	private static final String ROLE_INS_FMT="  INSERT INTO authz.role "
+			+ "(ns,name,description,perms) "
+			+ "VALUES (%s,%s,%s,%s);\n";
+	private static final Boolean[] ROLE_QUOTES = new Boolean[]
+			{true,true,true,false};
+	// ROLE
+	private static final String PERM_INS_FMT="  INSERT INTO authz.perm "
+			+ "(ns,type,instance,action,description,roles) "
+			+ "VALUES (%s,%s,%s,%s,%s,%s);\n";
+	private static final Boolean[] PERM_QUOTES = new Boolean[]
+			{true,true,true,true,true,false};
+
+
+	private String build(String feed, List<String> array) {
+		String rv;
+		switch(feed) {
+			case "approval":
+				rv = String.format(APPR_INS_FMT,array.toArray());
+				break;
+			case "artifact":
+				rv = String.format(ARTI_INS_FMT,array.toArray());
+				break;
+			case "cred":
+				rv = String.format(CRED_INS_FMT,array.toArray());
+				break;
+			case "ns":
+				rv = String.format(NS_INS_FMT,array.toArray());
+				break;
+			case "role":
+				rv = String.format(ROLE_INS_FMT,array.toArray());
+				break;
+			case "perm":
+				rv = String.format(PERM_INS_FMT,array.toArray());
+				break;
+			case "x509":
+				rv = String.format(X509_INS_FMT,array.toArray());
+				break;
+			default:
+				rv = "";
+		}
+		array.clear();
+		return rv;
+	}
+	
+	private void addField(String feed, int fldcnt, List<String> array, StringBuilder sb) {
+		Boolean[] ba;
+		switch(feed) {
+			case "approval":
+				ba = APPR_QUOTES;
+				break;
+			case "artifact":
+				ba = ARTI_QUOTES;
+				break;
+			case "cred":
+				ba = CRED_QUOTES;
+				break;
+			case "ns":
+				ba = NS_QUOTES;
+				break;
+			case "role":
+				ba = ROLE_QUOTES;
+				break;
+			case "perm":
+				ba = PERM_QUOTES;
+				break;
+			case "x509":
+				ba = X509_QUOTES;
+				break;
+			default:
+				ba = null;
+		}
+		if(ba!=null) {
+			if(sb.toString().length()==0) {
+				array.add("null");
+			} else {
+				if(ba[fldcnt]) {
+					String s = null;
+					if(sb.indexOf("'")>=0) {
+						s = sb.toString().replace("'","''");
+					}
+					if(sb.indexOf("\\n")>=0) {
+						if(s==null) {
+							s = sb.toString().replace("\\n","\n");
+						} else {
+							s = s.replace("\\n","\n");
+						}
+					}
+					if(sb.indexOf("\\t")>=0) {
+						if(s==null) {
+							s = sb.toString().replace("\\t","\t");
+						} else {
+							s = s.replace("\\t","\t");
+						}
+					}
+					if(s==null) {
+						array.add("'" + sb + '\'');
+					} else {
+						array.add("'" + s + '\'');
+					}
+				} else {
+					array.add(sb.toString());
+				}
+			}
+			sb.setLength(0);
+		}
+	}
+
+	private void applyBatch(StringBuilder query) {
+		query.append("APPLY BATCH;");
+		ResultSet rv = session.execute(query.toString());
+		if(rv.wasApplied()) {
+			System.out.print('.');
+			if((++batchCnt % 60)==0) {
+				System.out.println();
+			}
+		} else {
+			System.out.print("Data NOT APPLIED");
+		}
+		query.setLength(0);
+	}
+
+
+	@Override
+	protected void _close(AuthzTrans trans) {
+        session.close();
+	}
+
+}
+