Further Batch Refactor

Issue-ID: AAF-670
Change-Id: Ie68280edb7e9d6c495d3905de61372be895dcf3d
Signed-off-by: Instrumental <jonathan.gathman@att.com>

14 files changed, 848 insertions, 145 deletions

diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
index 7920e481..ff605203 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
@@ -34,22 +34,20 @@ import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TimeZone;
 
+import org.apache.log4j.Logger;
 import org.onap.aaf.auth.common.Define;
 import org.onap.aaf.auth.dao.CassAccess;
-import org.onap.aaf.auth.dao.cass.RoleDAO;
-import org.onap.aaf.auth.dao.cass.UserRoleDAO;
-import org.onap.aaf.auth.dao.hl.Question;
 import org.onap.aaf.auth.env.AuthzEnv;
 import org.onap.aaf.auth.env.AuthzTrans;
-import org.onap.aaf.auth.layer.Result;
+import org.onap.aaf.auth.log4j.Log4JAccessAppender;
 import org.onap.aaf.auth.org.Organization;
-import org.onap.aaf.auth.org.Organization.Identity;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.auth.org.OrganizationFactory;
 import org.onap.aaf.cadi.Access.Level;
@@ -59,6 +57,9 @@ import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.StaticSlot;
 import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+import org.onap.aaf.misc.env.util.Split;
+import org.onap.aaf.misc.env.util.StringBuilderOutputStream;
 
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.ResultSet;
@@ -68,8 +69,6 @@ import com.datastax.driver.core.Statement;
 
 public abstract class Batch {
 
-    private static String rootNs;
-
     private static StaticSlot ssargs;
 
     protected static final String STARS = "*****";
@@ -84,22 +83,14 @@ public abstract class Batch {
 
     public static final String CASS_ENV = "CASS_ENV";
     public static final String LOG_DIR = "LOG_DIR";
-    protected static final String PUNT="punt";
     protected static final String MAX_EMAILS="MAX_EMAILS";
     protected static final String VERSION="VERSION";
     public static final String GUI_URL="GUI_URL";
     
     protected final Organization org;
-
-
     
     protected Batch(AuthzEnv env) throws APIException, IOException, OrganizationException {
-        // Be able to change Environments
-        // load extra properties, i.e.
-        // PERF.cassandra.clusters=....
-        batchEnv = env.getProperty(CASS_ENV);
         if (batchEnv != null) {
-            batchEnv = batchEnv.trim();
             env.info().log("Redirecting to ",batchEnv,"environment");
             String str;
             for (String key : new String[]{
@@ -107,7 +98,7 @@ public abstract class Batch {
                     CassAccess.CASSANDRA_CLUSTERS_PORT,
                     CassAccess.CASSANDRA_CLUSTERS_USER_NAME,
                     CassAccess.CASSANDRA_CLUSTERS_PASSWORD,
-                    VERSION,GUI_URL,PUNT,MAX_EMAILS,
+                    VERSION,GUI_URL,MAX_EMAILS,
                     LOG_DIR,
                     "SPECIAL_NAMES"
                     }) {
@@ -333,35 +324,6 @@ public abstract class Batch {
         }
     }
     
-    // IMPORTANT! VALIDATE Organization isUser method
-    protected void checkOrganizationAcccess(AuthzTrans trans, Question q) throws APIException, OrganizationException {
-            Set<String> testUsers = new HashSet<>();
-            Result<List<RoleDAO.Data>> rrd = q.roleDAO.readNS(trans, rootNs);
-            if (rrd.isOK()) {
-                for (RoleDAO.Data r : rrd.value) {
-                    Result<List<UserRoleDAO.Data>> rur = q.userRoleDAO.readByRole(trans, r.fullName());
-                    if (!rur.isOK()) {
-                        continue;
-                    }
-                    for (UserRoleDAO.Data udd : rur.value) {
-                        testUsers.add(udd.user);
-                    }
-                }
-                if (testUsers.size() < 2) {
-                    throw new APIException("Not enough Users in Roles for " + rootNs + " to Validate");
-                }
-
-                Identity iden;
-                for (String user : testUsers) {
-                    if ((iden = org.getIdentity(trans, user)) == null) {
-                        throw new APIException("Failed Organization Entity Validation Check: " + user);
-                    } else {
-                        trans.info().log("Organization Validation Check: " + iden.id());
-                    }
-                }
-            }
-        }
-    
     protected static String logDir() {
         String ld = env.getProperty(LOG_DIR);
         if (ld==null) {
@@ -391,13 +353,21 @@ public abstract class Batch {
     }
 
     public static void main(String[] args) {
-        PropAccess access = new PropAccess(args);
+    	// Use a StringBuilder to save off logs until a File can be setup
+    	StringBuilderOutputStream sbos = new StringBuilderOutputStream();
+        PropAccess access = new PropAccess(new PrintStream(sbos),args);
+        access.log(Level.INIT, "------- Starting Batch ------\n  Args: ");
+        for(String s: args) {
+        	sbos.getBuffer().append(s);
+        	sbos.getBuffer().append(' ');
+        }
+        
         InputStream is = null;
         String filename;
         String propLoc;
         try {
             Define.set(access);
-            rootNs =Define.ROOT_NS();
+            
             if(access.getProperty(Config.CADI_PROP_FILES)==null) {
 	            File f = new File("authBatch.props");
 	            try {
@@ -427,97 +397,118 @@ public abstract class Batch {
 	            access.log(Level.INFO,"Configuring from", propLoc);
 
             }
-            env = new AuthzEnv(access);
 
+            env = new AuthzEnv(access);
+            
             transferVMProps(env, CASS_ENV, "DRY_RUN", "NS", "Organization");
 
-            // Flow all Env Logs to Log4j, with ENV
-
-//            LogFileNamer lfn;
-//            lfn = new LogFileNamer(logDir(),"").noPID();
-//            lfn.setAppender("authz-batch");
-//            lfn.setAppender("aspr|ASPR");
-//            lfn.setAppender("sync");
-//            lfn.setAppender("jobchange");
-//            lfn.setAppender("validateuser");
-//            aspr = Logger.getLogger("aspr");
-//            Log4JLogTarget.setLog4JEnv("authz-batch", env);
-//            propLoc = null;
-
-            Batch batch = null;
-            // setup ATTUser and Organization Slots before starting this:
-            // TODO redo this
-            // env.slot(ATT.ATT_USERSLOT);
-            //
-            // OrganizationFactory.setDefaultOrg(env, ATT.class.getName());
-            AuthzTrans trans = env.newTrans();
-
-            TimeTaken tt = trans.start("Total Run", Env.SUB);
-            try {
-                int len = args.length;
-                if (len > 0) {
-                    String toolName = args[0];
-                    len -= 1;
-                    if (len < 0)
-                        len = 0;
-                    String nargs[] = new String[len];
-                    if (len > 0) {
-                        System.arraycopy(args, 1, nargs, 0, len);
-                    }
-
-                    env.put(ssargs = env.staticSlot("ARGS"), nargs);
-
-                    /*
-                     * Add New Batch Programs (inherit from Batch) here
-                     */
-
-                    // Might be a Report, Update or Temp Batch
-                    Class<?> cls;
-                    String classifier = "";
-                    try {
-                        cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.update." + toolName);
-                        classifier = "Update:";
-                    } catch (ClassNotFoundException e) {
-                        try {
-                            cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.reports." + toolName);
-                            classifier = "Report:";
-                        } catch (ClassNotFoundException e2) {
-                            try {
-                                cls = ClassLoader.getSystemClassLoader()
-                                        .loadClass("org.onap.aaf.auth.batch.temp." + toolName);
-                                classifier = "Temp Utility:";
-                            } catch (ClassNotFoundException e3) {
-                                cls = null;
-                            }
-                        }
-                    }
-                    if (cls != null) {
-                        Constructor<?> cnst = cls.getConstructor(AuthzTrans.class);
-                        batch = (Batch) cnst.newInstance(trans);
-                        env.info().log("Begin", classifier, toolName);
-                    }
-                
-
-                    if (batch == null) {
-                        trans.error().log("No Batch named", toolName, "found");
-                    }
-                    /*
-                     * End New Batch Programs (inherit from Batch) here
-                     */
+            // Be able to change Environments
+            // load extra properties, i.e.
+            // PERF.cassandra.clusters=....
+            batchEnv = env.getProperty(CASS_ENV);
+            if(batchEnv!=null) {
+            	batchEnv = batchEnv.trim();
+            }
 
-                }
-                if (batch != null) {
-                    batch.run(trans);
-                }
+            File logFile = new File(logDir() + "/batch" + Chrono.dateOnlyStamp(new Date()) + ".log" );
+            PrintStream batchLog = new PrintStream(new FileOutputStream(logFile,true));
+            try {
+	            access.setStreamLogIt(batchLog);
+	            sbos.flush();
+	            batchLog.print(sbos.getBuffer());
+	            sbos = null;
+	            Logger.getRootLogger().addAppender(new Log4JAccessAppender(access));
+
+	            Batch batch = null;
+	            AuthzTrans trans = env.newTrans();
+	
+	            TimeTaken tt = trans.start("Total Run", Env.SUB);
+	            try {
+	                int len = args.length;
+	                if (len > 0) {
+	                    String toolName = args[0];
+	                    len -= 1;
+	                    if (len < 0)
+	                        len = 0;
+	                    String nargs[] = new String[len];
+	                    if (len > 0) {
+	                        System.arraycopy(args, 1, nargs, 0, len);
+	                    }
+	
+	                    env.put(ssargs = env.staticSlot("ARGS"), nargs);
+	
+	                    /*
+	                     * Add New Batch Programs (inherit from Batch) here
+	                     */
+	
+	                    // Might be a Report, Update or Temp Batch
+	                    Class<?> cls = null;
+	                    String classifier = "";
+	
+	                    String[] pkgs = new String[] {
+	                    		"org.onap.aaf.auth.batch.update",
+	                    		"org.onap.aaf.auth.batch.reports",
+	                    		"org.onap.aaf.auth.batch.temp"
+	                    		};
+	                    		
+	                    String ebp = env.getProperty("EXTRA_BATCH_PKGS");
+	                    if(ebp!=null) {
+	                        String[] ebps = Split.splitTrim(':', ebp);
+	                    	String[] temp = new String[ebps.length + pkgs.length];
+	                    	System.arraycopy(pkgs,0, temp, 0, pkgs.length);
+	                    	System.arraycopy(ebps,0,temp,pkgs.length,ebps.length);
+	                    	pkgs = temp;
+	                    }
+	                    
+	                    for(String p : pkgs) {
+		                    try {
+		                        cls = ClassLoader.getSystemClassLoader().loadClass(p + '.' + toolName);
+		                        int lastDot = p.lastIndexOf('.');
+		                        if(p.length()>0 || p.length()!=lastDot) { 
+			                        StringBuilder sb = new StringBuilder();
+			                        sb.append(Character.toUpperCase(p.charAt(++lastDot)));
+			                        while(++lastDot<p.length()) {
+			                        	sb.append(p.charAt(lastDot));
+			                        }
+			                        sb.append(':');
+			                        classifier = sb.toString();
+			                        break;
+		                        }
+		                    } catch (ClassNotFoundException e) {
+		                    	cls = null;
+		                    }
+	                    }
+	                    if (cls != null) {
+	                        Constructor<?> cnst = cls.getConstructor(AuthzTrans.class);
+	                        batch = (Batch) cnst.newInstance(trans);
+	                        env.info().log("Begin", classifier, toolName);
+	                    }
+	                
+	
+	                    if (batch == null) {
+	                        trans.error().log("No Batch named", toolName, "found");
+	                    }
+	                    /*
+	                     * End New Batch Programs (inherit from Batch) here
+	                     */
+	
+	                }
+	                if (batch != null) {
+	                    batch.run(trans);
+	                }
+	            } finally {
+	                tt.done();
+	                if (batch != null) {
+	                    batch.close(trans);
+	                }
+	                StringBuilder sb = new StringBuilder("Task Times\n");
+	                trans.auditTrail(4, sb, AuthzTrans.SUB, AuthzTrans.REMOTE);
+	                trans.info().log(sb);
+	            }
             } finally {
-                tt.done();
-                if (batch != null) {
-                    batch.close(trans);
-                }
-                StringBuilder sb = new StringBuilder("Task Times\n");
-                trans.auditTrail(4, sb, AuthzTrans.SUB, AuthzTrans.REMOTE);
-                trans.info().log(sb);
+            	batchLog.close();
             }
+
         } catch (Exception e) {
             e.printStackTrace(System.err);
             // Exceptions thrown by DB aren't stopping the whole process.
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
index c65026a8..efd1ec9e 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
@@ -20,6 +20,8 @@
 
 package org.onap.aaf.auth.batch.helpers;
 
+import org.onap.aaf.misc.env.LogTarget;
+
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Session;
 
@@ -27,8 +29,10 @@ public class CQLBatch {
 	private Session session;
 	private StringBuilder sb;
 	private int hasAdded;
+	private LogTarget log;
 
-	public CQLBatch(Session session) {
+	public CQLBatch(LogTarget log, Session session) {
+		this.log = log;
 		this.session = session;
 		sb = new StringBuilder();
 		hasAdded = 0;
@@ -42,11 +46,10 @@ public class CQLBatch {
 	
 	private boolean end() {
 		if(sb.length()==hasAdded) {
-			System.out.println("Nothing to Process");
 			return false;
 		} else {
 			sb.append("APPLY BATCH;\n");
-			System.out.println(sb);
+			log.log(sb);
 			return true;
 		}
 	}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
index b58506ea..e3ac9e63 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
@@ -280,7 +280,7 @@ public class Cred  {
     }
 
 
-    public static void row(StringBuilder sb, List<String> row) {
+    public static void batchDelete(StringBuilder sb, List<String> row) {
     	sb.append("DELETE from authz.cred WHERE id='");
     	sb.append(row.get(1));
     	sb.append("' AND type=");
@@ -291,7 +291,6 @@ public class Cred  {
     	sb.append("));\n");
 	}
 
-
 	public String toString() {
         StringBuilder sb = new StringBuilder(id);
         sb.append('[');
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java
index f6a25e7f..a985640b 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java
@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.onap.aaf.cadi.Access;
+import org.onap.aaf.cadi.PropAccess;
 
 public class ExpireRange {
 	private static final String AAF_BATCH_RANGE = "aaf_batch_range.";
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
index ecf66fed..bea3b5ec 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
@@ -309,10 +309,10 @@ public class UserRole implements Cloneable, CacheChange.Data  {
     }
 
     public void row(final CSV.Writer csvw) {
-    	csvw.row("ur",user(),ns(),rname(),Chrono.dateOnlyStamp(expires()));
+    	csvw.row("ur",user(),ns(),rname(),Chrono.dateOnlyStamp(expires()),expires().getTime());
     }
     
-    public static void row(StringBuilder sb, List<String> row) {
+    public static void batchDelete(StringBuilder sb, List<String> row) {
     	sb.append("DELETE from authz.user_role WHERE user='");
     	sb.append(row.get(1));
     	sb.append("' AND role='");
@@ -322,6 +322,18 @@ public class UserRole implements Cloneable, CacheChange.Data  {
     	sb.append("';\n");
     }
 
+    public static void batchExtend(StringBuilder sb, List<String> row, String newDate ) {
+    	sb.append("UPDATE authz.user_role SET expires='");
+    	sb.append(newDate);
+    	sb.append("' WHERE user='");
+    	sb.append(row.get(1));
+    	sb.append("' AND role='");
+    	sb.append(row.get(2));
+    	sb.append('.');
+    	sb.append(row.get(3));
+    	sb.append("';\n");
+    }
+    
 	public static String histMemo(String fmt, List<String> row) {
 		return String.format(fmt, row.get(1),row.get(2)+'.'+row.get(3), row.get(4));
 	}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
index e9fd818d..d8eee6d5 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
@@ -101,6 +101,7 @@ public class Expiring extends Batch {
                     	if("Delete".equals(r.name())) {
                     		deleteDate = r.getEnd();
                     	}
+                    	trans.init().log("Creating File:",file.getAbsolutePath());
             		}
             	}
             }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/NotInOrg.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/NotInOrg.java
new file mode 100644
index 00000000..2ba5d022
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/NotInOrg.java
@@ -0,0 +1,200 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+
+package org.onap.aaf.auth.batch.reports;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.onap.aaf.auth.batch.Batch;
+import org.onap.aaf.auth.batch.helpers.Cred;
+import org.onap.aaf.auth.batch.helpers.Cred.Instance;
+import org.onap.aaf.auth.batch.helpers.UserRole;
+import org.onap.aaf.auth.batch.helpers.Visitor;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.org.Organization;
+import org.onap.aaf.auth.org.Organization.Identity;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.cadi.util.CSV.Writer;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+
+
+public class NotInOrg extends Batch {
+    
+	private static final String NOT_IN_ORG = "NotInOrg";
+	private static final String CSV = ".csv";
+	private static final String INFO = "info";
+	private Map<String, CSV.Writer> writerList;
+	private Map<String, CSV.Writer> whichWriter; 
+	private File logDir;
+	private Date now;
+	private Writer notInOrgW;
+	private Writer notInOrgDeleteW;
+	
+	public NotInOrg(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+        super(trans.env());
+        trans.info().log("Starting Connection Process");
+        
+        TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+        try {
+            TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
+            try {
+                session = cluster.connect();
+            } finally {
+                tt.done();
+            }
+            
+            // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
+            Cred.load(trans, session);
+
+            // Create Intermediate Output 
+            writerList = new HashMap<>();
+            whichWriter = new TreeMap<>();
+            logDir = new File(logDir());
+            logDir.mkdirs();
+
+            now = new Date();
+            String sdate = Chrono.dateOnlyStamp(now);
+           	File file = new File(logDir,NOT_IN_ORG + sdate +CSV);
+            CSV csv = new CSV(file);
+            notInOrgW = csv.writer(false);
+            notInOrgW.row(INFO,NOT_IN_ORG,Chrono.dateOnlyStamp(now),0);
+            writerList.put(NOT_IN_ORG,notInOrgW);
+            
+            // These will have been double-checked by the Organization, and can be deleted immediately.
+            String fn = NOT_IN_ORG+"Delete";
+            file = new File(logDir,fn + sdate +CSV);
+            CSV csvDelete = new CSV(file);
+            notInOrgDeleteW = csvDelete.writer(false);
+            notInOrgDeleteW.row(INFO,fn,Chrono.dateOnlyStamp(now),0);
+            writerList.put(NOT_IN_ORG,notInOrgW);
+            
+        } finally {
+            tt0.done();
+        }
+    }
+
+    @Override
+    protected void run(AuthzTrans trans) {
+		try {
+			Map<String,Boolean> checked = new TreeMap<String, Boolean>();
+			trans.info().log("Process Organization Identities");
+			trans.info().log("User Roles");
+			
+			final AuthzTrans transNoAvg = trans.env().newTransNoAvg();
+			UserRole.load(trans, session, UserRole.v2_0_11, new Visitor<UserRole>() {
+				@Override
+				public void visit(UserRole ur) {
+					try {
+						if(!check(transNoAvg, checked, ur.user())) {
+							ur.row(whichWriter(transNoAvg,ur.user()));
+						}
+					} catch (OrganizationException e) {
+						trans.error().log(e, "Error Decrypting X509");
+					}
+				}
+			});
+			
+			trans.info().log("Checking for Creds without IDs");
+			
+			for (Cred cred : Cred.data.values()) {
+				if(!check(transNoAvg,checked, cred.id)) {
+					CSV.Writer cw = whichWriter(transNoAvg, cred.id);
+					for(Instance inst : cred.instances) {
+						cred.row(cw, inst);
+					}
+				}
+			}
+			
+			/* 
+			 * Do we delete now?  Or work on Revocation semantics 
+			 * 
+			trans.info().log("Checking for X509s without IDs");
+			X509.load(trans, session, new Visitor<X509>() {
+				@Override
+				public void visit(X509 x509) {
+					try {
+						for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
+							X509Certificate xc = (X509Certificate)cert;
+							xc.getSubjectDN();
+							if(!check(transNoAvg,checked, (X))) {
+								x509.row(notInOrgW,);
+							}
+						}
+					} catch (CertificateException | IOException e) {
+						trans.error().log(e, "Error Decrypting X509");
+					}					
+				}
+			});
+			 */
+		} catch (OrganizationException e) {
+			trans.info().log(e);
+		}
+	}
+    
+ 
+	private Writer whichWriter(AuthzTrans transNoAvg, String id) {
+		Writer w = whichWriter.get(id);
+		if(w==null) {
+			w = org.mayAutoDelete(transNoAvg, id)?
+					notInOrgDeleteW:
+					notInOrgW;
+			whichWriter.put(id,w);
+		}
+		return w;
+	}
+
+	private boolean check(AuthzTrans trans, Map<String, Boolean> checked, String id) throws OrganizationException {
+		Boolean rv = checked.get(id);
+		if(rv==null) {
+			if(isSpecial(id)) { // do not check against org... too important to delete.
+				return true; 
+			}
+			Organization org = trans.org();
+			if(org != null) {
+				Identity identity = org.getIdentity(trans, id);
+				rv = identity!=null;
+				checked.put(id, rv);
+			} else {
+				throw new OrganizationException("No Organization Found for " + id + ": required for processing");
+			}
+		}
+		return rv;
+	}
+
+    
+	@Override
+    protected void _close(AuthzTrans trans) {
+        session.close();
+    	for(CSV.Writer cw : writerList.values()) {
+    		cw.close();
+    	}
+    }
+
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Notify.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Notify.java
new file mode 100644
index 00000000..cb57497e
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Notify.java
@@ -0,0 +1,185 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */package org.onap.aaf.auth.batch.reports;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.onap.aaf.auth.batch.Batch;
+import org.onap.aaf.auth.batch.reports.bodies.NotifyBody;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.org.Mailer;
+import org.onap.aaf.auth.org.Organization.Identity;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.Access;
+import org.onap.aaf.cadi.CadiException;
+import org.onap.aaf.cadi.client.Holder;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.misc.env.APIException;
+
+public class Notify extends Batch {
+	private final Mailer mailer;
+	private final String mailFrom;
+	private final String header;
+	private final String footer;
+	private List<File> notifyFile;
+
+	public Notify(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+		super(trans.env());
+		String mailerCls = env.getProperty("MAILER");
+		mailFrom = env.getProperty("MAIL_FROM");
+		String header_html = env.getProperty("HEADER_HTML");
+		String footer_html = env.getProperty("FOOTER_HTML");
+		if(mailerCls==null || mailFrom==null || header_html==null || footer_html==null) {
+			throw new APIException("Notify requires MAILER, MAILER_FROM, HEADER_HTML and FOOTER_HTML properties");
+		}
+		try {
+			Class<?> mailc = Class.forName(mailerCls);
+			Constructor<?> mailcst = mailc.getConstructor(Access.class);
+			mailer = (Mailer)mailcst.newInstance(env.access());
+		} catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+			throw new APIException("Unable to construct " + mailerCls,e);
+		}
+		
+		FileInputStream fis = new FileInputStream(header_html);
+		try {
+			byte[] content = new byte[(int)fis.getChannel().size()];
+			fis.read(content);
+			header = new String(content);
+		} finally {
+			fis.close();
+		}
+		
+		fis = new FileInputStream(footer_html);
+		try {
+			byte[] content = new byte[(int)fis.getChannel().size()];
+			fis.read(content);
+			footer = new String(content);
+		} finally {
+			fis.close();
+		}
+	
+		// Class Load possible data
+		NotifyBody.load(env.access());
+		
+        // Create Intermediate Output 
+        File logDir = new File(logDir());
+        notifyFile = new ArrayList<>();
+        if(args().length>0) {
+        	for(int i=0;i<args().length;++i) {
+        		notifyFile.add(new File(logDir, args()[i]));
+        	}
+        }
+	}
+
+	@Override
+	protected void run(AuthzTrans trans) {
+		List<String> toList = new ArrayList<>();
+		List<String> ccList = new ArrayList<>();
+		AuthzTrans noAvg = trans.env().newTransNoAvg();
+		String subject = "Test Notify";
+		boolean urgent = false;
+		
+
+		
+		final Notify notify = this;
+		final Holder<List<String>> info = new Holder<>(null);
+		final Set<String> errorSet = new HashSet<>();
+		
+		try {
+			for(File f : notifyFile) {
+				CSV csv = new CSV(f);
+				try {
+					csv.visit(new CSV.Visitor() {
+						@Override
+						public void visit(List<String> row) throws IOException, CadiException {
+							if("info".equals(row.get(0))) {
+								info.set(row);
+							}
+							if(info.get()==null) {
+								throw new CadiException("First line of Feed MUST contain 'info' record");
+							}
+							String key = row.get(0)+'|'+info.get().get(1);
+							NotifyBody body = NotifyBody.get(key);
+							if(body==null) {
+								errorSet.add("No NotifyBody defined for " + key);
+							} else {
+								body.store(row);
+							}
+						}
+					});
+				} catch (IOException | CadiException e) {
+					e.printStackTrace();
+				}
+				
+				// now create Notification
+				for(NotifyBody nb : NotifyBody.getAll()) {
+					for(String id : nb.users()) {
+						toList.clear();
+						ccList.clear();
+						try {
+							String bodyS = nb.body(noAvg, notify, id);
+							Identity identity = trans.org().getIdentity(noAvg, id);
+							if(!identity.isPerson()) {
+								identity = identity.responsibleTo();
+							}
+							for(int i=1;i<nb.escalation();++i) {
+								if(identity != null) {
+									if(i==1) {
+										toList.add(identity.email());
+									} else {
+										identity=identity.responsibleTo();
+										ccList.add(identity.email());
+									}
+								}
+							}
+								
+							mailer.sendEmail(noAvg, dryRun, mailFrom, toList, ccList, subject, 
+									String.format(header,"2.1.9",Identity.mixedCase(identity.firstName()))+
+									bodyS +
+									footer, urgent);
+						} catch (OrganizationException e) {
+							trans.error().log(e);
+						}
+					}
+				}
+
+			}
+		} finally {
+			for(String s : errorSet) {
+				trans.audit().log(s);
+			}
+		}
+	}
+	
+	@Override
+	protected void _close(AuthzTrans trans) {
+	}
+
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/AbsCredBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/AbsCredBody.java
new file mode 100644
index 00000000..c0d16b06
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/AbsCredBody.java
@@ -0,0 +1,39 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+package org.onap.aaf.auth.batch.reports.bodies;
+
+import java.io.IOException;
+import java.util.List;
+
+public abstract class AbsCredBody extends NotifyBody {
+
+	public AbsCredBody(final String name) throws IOException {
+		super("cred",name);
+	}
+
+	@Override
+	public String user(List<String> row) {
+		if(row.size()>0) {
+			return row.get(1);
+		}
+		return null;
+	}
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyBody.java
new file mode 100644
index 00000000..429ea6d2
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyBody.java
@@ -0,0 +1,185 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+package org.onap.aaf.auth.batch.reports.bodies;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Modifier;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.onap.aaf.auth.batch.reports.Notify;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.cadi.Access;
+import org.onap.aaf.misc.env.APIException;
+
+public abstract class NotifyBody {
+	private static final Map<String,NotifyBody> bodyMap = new HashMap<>();
+
+	protected Map<String,List<List<String>>> rows;
+	private final String name;
+	private final String type;
+	private String date;
+	private int escalation;
+	
+	public NotifyBody(final String type, final String name) {
+		rows = new TreeMap<>();
+		this.name = name;
+		this.type = type;
+		date="";
+		escalation = 1;
+	}
+	
+	public void store(List<String> row) {
+		if(!row.isEmpty()) {
+			if("info".equals(row.get(0))) {
+				if(row.size()>2) {
+					date = row.get(2);
+				}
+				if(row.size()>3) {
+					escalation = Integer.parseInt(row.get(3));
+				}
+				return;
+			} else if(type.equals(row.get(0))) {
+				String user = user(row);
+				if(user!=null) {
+					List<List<String>> lss = rows.get(user); 
+					if(lss == null) {
+						lss = new ArrayList<>();
+						rows.put(user,lss);
+					}
+					lss.add(row);
+				}
+			}
+		}
+	}
+
+	public String name() {
+		return name;
+	}
+	
+	public String date() {
+		return date;
+	}
+	public int escalation() {
+		return escalation;
+	}
+	
+	public Set<String> users() {
+		return rows.keySet();
+	}
+	
+	/**
+	 * ID must be set from Row for Email lookup
+	 * 
+	 * @param trans
+	 * @param n
+	 * @param id
+	 * @param row
+	 * @return
+	 */
+	public abstract String body(AuthzTrans trans, Notify n, String id);
+	
+	/**
+	 * Return "null" if user not found in row... Code will handle.
+	 * @param row
+	 * @return
+	 */
+	protected abstract String user(List<String> row);
+	
+	/**
+	 * Get Notify Body based on key of
+	 * type|name
+	 */
+	public static NotifyBody get(String key) {
+		return bodyMap.get(key);
+	}
+	
+	/**
+	 * Return set of loaded NotifyBodies
+	 * 
+	 */
+	public static Collection<NotifyBody> getAll() {
+		return bodyMap.values();
+	}
+	
+	/**
+	 * @param propAccess 
+	 * @throws URISyntaxException 
+	 * 
+	 */
+	public static void load(Access access) throws APIException, IOException {
+		// class load available NotifyBodies
+		ClassLoader cl = Thread.currentThread().getContextClassLoader();
+		Package pkg = NotifyBody.class.getPackage();
+		String path = pkg.getName().replace('.', '/');
+//		Enumeration<URL> urls = cl.getResources(path);
+//		while(urls.hasMoreElements()) {
+//			URL url = urls.nextElement();
+		URL url = cl.getResource(path);
+		if(url == null) {
+			throw new APIException("Cannot load resources from " + path);
+		}
+		System.out.println(url);
+		File dir;
+		try {
+			dir = new File(url.toURI());
+		} catch (URISyntaxException e) {
+			throw new APIException(e);
+		}
+		if(dir.exists()) {
+			String[] files = dir.list();
+			if(files!=null) {
+				for(String sf : files) {
+					int dot = sf.indexOf('.');
+					if(dot>=0) {
+						String cls = pkg.getName()+'.'+sf.substring(0,dot);
+						try {
+							Class<?> c = cl.loadClass(cls);
+							if(c!=null) {
+								if(!Modifier.isAbstract(c.getModifiers())) {
+									Constructor<?> cst = c.getConstructor(Access.class);
+									NotifyBody nb = (NotifyBody)cst.newInstance(access);
+									if(nb!=null) {
+										bodyMap.put("info|"+nb.name, nb);
+										bodyMap.put(nb.type+'|'+nb.name, nb);
+									}
+								}
+							}
+						} catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+							e.printStackTrace();
+						}
+					}
+				}
+			}
+		}
+//		}
+	}
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyCredBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyCredBody.java
new file mode 100644
index 00000000..db96d50a
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyCredBody.java
@@ -0,0 +1,54 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+package org.onap.aaf.auth.batch.reports.bodies;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.onap.aaf.auth.batch.reports.Notify;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.cadi.Access;
+
+public class NotifyCredBody extends AbsCredBody {
+	private final String explanation;
+	public NotifyCredBody(Access access, String name) throws IOException {
+		super(name);
+		
+		// Default
+		explanation = "The following Credentials are expiring on the dates shown. "
+				+ "Failure to act before the expiration date will cause your App's Authentications to fail.";
+	}
+
+	@Override
+	public String body(AuthzTrans trans, Notify n, String id) {
+		StringBuilder sb = new StringBuilder();
+		sb.append(explanation);
+		sb.append("<br>");
+		sb.append("<tr>\n" + 
+				"<th>Role</th>\n" + 
+				"<th>Expires</th>\n" + 
+				"</tr>\n");
+		for(List<String> row : rows.get(id)) {
+			
+		}
+		return sb.toString();
+	}
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/TwoWeeksNotifyCredBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/TwoWeeksNotifyCredBody.java
new file mode 100644
index 00000000..97f09ac2
--- /dev/null
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/TwoWeeksNotifyCredBody.java
@@ -0,0 +1,31 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+package org.onap.aaf.auth.batch.reports.bodies;
+
+import java.io.IOException;
+
+import org.onap.aaf.cadi.Access;
+
+public class TwoWeeksNotifyCredBody extends NotifyCredBody {
+	public TwoWeeksNotifyCredBody(Access access) throws IOException {
+		super(access, "CredTwoWeek");
+	}
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/ExpiringOrig.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/ExpiringOrig.java
index 7b6487a0..6d759a06 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/ExpiringOrig.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/ExpiringOrig.java
@@ -95,7 +95,7 @@ public class ExpiringOrig extends Batch {
             TimeTaken tt = trans.start("Connect to Cluster with DAOs", Env.REMOTE);
             try {
                 urFutureApprove = new URFutureApprove(trans, cluster,isDryRun());
-                checkOrganizationAcccess(trans, urFutureApprove.question());
+//                checkOrganizationAcccess(trans, urFutureApprove.question());
                 urFutureApproveExec = new URFutureApproveExec(trans, urFutureApprove);
                 urPrint = new URPrint("User Roles:");
                 crDelete = new CredDelete(trans, urFutureApprove);
@@ -142,8 +142,6 @@ public class ExpiringOrig extends Batch {
             }
             email.addTo(address);
     
-        } catch (OrganizationException e) {
-            throw new APIException("Error getting valid Organization",e);
         } finally {
             tt0.done();
         }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
index 3e1a8df2..5f6021b3 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
@@ -71,7 +71,7 @@ public class Remove extends Batch {
             } finally {
                 tt2.done();
             }
-            cqlBatch = new CQLBatch(session); 
+            cqlBatch = new CQLBatch(noAvg.info(),session); 
             
 
         } finally {
@@ -95,6 +95,10 @@ public class Remove extends Batch {
         	remove.add(new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv"));
         }
         
+        for(File f : remove) {
+        	trans.init().log("Processing File:",f.getAbsolutePath());
+        }
+        
         final Holder<Boolean> ur = new Holder<>(false);
         final Holder<Boolean> cred = new Holder<>(false);
         final Holder<Boolean> x509 = new Holder<>(false);
@@ -143,7 +147,7 @@ public class Remove extends Batch {
 											ur.set(true);
 										}
 										hi.set(++i);
-										UserRole.row(sb,row);
+										UserRole.batchDelete(sb,row);
 										hdd.target=UserRoleDAO.TABLE; 
 										hdd.subject=UserRole.histSubject(row);
 										hdd.memo=UserRole.histMemo(memoFmt.get(), row);
@@ -154,7 +158,7 @@ public class Remove extends Batch {
 											cred.set(true);
 										}
 										hi.set(++i);
-										Cred.row(sb,row);
+										Cred.batchDelete(sb,row);
 										hdd.target=CredDAO.TABLE; 
 										hdd.subject=Cred.histSubject(row);
 										hdd.memo=Cred.histMemo(memoFmt.get(), orgName,row);