Refactor Batch: Remove

Issue-ID: AAF-670
Change-Id: If5c961500340826e63fb224c10b44d9aed1d6b5e
Signed-off-by: Instrumental <jonathan.gathman@att.com>

16 files changed, 284 insertions, 66 deletions

diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
index 3d742167..7920e481 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
@@ -33,6 +33,7 @@ import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.GregorianCalendar;
 import java.util.HashSet;
 import java.util.List;
@@ -51,8 +52,8 @@ import org.onap.aaf.auth.org.Organization;
 import org.onap.aaf.auth.org.Organization.Identity;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.auth.org.OrganizationFactory;
-import org.onap.aaf.cadi.PropAccess;
 import org.onap.aaf.cadi.Access.Level;
+import org.onap.aaf.cadi.PropAccess;
 import org.onap.aaf.cadi.config.Config;
 import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
@@ -77,6 +78,7 @@ public abstract class Batch {
     protected static AuthzEnv env;
     protected static Session session;
     protected static Set<String> specialNames;
+    protected static List<String> specialDomains;
     protected static boolean dryRun; 
     protected static String batchEnv;
 
@@ -131,14 +133,18 @@ public abstract class Batch {
 
         // Special names to allow behaviors beyond normal rules
         specialNames = new HashSet<>();
+        specialDomains = new ArrayList<>();
         String names = env.getProperty( "SPECIAL_NAMES" );
         if ( names != null )
         {
             env.info().log("Loading SPECIAL_NAMES");
-            for (String s :names.split(",") )
-            {
+            for (String s :names.split(",") ) {
                 env.info().log("\tspecial: " + s );
-                specialNames.add( s.trim() );
+                if(s.indexOf('@')>0) {
+                	specialNames.add( s.trim() );
+                } else {
+                	specialDomains.add(s.trim());
+                }
             }
         }
     }
@@ -156,13 +162,23 @@ public abstract class Batch {
     }
     
     public boolean isSpecial(String user) {
+    	if(user==null) {
+    		return false;
+    	}
         if (specialNames != null && specialNames.contains(user)) {
             env.info().log("specialName: " + user);
-
             return (true);
         } else {
-            return (false);
+        	if(specialDomains!=null) {
+	        	for(String sd : specialDomains) {
+	        		if(user.endsWith(sd)) {
+	        			env.info().log("specialDomain: " + user + " matches " + sd);
+	        			return (true);
+	        		}
+	        	}
+        	}
         }
+        return (false);
     }
 
 
@@ -459,16 +475,16 @@ public abstract class Batch {
                     Class<?> cls;
                     String classifier = "";
                     try {
-                        cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.update." + toolName);
+                        cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.update." + toolName);
                         classifier = "Update:";
                     } catch (ClassNotFoundException e) {
                         try {
-                            cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.reports." + toolName);
+                            cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.reports." + toolName);
                             classifier = "Report:";
                         } catch (ClassNotFoundException e2) {
                             try {
                                 cls = ClassLoader.getSystemClassLoader()
-                                        .loadClass("org.onap.aaf.auth.temp." + toolName);
+                                        .loadClass("org.onap.aaf.auth.batch.temp." + toolName);
                                 classifier = "Temp Utility:";
                             } catch (ClassNotFoundException e3) {
                                 cls = null;
@@ -476,7 +492,7 @@ public abstract class Batch {
                         }
                     }
                     if (cls != null) {
-                        Constructor<?> cnst = cls.getConstructor(new Class[] { AuthzTrans.class });
+                        Constructor<?> cnst = cls.getConstructor(AuthzTrans.class);
                         batch = (Batch) cnst.newInstance(trans);
                         env.info().log("Begin", classifier, toolName);
                     }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java
index a4f4dcf3..94df581b 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java
@@ -45,6 +45,7 @@ public class CacheTouch extends ActionDAO<String,Void, String> {
             trans.info().printf("Would mark %s cache in DB for clearing: %s",table, text);
             return Result.ok();
         } else {
+        	
             Result<Void> rv = q.clearCache(trans, table);
             trans.info().printf("Set DB Cache %s for clearing: %s",table, text);
             return rv;
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
index 5df5dcdc..c65026a8 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
@@ -66,6 +66,17 @@ public class CQLBatch {
 		} else {
 			return execute();
 		}
-		
+	}
+	
+	public void touch(String table, int begin, int end, boolean dryRun) {
+		StringBuilder sb = begin();
+		for(int i=begin;i<end;++i) {
+			sb.append("UPDATE cache SET touched=dateof(now()) WHERE name='");
+			sb.append(table);
+			sb.append("' AND seg=");
+			sb.append(i);
+			sb.append(";\n");
+		}
+		execute(dryRun);
 	}
 }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
index 979cdf0b..b58506ea 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
@@ -322,4 +322,14 @@ public class Cred  {
         return id.equals(obj);
     }
 
+
+	public static String histSubject(List<String> row) {
+		return row.get(1);
+	}
+
+
+	public static String histMemo(String fmt, String orgName, List<String> row) {
+		return String.format(fmt, row.get(1),orgName,row.get(4));
+	}
+
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
index 30069d64..ecf66fed 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
@@ -321,6 +321,12 @@ public class UserRole implements Cloneable, CacheChange.Data  {
     	sb.append(row.get(3));
     	sb.append("';\n");
     }
-    
 
+	public static String histMemo(String fmt, List<String> row) {
+		return String.format(fmt, row.get(1),row.get(2)+'.'+row.get(3), row.get(4));
+	}
+
+	public static String histSubject(List<String> row) {
+		return row.get(1) + '|' + row.get(2)+'.'+row.get(3);	
+	}
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java
index a59064ee..17f289a1 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java
@@ -22,4 +22,19 @@ package org.onap.aaf.auth.batch.helpers;
 
 public interface Visitor<T> {
 	void visit(T t);
+	
+	public static class Multi<T> implements Visitor<T> {
+		private final Visitor<T>[] visitors;
+		@SafeVarargs
+		public Multi(Visitor<T> ... vs) {
+			visitors  = vs;
+		}
+		
+		@Override
+		public void visit(T t) {
+			for(Visitor<T> v : visitors) {
+				v.visit(t);
+			}
+		}
+	};
 }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java
index 8bdcd100..0ffaa8f5 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java
@@ -33,6 +33,7 @@ import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
 import org.onap.aaf.misc.env.Trans;
 import org.onap.aaf.misc.env.util.Chrono;
+import org.onap.aaf.misc.env.util.Split;
 
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
@@ -59,7 +60,7 @@ public class X509 {
 
     private static void load(Trans trans, Session session, String query, Visitor<X509> visitor) {
         trans.info().log( "query: " + query );
-        TimeTaken tt = trans.start("Read Roles", Env.REMOTE);
+        TimeTaken tt = trans.start("Read X509", Env.REMOTE);
        
         ResultSet results;
         try {
@@ -116,4 +117,25 @@ public class X509 {
     	sb.append(";\n");
 	}
 
+
+	public static String histSubject(List<String> row) {
+		return row.get(4);
+	}
+
+
+	public static String histMemo(String fmt, List<String> row) {
+		String id="n/a";
+		for(String s : Split.splitTrim(',', row.get(4))) {
+			if(s.startsWith("OU=") && s.indexOf('@')>=0) {
+				int colon = s.indexOf(':');
+				if(colon<0) {
+					colon=s.length();
+				}
+				id=s.substring(3,colon);
+				break;
+			}
+		}
+		return String.format(fmt, "Cert for " + id,"CA " + row.get(1),row.get(3));
+	}
+
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
index 1a7db055..e9fd818d 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
@@ -80,7 +80,6 @@ public class Expiring extends Batch {
             
             // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
             Cred.load(trans, session);
-            UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor());
 
             minOwners=1;
 
@@ -100,7 +99,7 @@ public class Expiring extends Batch {
                     	cw.row(INFO,r.name(),Chrono.dateOnlyStamp(expireRange.now),r.reportingLevel());
                     	writerList.put(r.name(),cw);
                     	if("Delete".equals(r.name())) {
-                    		deleteDate = r.getStart();
+                    		deleteDate = r.getEnd();
                     	}
             		}
             	}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
index a884006c..3e1a8df2 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
@@ -23,15 +23,23 @@ package org.onap.aaf.auth.batch.update;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 import org.onap.aaf.auth.batch.Batch;
 import org.onap.aaf.auth.batch.BatchPrincipal;
-import org.onap.aaf.auth.batch.actions.CacheTouch;
 import org.onap.aaf.auth.batch.helpers.CQLBatch;
 import org.onap.aaf.auth.batch.helpers.Cred;
 import org.onap.aaf.auth.batch.helpers.UserRole;
 import org.onap.aaf.auth.batch.helpers.X509;
+import org.onap.aaf.auth.dao.CassAccess;
+import org.onap.aaf.auth.dao.cass.CertDAO;
+import org.onap.aaf.auth.dao.cass.CredDAO;
+import org.onap.aaf.auth.dao.cass.HistoryDAO;
+import org.onap.aaf.auth.dao.cass.UserRoleDAO;
 import org.onap.aaf.auth.env.AuthzTrans;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.cadi.CadiException;
@@ -44,7 +52,7 @@ import org.onap.aaf.misc.env.util.Chrono;
 
 public class Remove extends Batch {
     private final AuthzTrans noAvg;
-	private CacheTouch cacheTouch;
+    private HistoryDAO historyDAO;
 	private CQLBatch cqlBatch;
 
     public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException {
@@ -52,14 +60,14 @@ public class Remove extends Batch {
         trans.info().log("Starting Connection Process");
         
         noAvg = env.newTransNoAvg();
-        noAvg.setUser(new BatchPrincipal("batch:RemoveExpired"));
+        noAvg.setUser(new BatchPrincipal("Remove"));
 
         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
         try {
-        	cacheTouch = new CacheTouch(trans, cluster, dryRun);
+        	historyDAO = new HistoryDAO(trans, cluster, CassAccess.KEYSPACE);
             TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE);
             try {
-                session = cacheTouch.getSession(trans);
+                session = historyDAO.getSession(trans);
             } finally {
                 tt2.done();
             }
@@ -73,52 +81,123 @@ public class Remove extends Batch {
 
     @Override
     protected void run(AuthzTrans trans) {
-        final int maxBatch = 50;
+        final int maxBatch = 25;
 
         // Create Intermediate Output 
         File logDir = new File(logDir());
         
-        File expired = new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv");
-        CSV expiredCSV = new CSV(expired);
-        try {
-        	final StringBuilder sb = cqlBatch.begin();
-            final Holder<Integer> hi = new Holder<Integer>(0);
-			expiredCSV.visit(new CSV.Visitor() {
-				@Override
-				public void visit(List<String> row) throws IOException, CadiException {
-					int i = hi.get();
-					if(i>=maxBatch) {
+        List<File> remove = new ArrayList<>();
+        if(args().length>0) {
+        	for(int i=0;i<args().length;++i) {
+        		remove.add(new File(logDir, args()[i]));
+        	}
+        } else {
+        	remove.add(new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv"));
+        }
+        
+        final Holder<Boolean> ur = new Holder<>(false);
+        final Holder<Boolean> cred = new Holder<>(false);
+        final Holder<Boolean> x509 = new Holder<>(false);
+        final Holder<String> memoFmt = new Holder<String>("");
+        final HistoryDAO.Data hdd = new HistoryDAO.Data();
+        final String orgName = trans.org().getName();
+        
+        hdd.action="delete";
+        hdd.reconstruct = ByteBuffer.allocate(0);
+        hdd.user = noAvg.user();
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
+        hdd.yr_mon = Integer.parseInt(sdf.format(new Date()));
+        
+        try { 
+	        for(File f : remove) {
+	        	trans.info().log("Processing ",f.getAbsolutePath(),"for Deletions");
+	        	if(f.exists()) {
+			        CSV removeCSV = new CSV(f);
+		        		
+			        try {
+			        	final StringBuilder sb = cqlBatch.begin();
+			            final Holder<Integer> hi = new Holder<Integer>(0);
+						removeCSV.visit(new CSV.Visitor() {
+							@Override
+							public void visit(List<String> row) throws IOException, CadiException {
+								int i = hi.get();
+								if(i>=maxBatch) {
+									cqlBatch.execute(dryRun);
+									hi.set(0);
+									cqlBatch.begin();
+									i=0;
+								}
+								switch(row.get(0)) {
+									case "info":
+										switch(row.get(1)) {
+											case "Delete":
+												memoFmt.set("%s expired from %s on %s");
+												break;
+											case "NotInOrgDelete":
+												memoFmt.set("Identity %s was removed from %s on %s");
+												break;
+										}
+										break;
+									case "ur":
+										if(!ur.get()) {
+											ur.set(true);
+										}
+										hi.set(++i);
+										UserRole.row(sb,row);
+										hdd.target=UserRoleDAO.TABLE; 
+										hdd.subject=UserRole.histSubject(row);
+										hdd.memo=UserRole.histMemo(memoFmt.get(), row);
+										historyDAO.createBatch(sb, hdd);
+										break;
+									case "cred":
+										if(!cred.get()) {
+											cred.set(true);
+										}
+										hi.set(++i);
+										Cred.row(sb,row);
+										hdd.target=CredDAO.TABLE; 
+										hdd.subject=Cred.histSubject(row);
+										hdd.memo=Cred.histMemo(memoFmt.get(), orgName,row);
+										historyDAO.createBatch(sb, hdd);
+								    	break;
+									case "x509":
+										if(!x509.get()) {
+											x509.set(true);
+										}
+										hi.set(++i);
+										X509.row(sb,row);
+										hdd.target=CertDAO.TABLE; 
+										hdd.subject=X509.histSubject(row);
+										hdd.memo=X509.histMemo(memoFmt.get(),row);
+										historyDAO.createBatch(sb, hdd);
+										break;
+								}
+							}
+						});
 						cqlBatch.execute(dryRun);
-						hi.set(0);
-						cqlBatch.begin();
-						i=0;
-					}
-					switch(row.get(0)) {
-						case "ur":
-							hi.set(++i);
-							UserRole.row(sb,row);
-							break;
-						case "cred":
-							hi.set(++i);
-							Cred.row(sb,row);
-					    	break;
-						case "x509":
-							hi.set(++i);
-							X509.row(sb,row);
-							break;
+					} catch (IOException | CadiException e) {
+						e.printStackTrace();
 					}
-				}
-			});
-			cqlBatch.execute(dryRun);
-		} catch (IOException | CadiException e) {
-			e.printStackTrace();
-		}
+	        	} else {
+	        		trans.error().log("File",f.getAbsolutePath(),"does not exist.");
+	        	}
+	        }
+        } finally {
+        	if(ur.get()) {
+        		cqlBatch.touch(UserRoleDAO.TABLE, 0, UserRoleDAO.CACHE_SEG, dryRun);
+        	}
+        	if(cred.get()) {
+        		cqlBatch.touch(CredDAO.TABLE, 0, CredDAO.CACHE_SEG, dryRun);
+        	}
+        	if(x509.get()) {
+        		cqlBatch.touch(CertDAO.TABLE, 0, CertDAO.CACHE_SEG, dryRun);
+        	}
+        }
     }
     
     @Override
     protected void _close(AuthzTrans trans) {
         session.close();
-        cacheTouch.close(trans);
     }
 
 }
diff --git a/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/cass/HistoryDAO.java b/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/cass/HistoryDAO.java
index a40b28fb..69d1d26e 100644
--- a/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/cass/HistoryDAO.java
+++ b/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/cass/HistoryDAO.java
@@ -171,6 +171,24 @@ public class HistoryDAO extends CassDAOImpl<AuthzTrans, HistoryDAO.Data> {
         // data.day_time = Integer.parseInt(dayTimeFormat.format(now));
         return data;        
     }
+    
+    public void createBatch(StringBuilder sb, Data data) {
+    	sb.append("INSERT INTO history (");
+    	sb.append(helpers[FIELD_COMMAS]);
+    	sb.append(") VALUES(now(),");
+    	sb.append(data.yr_mon);
+    	sb.append(",'");
+    	sb.append(data.user);
+    	sb.append("','");
+    	sb.append(data.action);
+    	sb.append("','");
+    	sb.append(data.target);
+    	sb.append("','");
+    	sb.append(data.subject);
+    	sb.append("','");
+    	sb.append(data.memo);
+    	sb.append("',null);\n");
+    }
 
     public Result<List<Data>> readByYYYYMM(AuthzTrans trans, int yyyymm) {
         Result<ResultSet> rs = readByYRMN.exec(trans, "yr_mon", yyyymm);
diff --git a/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/hl/Question.java b/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/hl/Question.java
index f55e1c1c..4a307693 100644
--- a/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/hl/Question.java
+++ b/auth/auth-cass/src/main/java/org/onap/aaf/auth/dao/hl/Question.java
@@ -953,27 +953,27 @@ public class Question {
             rv = cacheInfoDAO.touch(trans, NsDAO.TABLE, seg);
         }
         if (all || PermDAO.TABLE.equals(cname)) {
-            int seg[] = series(NsDAO.CACHE_SEG);
+            int seg[] = series(PermDAO.CACHE_SEG);
             for (int i: seg) {cacheClear(trans, PermDAO.TABLE,i);}
             rv = cacheInfoDAO.touch(trans, PermDAO.TABLE,seg);
         }
         if (all || RoleDAO.TABLE.equals(cname)) {
-            int seg[] = series(NsDAO.CACHE_SEG);
+            int seg[] = series(RoleDAO.CACHE_SEG);
             for (int i: seg) {cacheClear(trans, RoleDAO.TABLE,i);}
             rv = cacheInfoDAO.touch(trans, RoleDAO.TABLE,seg);
         }
         if (all || UserRoleDAO.TABLE.equals(cname)) {
-            int seg[] = series(NsDAO.CACHE_SEG);
+            int seg[] = series(UserRoleDAO.CACHE_SEG);
             for (int i: seg) {cacheClear(trans, UserRoleDAO.TABLE,i);}
             rv = cacheInfoDAO.touch(trans, UserRoleDAO.TABLE,seg);
         }
         if (all || CredDAO.TABLE.equals(cname)) {
-            int seg[] = series(NsDAO.CACHE_SEG);
+            int seg[] = series(CredDAO.CACHE_SEG);
             for (int i: seg) {cacheClear(trans, CredDAO.TABLE,i);}
             rv = cacheInfoDAO.touch(trans, CredDAO.TABLE,seg);
         }
         if (all || CertDAO.TABLE.equals(cname)) {
-            int seg[] = series(NsDAO.CACHE_SEG);
+            int seg[] = series(CertDAO.CACHE_SEG);
             for (int i: seg) {cacheClear(trans, CertDAO.TABLE,i);}
             rv = cacheInfoDAO.touch(trans, CertDAO.TABLE,seg);
         }
diff --git a/auth/auth-cmd/src/main/java/org/onap/aaf/auth/cmd/role/ListByPerm.java b/auth/auth-cmd/src/main/java/org/onap/aaf/auth/cmd/role/ListByPerm.java
index 3431a0ea..feb1dec8 100644
--- a/auth/auth-cmd/src/main/java/org/onap/aaf/auth/cmd/role/ListByPerm.java
+++ b/auth/auth-cmd/src/main/java/org/onap/aaf/auth/cmd/role/ListByPerm.java
@@ -3,6 +3,7 @@
  * org.onap.aaf
  * ===========================================================================
  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * Modifications Copyright (C) 2018 IBM.
  * ===========================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -49,8 +50,8 @@ public class ListByPerm extends Cmd {
     }
 
     @Override
-    public int _exec(int _idx, final String ... args) throws CadiException, APIException, LocatorException {
-            int idx = _idx;
+    public int _exec(int idx0, final String ... args) throws CadiException, APIException, LocatorException {
+            int idx = idx0;
         final String type=args[idx];
         final String instance=args[++idx];
         final String action=args[++idx];
diff --git a/auth/auth-core/src/main/java/org/onap/aaf/auth/org/Organization.java b/auth/auth-core/src/main/java/org/onap/aaf/auth/org/Organization.java
index fd252fe4..69cfc7d7 100644
--- a/auth/auth-core/src/main/java/org/onap/aaf/auth/org/Organization.java
+++ b/auth/auth-core/src/main/java/org/onap/aaf/auth/org/Organization.java
@@ -64,6 +64,18 @@ public interface Organization {
         public boolean isPerson();                // Whether a Person or a Machine (App)
         public Organization org();                 // Organization of Identity
 
+
+    	public static String mixedCase(String in) {
+    		StringBuilder sb = new StringBuilder();
+    		for(int i=0;i<in.length();++i) {
+    			if(i==0) {
+    				sb.append(Character.toUpperCase(in.charAt(i)));
+    			} else {
+    				sb.append(Character.toLowerCase(in.charAt(i)));
+    			}
+    		}
+    		return sb.toString();
+    	}
     }
 
 
@@ -83,9 +95,7 @@ public interface Organization {
 
     public void addSupportedRealm(String r);
 
-
-
-    String getDomain();
+    public String getDomain();
 
     /**
      * Get Identity information based on userID
@@ -95,6 +105,20 @@ public interface Organization {
      */
     public Identity getIdentity(AuthzTrans trans, String id) throws OrganizationException;
     
+    /**
+     * May AutoDelete
+     * 
+     * Deletion of an Identity that has been removed from an Organization can be dangerous.  Mistakes may have been made 
+     * in the Organization side, a Feed might be corrupted, an API might not be quite right.  
+     * 
+     * The implementation of this method can use a double check of some sort, such as comparsion of missing ID in Organization
+     * feed with a "Deleted ID" feed.  
+     * 
+     * The failure to be in Organization will still be reported, if returned "false", but if true, it is taken as an 
+     * ok to proceed with deletion. 
+     */
+	public boolean mayAutoDelete(AuthzTrans trans, String id);
+
 
     /**
      * Does the ID pass Organization Standards
@@ -516,15 +540,22 @@ public interface Organization {
                 }
 
             };
+            
+
         }
 
         @Override
         public String[] getPasswordRules() {
             return nullStringArray; 
         }
+        
+    	@Override
+    	public boolean mayAutoDelete(AuthzTrans trans, String id) {
+    		// provide a corresponding feed that indicates that an ID has been intentionally removed from identities.dat table.
+    		return false;
+    	}
 
     };
-
 }
 
 
diff --git a/auth/auth-deforg/src/main/java/org/onap/aaf/org/DefaultOrg.java b/auth/auth-deforg/src/main/java/org/onap/aaf/org/DefaultOrg.java
index 3c6176e6..c7dd3d33 100644
--- a/auth/auth-deforg/src/main/java/org/onap/aaf/org/DefaultOrg.java
+++ b/auth/auth-deforg/src/main/java/org/onap/aaf/org/DefaultOrg.java
@@ -597,4 +597,10 @@ public class DefaultOrg implements Organization {
             return 0;
         }
     }
+
+	@Override
+	public boolean mayAutoDelete(AuthzTrans trans, String user) {
+		// provide a corresponding feed that indicates that an ID has been intentionally removed from identities.dat table.
+		return false;
+	}
 }
diff --git a/auth/docker/.gitignore b/auth/docker/.gitignore
index d3c3ab60..9e4bcf51 100644
--- a/auth/docker/.gitignore
+++ b/auth/docker/.gitignore
@@ -4,3 +4,5 @@
 /aaf.props
 /cass.props
 /policy*
+/*.yaml
+/*.orig
diff --git a/auth/sample/.gitignore b/auth/sample/.gitignore
index 62fd177d..80014ba2 100644
--- a/auth/sample/.gitignore
+++ b/auth/sample/.gitignore
@@ -1 +1,2 @@
 theme
+.DS_Store