Refactor Batch: Remove

Issue-ID: AAF-670
Change-Id: If5c961500340826e63fb224c10b44d9aed1d6b5e
Signed-off-by: Instrumental <jonathan.gathman@att.com>

9 files changed, 214 insertions, 55 deletions

diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
index 3d742167..7920e481 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java
@@ -33,6 +33,7 @@ import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.GregorianCalendar;
 import java.util.HashSet;
 import java.util.List;
@@ -51,8 +52,8 @@ import org.onap.aaf.auth.org.Organization;
 import org.onap.aaf.auth.org.Organization.Identity;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.auth.org.OrganizationFactory;
-import org.onap.aaf.cadi.PropAccess;
 import org.onap.aaf.cadi.Access.Level;
+import org.onap.aaf.cadi.PropAccess;
 import org.onap.aaf.cadi.config.Config;
 import org.onap.aaf.misc.env.APIException;
 import org.onap.aaf.misc.env.Env;
@@ -77,6 +78,7 @@ public abstract class Batch {
     protected static AuthzEnv env;
     protected static Session session;
     protected static Set<String> specialNames;
+    protected static List<String> specialDomains;
     protected static boolean dryRun; 
     protected static String batchEnv;
 
@@ -131,14 +133,18 @@ public abstract class Batch {
 
         // Special names to allow behaviors beyond normal rules
         specialNames = new HashSet<>();
+        specialDomains = new ArrayList<>();
         String names = env.getProperty( "SPECIAL_NAMES" );
         if ( names != null )
         {
             env.info().log("Loading SPECIAL_NAMES");
-            for (String s :names.split(",") )
-            {
+            for (String s :names.split(",") ) {
                 env.info().log("\tspecial: " + s );
-                specialNames.add( s.trim() );
+                if(s.indexOf('@')>0) {
+                	specialNames.add( s.trim() );
+                } else {
+                	specialDomains.add(s.trim());
+                }
             }
         }
     }
@@ -156,13 +162,23 @@ public abstract class Batch {
     }
     
     public boolean isSpecial(String user) {
+    	if(user==null) {
+    		return false;
+    	}
         if (specialNames != null && specialNames.contains(user)) {
             env.info().log("specialName: " + user);
-
             return (true);
         } else {
-            return (false);
+        	if(specialDomains!=null) {
+	        	for(String sd : specialDomains) {
+	        		if(user.endsWith(sd)) {
+	        			env.info().log("specialDomain: " + user + " matches " + sd);
+	        			return (true);
+	        		}
+	        	}
+        	}
         }
+        return (false);
     }
 
 
@@ -459,16 +475,16 @@ public abstract class Batch {
                     Class<?> cls;
                     String classifier = "";
                     try {
-                        cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.update." + toolName);
+                        cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.update." + toolName);
                         classifier = "Update:";
                     } catch (ClassNotFoundException e) {
                         try {
-                            cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.reports." + toolName);
+                            cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.reports." + toolName);
                             classifier = "Report:";
                         } catch (ClassNotFoundException e2) {
                             try {
                                 cls = ClassLoader.getSystemClassLoader()
-                                        .loadClass("org.onap.aaf.auth.temp." + toolName);
+                                        .loadClass("org.onap.aaf.auth.batch.temp." + toolName);
                                 classifier = "Temp Utility:";
                             } catch (ClassNotFoundException e3) {
                                 cls = null;
@@ -476,7 +492,7 @@ public abstract class Batch {
                         }
                     }
                     if (cls != null) {
-                        Constructor<?> cnst = cls.getConstructor(new Class[] { AuthzTrans.class });
+                        Constructor<?> cnst = cls.getConstructor(AuthzTrans.class);
                         batch = (Batch) cnst.newInstance(trans);
                         env.info().log("Begin", classifier, toolName);
                     }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java
index a4f4dcf3..94df581b 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/actions/CacheTouch.java
@@ -45,6 +45,7 @@ public class CacheTouch extends ActionDAO<String,Void, String> {
             trans.info().printf("Would mark %s cache in DB for clearing: %s",table, text);
             return Result.ok();
         } else {
+        	
             Result<Void> rv = q.clearCache(trans, table);
             trans.info().printf("Set DB Cache %s for clearing: %s",table, text);
             return rv;
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
index 5df5dcdc..c65026a8 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatch.java
@@ -66,6 +66,17 @@ public class CQLBatch {
 		} else {
 			return execute();
 		}
-		
+	}
+	
+	public void touch(String table, int begin, int end, boolean dryRun) {
+		StringBuilder sb = begin();
+		for(int i=begin;i<end;++i) {
+			sb.append("UPDATE cache SET touched=dateof(now()) WHERE name='");
+			sb.append(table);
+			sb.append("' AND seg=");
+			sb.append(i);
+			sb.append(";\n");
+		}
+		execute(dryRun);
 	}
 }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
index 979cdf0b..b58506ea 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
@@ -322,4 +322,14 @@ public class Cred  {
         return id.equals(obj);
     }
 
+
+	public static String histSubject(List<String> row) {
+		return row.get(1);
+	}
+
+
+	public static String histMemo(String fmt, String orgName, List<String> row) {
+		return String.format(fmt, row.get(1),orgName,row.get(4));
+	}
+
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
index 30069d64..ecf66fed 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java
@@ -321,6 +321,12 @@ public class UserRole implements Cloneable, CacheChange.Data  {
     	sb.append(row.get(3));
     	sb.append("';\n");
     }
-    
 
+	public static String histMemo(String fmt, List<String> row) {
+		return String.format(fmt, row.get(1),row.get(2)+'.'+row.get(3), row.get(4));
+	}
+
+	public static String histSubject(List<String> row) {
+		return row.get(1) + '|' + row.get(2)+'.'+row.get(3);	
+	}
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java
index a59064ee..17f289a1 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Visitor.java
@@ -22,4 +22,19 @@ package org.onap.aaf.auth.batch.helpers;
 
 public interface Visitor<T> {
 	void visit(T t);
+	
+	public static class Multi<T> implements Visitor<T> {
+		private final Visitor<T>[] visitors;
+		@SafeVarargs
+		public Multi(Visitor<T> ... vs) {
+			visitors  = vs;
+		}
+		
+		@Override
+		public void visit(T t) {
+			for(Visitor<T> v : visitors) {
+				v.visit(t);
+			}
+		}
+	};
 }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java
index 8bdcd100..0ffaa8f5 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java
@@ -33,6 +33,7 @@ import org.onap.aaf.misc.env.Env;
 import org.onap.aaf.misc.env.TimeTaken;
 import org.onap.aaf.misc.env.Trans;
 import org.onap.aaf.misc.env.util.Chrono;
+import org.onap.aaf.misc.env.util.Split;
 
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
@@ -59,7 +60,7 @@ public class X509 {
 
     private static void load(Trans trans, Session session, String query, Visitor<X509> visitor) {
         trans.info().log( "query: " + query );
-        TimeTaken tt = trans.start("Read Roles", Env.REMOTE);
+        TimeTaken tt = trans.start("Read X509", Env.REMOTE);
        
         ResultSet results;
         try {
@@ -116,4 +117,25 @@ public class X509 {
     	sb.append(";\n");
 	}
 
+
+	public static String histSubject(List<String> row) {
+		return row.get(4);
+	}
+
+
+	public static String histMemo(String fmt, List<String> row) {
+		String id="n/a";
+		for(String s : Split.splitTrim(',', row.get(4))) {
+			if(s.startsWith("OU=") && s.indexOf('@')>=0) {
+				int colon = s.indexOf(':');
+				if(colon<0) {
+					colon=s.length();
+				}
+				id=s.substring(3,colon);
+				break;
+			}
+		}
+		return String.format(fmt, "Cert for " + id,"CA " + row.get(1),row.get(3));
+	}
+
 }
\ No newline at end of file
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
index 1a7db055..e9fd818d 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java
@@ -80,7 +80,6 @@ public class Expiring extends Batch {
             
             // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
             Cred.load(trans, session);
-            UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor());
 
             minOwners=1;
 
@@ -100,7 +99,7 @@ public class Expiring extends Batch {
                     	cw.row(INFO,r.name(),Chrono.dateOnlyStamp(expireRange.now),r.reportingLevel());
                     	writerList.put(r.name(),cw);
                     	if("Delete".equals(r.name())) {
-                    		deleteDate = r.getStart();
+                    		deleteDate = r.getEnd();
                     	}
             		}
             	}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
index a884006c..3e1a8df2 100644
--- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
+++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java
@@ -23,15 +23,23 @@ package org.onap.aaf.auth.batch.update;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 import org.onap.aaf.auth.batch.Batch;
 import org.onap.aaf.auth.batch.BatchPrincipal;
-import org.onap.aaf.auth.batch.actions.CacheTouch;
 import org.onap.aaf.auth.batch.helpers.CQLBatch;
 import org.onap.aaf.auth.batch.helpers.Cred;
 import org.onap.aaf.auth.batch.helpers.UserRole;
 import org.onap.aaf.auth.batch.helpers.X509;
+import org.onap.aaf.auth.dao.CassAccess;
+import org.onap.aaf.auth.dao.cass.CertDAO;
+import org.onap.aaf.auth.dao.cass.CredDAO;
+import org.onap.aaf.auth.dao.cass.HistoryDAO;
+import org.onap.aaf.auth.dao.cass.UserRoleDAO;
 import org.onap.aaf.auth.env.AuthzTrans;
 import org.onap.aaf.auth.org.OrganizationException;
 import org.onap.aaf.cadi.CadiException;
@@ -44,7 +52,7 @@ import org.onap.aaf.misc.env.util.Chrono;
 
 public class Remove extends Batch {
     private final AuthzTrans noAvg;
-	private CacheTouch cacheTouch;
+    private HistoryDAO historyDAO;
 	private CQLBatch cqlBatch;
 
     public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException {
@@ -52,14 +60,14 @@ public class Remove extends Batch {
         trans.info().log("Starting Connection Process");
         
         noAvg = env.newTransNoAvg();
-        noAvg.setUser(new BatchPrincipal("batch:RemoveExpired"));
+        noAvg.setUser(new BatchPrincipal("Remove"));
 
         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
         try {
-        	cacheTouch = new CacheTouch(trans, cluster, dryRun);
+        	historyDAO = new HistoryDAO(trans, cluster, CassAccess.KEYSPACE);
             TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE);
             try {
-                session = cacheTouch.getSession(trans);
+                session = historyDAO.getSession(trans);
             } finally {
                 tt2.done();
             }
@@ -73,52 +81,123 @@ public class Remove extends Batch {
 
     @Override
     protected void run(AuthzTrans trans) {
-        final int maxBatch = 50;
+        final int maxBatch = 25;
 
         // Create Intermediate Output 
         File logDir = new File(logDir());
         
-        File expired = new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv");
-        CSV expiredCSV = new CSV(expired);
-        try {
-        	final StringBuilder sb = cqlBatch.begin();
-            final Holder<Integer> hi = new Holder<Integer>(0);
-			expiredCSV.visit(new CSV.Visitor() {
-				@Override
-				public void visit(List<String> row) throws IOException, CadiException {
-					int i = hi.get();
-					if(i>=maxBatch) {
+        List<File> remove = new ArrayList<>();
+        if(args().length>0) {
+        	for(int i=0;i<args().length;++i) {
+        		remove.add(new File(logDir, args()[i]));
+        	}
+        } else {
+        	remove.add(new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv"));
+        }
+        
+        final Holder<Boolean> ur = new Holder<>(false);
+        final Holder<Boolean> cred = new Holder<>(false);
+        final Holder<Boolean> x509 = new Holder<>(false);
+        final Holder<String> memoFmt = new Holder<String>("");
+        final HistoryDAO.Data hdd = new HistoryDAO.Data();
+        final String orgName = trans.org().getName();
+        
+        hdd.action="delete";
+        hdd.reconstruct = ByteBuffer.allocate(0);
+        hdd.user = noAvg.user();
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
+        hdd.yr_mon = Integer.parseInt(sdf.format(new Date()));
+        
+        try { 
+	        for(File f : remove) {
+	        	trans.info().log("Processing ",f.getAbsolutePath(),"for Deletions");
+	        	if(f.exists()) {
+			        CSV removeCSV = new CSV(f);
+		        		
+			        try {
+			        	final StringBuilder sb = cqlBatch.begin();
+			            final Holder<Integer> hi = new Holder<Integer>(0);
+						removeCSV.visit(new CSV.Visitor() {
+							@Override
+							public void visit(List<String> row) throws IOException, CadiException {
+								int i = hi.get();
+								if(i>=maxBatch) {
+									cqlBatch.execute(dryRun);
+									hi.set(0);
+									cqlBatch.begin();
+									i=0;
+								}
+								switch(row.get(0)) {
+									case "info":
+										switch(row.get(1)) {
+											case "Delete":
+												memoFmt.set("%s expired from %s on %s");
+												break;
+											case "NotInOrgDelete":
+												memoFmt.set("Identity %s was removed from %s on %s");
+												break;
+										}
+										break;
+									case "ur":
+										if(!ur.get()) {
+											ur.set(true);
+										}
+										hi.set(++i);
+										UserRole.row(sb,row);
+										hdd.target=UserRoleDAO.TABLE; 
+										hdd.subject=UserRole.histSubject(row);
+										hdd.memo=UserRole.histMemo(memoFmt.get(), row);
+										historyDAO.createBatch(sb, hdd);
+										break;
+									case "cred":
+										if(!cred.get()) {
+											cred.set(true);
+										}
+										hi.set(++i);
+										Cred.row(sb,row);
+										hdd.target=CredDAO.TABLE; 
+										hdd.subject=Cred.histSubject(row);
+										hdd.memo=Cred.histMemo(memoFmt.get(), orgName,row);
+										historyDAO.createBatch(sb, hdd);
+								    	break;
+									case "x509":
+										if(!x509.get()) {
+											x509.set(true);
+										}
+										hi.set(++i);
+										X509.row(sb,row);
+										hdd.target=CertDAO.TABLE; 
+										hdd.subject=X509.histSubject(row);
+										hdd.memo=X509.histMemo(memoFmt.get(),row);
+										historyDAO.createBatch(sb, hdd);
+										break;
+								}
+							}
+						});
 						cqlBatch.execute(dryRun);
-						hi.set(0);
-						cqlBatch.begin();
-						i=0;
-					}
-					switch(row.get(0)) {
-						case "ur":
-							hi.set(++i);
-							UserRole.row(sb,row);
-							break;
-						case "cred":
-							hi.set(++i);
-							Cred.row(sb,row);
-					    	break;
-						case "x509":
-							hi.set(++i);
-							X509.row(sb,row);
-							break;
+					} catch (IOException | CadiException e) {
+						e.printStackTrace();
 					}
-				}
-			});
-			cqlBatch.execute(dryRun);
-		} catch (IOException | CadiException e) {
-			e.printStackTrace();
-		}
+	        	} else {
+	        		trans.error().log("File",f.getAbsolutePath(),"does not exist.");
+	        	}
+	        }
+        } finally {
+        	if(ur.get()) {
+        		cqlBatch.touch(UserRoleDAO.TABLE, 0, UserRoleDAO.CACHE_SEG, dryRun);
+        	}
+        	if(cred.get()) {
+        		cqlBatch.touch(CredDAO.TABLE, 0, CredDAO.CACHE_SEG, dryRun);
+        	}
+        	if(x509.get()) {
+        		cqlBatch.touch(CertDAO.TABLE, 0, CertDAO.CACHE_SEG, dryRun);
+        	}
+        }
     }
     
     @Override
     protected void _close(AuthzTrans trans) {
         session.close();
-        cacheTouch.close(trans);
     }
 
 }