From f95fc5f7ae9cf1832eae2451be883477bbd08f16 Mon Sep 17 00:00:00 2001 From: Instrumental Date: Wed, 21 Nov 2018 16:06:17 -0600 Subject: Batch upgrades for ONAP Issue-ID: AAF-641 Change-Id: I7b5e8b97d8c6c484418c995b6e51507af78fc765 Signed-off-by: Instrumental --- .../src/main/java/org/onap/aaf/auth/Batch.java | 90 ++-- .../java/org/onap/aaf/auth/helpers/CQLBatch.java | 71 +++ .../main/java/org/onap/aaf/auth/helpers/Cred.java | 38 +- .../java/org/onap/aaf/auth/helpers/UserRole.java | 43 +- .../java/org/onap/aaf/auth/helpers/Visitor.java | 25 + .../main/java/org/onap/aaf/auth/helpers/X509.java | 121 +++++ .../java/org/onap/aaf/auth/reports/Expiring.java | 276 +++++++++++ .../org/onap/aaf/auth/reports/ExpiringNext.java | 143 ------ .../java/org/onap/aaf/auth/update/Expiring.java | 506 --------------------- .../org/onap/aaf/auth/update/ExpiringOrig.java | 506 +++++++++++++++++++++ .../java/org/onap/aaf/auth/update/ExpiringP2.java | 2 +- .../onap/aaf/auth/update/NotifyCredExpiring.java | 9 +- .../main/java/org/onap/aaf/auth/update/Remove.java | 123 +++++ .../main/java/org/onap/aaf/auth/update/Upload.java | 302 ++++++++++++ 14 files changed, 1527 insertions(+), 728 deletions(-) create mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/CQLBatch.java create mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Visitor.java create mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java create mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java delete mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/ExpiringNext.java delete mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Expiring.java create mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringOrig.java create mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java create mode 100644 auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java (limited to 'auth/auth-batch/src/main') diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java index 5670ac95..5a19ab38 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/Batch.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.Set; import java.util.TimeZone; -import org.apache.log4j.Logger; import org.onap.aaf.auth.common.Define; import org.onap.aaf.auth.dao.CassAccess; import org.onap.aaf.auth.dao.cass.RoleDAO; @@ -53,12 +52,12 @@ import org.onap.aaf.auth.org.Organization.Identity; import org.onap.aaf.auth.org.OrganizationException; import org.onap.aaf.auth.org.OrganizationFactory; import org.onap.aaf.cadi.PropAccess; +import org.onap.aaf.cadi.Access.Level; +import org.onap.aaf.cadi.config.Config; import org.onap.aaf.misc.env.APIException; import org.onap.aaf.misc.env.Env; import org.onap.aaf.misc.env.StaticSlot; import org.onap.aaf.misc.env.TimeTaken; -import org.onap.aaf.misc.env.impl.Log4JLogTarget; -import org.onap.aaf.misc.env.log4j.LogFileNamer; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ResultSet; @@ -77,7 +76,6 @@ public abstract class Batch { protected final Cluster cluster; protected static AuthzEnv env; protected static Session session; - protected static Logger aspr; protected static Set specialNames; protected static boolean dryRun; protected static String batchEnv; @@ -166,14 +164,7 @@ public abstract class Batch { return (false); } } - - public boolean isMechID(String user) { - if (user.matches("m[0-9][0-9][0-9][0-9][0-9]")) { - return (true); - } else { - return (false); - } - } + protected PrintStream fallout(PrintStream inFallout, String logType) throws IOException { @@ -391,50 +382,51 @@ public abstract class Batch { try { Define.set(access); rootNs =Define.ROOT_NS(); - - File f = new File("etc/authzBatch.props"); - try { - if (f.exists()) { - filename = f.getAbsolutePath(); - is = new FileInputStream(f); - propLoc = f.getPath(); - } else { - URL rsrc = ClassLoader.getSystemResource("authBatch.props"); - filename = rsrc.toString(); - is = rsrc.openStream(); - propLoc = rsrc.getPath(); - } - access.load(is); - } finally { - if (is == null) { - System.err.println("authBatch.props must exist in etc dir, or in Classpath"); - System.exit(1); - } - is.close(); - } + if(access.getProperty(Config.CADI_PROP_FILES)==null) { + File f = new File("authBatch.props"); + try { + if (f.exists()) { + filename = f.getAbsolutePath(); + is = new FileInputStream(f); + propLoc = f.getPath(); + } else { + URL rsrc = ClassLoader.getSystemResource("authBatch.props"); + filename = rsrc.toString(); + is = rsrc.openStream(); + propLoc = rsrc.getPath(); + } + access.load(is); + } finally { + if (is == null) { + System.err.println("authBatch.props must exist in current dir, or in Classpath"); + System.exit(1); + } + is.close(); + } + if (filename != null) { + access.log(Level.INFO,"Instantiated properties from", filename); + } + + // Log where Config found + access.log(Level.INFO,"Configuring from", propLoc); + } env = new AuthzEnv(access); transferVMProps(env, CASS_ENV, "DRY_RUN", "NS", "Organization"); // Flow all Env Logs to Log4j, with ENV - LogFileNamer lfn; - lfn = new LogFileNamer(logDir(),"").noPID(); - lfn.setAppender("authz-batch"); - lfn.setAppender("aspr|ASPR"); - lfn.setAppender("sync"); - lfn.setAppender("jobchange"); - lfn.setAppender("validateuser"); - aspr = Logger.getLogger("aspr"); - Log4JLogTarget.setLog4JEnv("authz-batch", env); - if (filename != null) { - env.init().log("Instantiated properties from", filename); - } - - // Log where Config found - env.info().log("Configuring from", propLoc); - propLoc = null; +// LogFileNamer lfn; +// lfn = new LogFileNamer(logDir(),"").noPID(); +// lfn.setAppender("authz-batch"); +// lfn.setAppender("aspr|ASPR"); +// lfn.setAppender("sync"); +// lfn.setAppender("jobchange"); +// lfn.setAppender("validateuser"); +// aspr = Logger.getLogger("aspr"); +// Log4JLogTarget.setLog4JEnv("authz-batch", env); +// propLoc = null; Batch batch = null; // setup ATTUser and Organization Slots before starting this: diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/CQLBatch.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/CQLBatch.java new file mode 100644 index 00000000..e4d487d6 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/CQLBatch.java @@ -0,0 +1,71 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + */ + +package org.onap.aaf.auth.helpers; + +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Session; + +public class CQLBatch { + private Session session; + private StringBuilder sb; + private int hasAdded; + + public CQLBatch(Session session) { + this.session = session; + sb = new StringBuilder(); + hasAdded = 0; + } + public StringBuilder begin() { + sb.setLength(0); + sb.append("BEGIN BATCH\n"); + hasAdded = sb.length(); + return sb; + } + + private boolean end() { + if(sb.length()==hasAdded) { + System.out.println("Nothing to Process"); + return false; + } else { + sb.append("APPLY BATCH;\n"); + System.out.println(sb); + return true; + } + } + + public ResultSet execute() { + if(end()) { + return session.execute(sb.toString()); + } else { + return null; + } + } + + public ResultSet execute(boolean dryRun) { + if(dryRun) { + end(); + return null; + } else { + return execute(); + } + + } +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java index 2f6ed415..483f70f4 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Cred.java @@ -32,9 +32,12 @@ import java.util.TreeMap; import org.onap.aaf.auth.dao.cass.CredDAO; import org.onap.aaf.auth.dao.hl.Question; +import org.onap.aaf.auth.helpers.Cred.Instance; +import org.onap.aaf.cadi.util.CSV; import org.onap.aaf.misc.env.Env; import org.onap.aaf.misc.env.TimeTaken; import org.onap.aaf.misc.env.Trans; +import org.onap.aaf.misc.env.util.Chrono; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; @@ -67,6 +70,10 @@ public class Cred { this.other = other; this.written = new Date(written); } + + public String toString() { + return expires.toString() + type; + } } public Date last(final int ... types) { @@ -175,7 +182,7 @@ public class Cred { trans.info().log("Found",count,"creds"); } } - + /** * Count entries in Cred data. * Note, as opposed to other methods, need to load the whole cred table for the Types. @@ -190,17 +197,6 @@ public class Cred { } } return cc; -// String query = "select count(*) from authz.cred LIMIT 1000000;"; -// trans.info().log( "query: " + query ); -// TimeTaken tt = trans.start("Count Credentials", Env.REMOTE); -// ResultSet results; -// try { -// Statement stmt = new SimpleStatement(query).setReadTimeoutMillis(12000); -// results = session.execute(stmt); -// return results.one().getLong(0); -// } finally { -// tt.done(); -// } } public static class CredCount { @@ -273,7 +269,23 @@ public class Cred { } - public String toString() { + public void row(CSV.Writer csvw, Instance inst) { + csvw.row("cred",id,ns,Integer.toString(inst.type),Chrono.dateOnlyStamp(inst.expires),Long.toString(inst.expires.getTime())); + } + + + public static void row(StringBuilder sb, List row) { + sb.append("DELETE from authz.cred WHERE id='"); + sb.append(row.get(1)); + sb.append("' AND type="); + sb.append(Integer.parseInt(row.get(3))); + sb.append(" AND expires=dateof(maxtimeuuid("); + sb.append(row.get(5)); + sb.append("));\n"); + } + + + public String toString() { StringBuilder sb = new StringBuilder(id); sb.append('['); for (Instance i : instances) { diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java index 762f6c66..eb1f821c 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/UserRole.java @@ -33,6 +33,7 @@ import org.onap.aaf.auth.actions.URDelete; import org.onap.aaf.auth.dao.cass.UserRoleDAO; import org.onap.aaf.auth.dao.cass.UserRoleDAO.Data; import org.onap.aaf.auth.env.AuthzTrans; +import org.onap.aaf.cadi.util.CSV; import org.onap.aaf.misc.env.Env; import org.onap.aaf.misc.env.TimeTaken; import org.onap.aaf.misc.env.Trans; @@ -105,19 +106,19 @@ public class UserRole implements Cloneable, CacheChange.Data { return byRole; } - public static void load(Trans trans, Session session, Creator creator ) { - load(trans,session,creator,null); + public static void load(Trans trans, Session session, Creator creator, Visitor visitor ) { + load(trans,session,creator,null,visitor); } - public static void loadOneRole(Trans trans, Session session, Creator creator, String role) { - load(trans,session,creator,"role='" + role +"' ALLOW FILTERING;"); + public static void loadOneRole(Trans trans, Session session, Creator creator, String role, Visitor visitor) { + load(trans,session,creator,"role='" + role +"' ALLOW FILTERING;",visitor); } - public static void loadOneUser(Trans trans, Session session, Creator creator, String user ) { - load(trans,session,creator,"role='"+ user +"';"); + public static void loadOneUser(Trans trans, Session session, Creator creator, String user, Visitor visitor ) { + load(trans,session,creator,"role='"+ user +"';",visitor); } - private static void load(Trans trans, Session session, Creator creator, String where) { + private static void load(Trans trans, Session session, Creator creator, String where, Visitor visitor) { String query = creator.query(where); trans.info().log( "query: " + query ); TimeTaken tt = trans.start("Read UserRoles", Env.REMOTE); @@ -132,7 +133,7 @@ public class UserRole implements Cloneable, CacheChange.Data { try { tt = trans.start("Load UserRole", Env.SUB); try { - iterateResults(creator, results.iterator()); + iterateResults(creator, results.iterator(), visitor); } finally { tt.done(); } @@ -141,12 +142,19 @@ public class UserRole implements Cloneable, CacheChange.Data { } } - private static void iterateResults(Creator creator, Iterator iter ) { + private static void iterateResults(Creator creator, Iterator iter, Visitor visit ) { Row row; while (iter.hasNext()) { ++totalLoaded; row = iter.next(); UserRole ur = creator.create(row); + visit.visit(ur); + } + } + + public static class DataLoadVisitor implements Visitor { + @Override + public void visit(UserRole ur) { data.add(ur); List lur = byUser.get(ur.urdd.user); @@ -162,9 +170,9 @@ public class UserRole implements Cloneable, CacheChange.Data { byRole.put(ur.urdd.role, lur); } lur.add(ur); - } + } } - + public int totalLoaded() { return totalLoaded; } @@ -296,4 +304,17 @@ public class UserRole implements Cloneable, CacheChange.Data { cache.resetLocalData(); } + public void row(CSV.Writer csvw) { + csvw.row("ur",user(),role(),Chrono.dateOnlyStamp(expires())); + } + + public static void row(StringBuilder sb, List row) { + sb.append("DELETE from authz.user_role WHERE user='"); + sb.append(row.get(1)); + sb.append("' AND role='"); + sb.append(row.get(2)); + sb.append("';\n"); + } + + } \ No newline at end of file diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Visitor.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Visitor.java new file mode 100644 index 00000000..1eb9ffec --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/Visitor.java @@ -0,0 +1,25 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + */ + +package org.onap.aaf.auth.helpers; + +public interface Visitor { + void visit(T t); +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java new file mode 100644 index 00000000..bb75e110 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/helpers/X509.java @@ -0,0 +1,121 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ + +package org.onap.aaf.auth.helpers; + +import java.awt.HeadlessException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.cert.X509Certificate; +import java.util.Iterator; +import java.util.List; + +import org.onap.aaf.cadi.Hash; +import org.onap.aaf.cadi.Symm; +import org.onap.aaf.cadi.util.CSV; +import org.onap.aaf.misc.env.Env; +import org.onap.aaf.misc.env.TimeTaken; +import org.onap.aaf.misc.env.Trans; +import org.onap.aaf.misc.env.util.Chrono; + +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Row; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SimpleStatement; +import com.datastax.driver.core.Statement; + +public class X509 { + public final String ca,id,x500,x509; + public ByteBuffer serial; + + public X509(String ca, String id, String x500, String x509, ByteBuffer serial) { + this.ca = ca; + this.id = id; + this.x500 = x500; + this.x509 = x509; + this.serial = serial; + } + + + public static void load(Trans trans, Session session, Visitor visitor) { + load(trans,session,"select ca, id, x500, x509, serial from authz.x509;", visitor); + } + + private static void load(Trans trans, Session session, String query, Visitor visitor) { + trans.info().log( "query: " + query ); + TimeTaken tt = trans.start("Read Roles", Env.REMOTE); + + ResultSet results; + try { + Statement stmt = new SimpleStatement( query ); + results = session.execute(stmt); + } finally { + tt.done(); + } + + int count = 0; + try { + Iterator iter = results.iterator(); + Row row; + tt = trans.start("Load X509s", Env.SUB); + try { + while (iter.hasNext()) { + ++count; + row = iter.next(); + visitor.visit(new X509(row.getString(0),row.getString(1), row.getString(2),row.getString(3),row.getBytes(4))); + } + } finally { + tt.done(); + } + } finally { + trans.info().log("Found",count,"X509 Certificates"); + } + } + + public static long count(Trans trans, Session session) { + String query = "select count(*) from authz.x509 LIMIT 1000000;"; + trans.info().log( "query: " + query ); + TimeTaken tt = trans.start("Count x509s", Env.REMOTE); + ResultSet results; + try { + Statement stmt = new SimpleStatement(query).setReadTimeoutMillis(12000); + results = session.execute(stmt); + return results.one().getLong(0); + } finally { + tt.done(); + } + } + + + public void row(CSV.Writer cw, X509Certificate x509Cert) throws IOException { + cw.row("x509",ca,Hash.toHex(serial.array()),Chrono.dateOnlyStamp(x509Cert.getNotAfter()),x500); + } + + + public static void row(StringBuilder sb, List row) throws IOException { + sb.append("DELETE from authz.x509 WHERE ca='"); + sb.append(row.get(1)); + sb.append("' AND serial="); + sb.append(row.get(2)); + sb.append(";\n"); + } + +} \ No newline at end of file diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java new file mode 100644 index 00000000..d283f414 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/Expiring.java @@ -0,0 +1,276 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ + +package org.onap.aaf.auth.reports; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.onap.aaf.auth.Batch; +import org.onap.aaf.auth.dao.cass.CredDAO; +import org.onap.aaf.auth.env.AuthzTrans; +import org.onap.aaf.auth.helpers.Cred; +import org.onap.aaf.auth.helpers.Cred.Instance; +import org.onap.aaf.auth.helpers.UserRole; +import org.onap.aaf.auth.helpers.Visitor; +import org.onap.aaf.auth.helpers.X509; +import org.onap.aaf.auth.org.OrganizationException; +import org.onap.aaf.cadi.configure.Factory; +import org.onap.aaf.cadi.util.CSV; +import org.onap.aaf.cadi.util.CSV.Writer; +import org.onap.aaf.misc.env.APIException; +import org.onap.aaf.misc.env.Env; +import org.onap.aaf.misc.env.TimeTaken; +import org.onap.aaf.misc.env.util.Chrono; + + +public class Expiring extends Batch { + + private int minOwners; + private ArrayList writerList; + private File logDir; + private Date now; + private Date twoWeeksPast; + private Writer twoWeeksPastCSV; + private Date twoWeeksAway; + private Writer twoWeeksAwayCSV; + private Date oneMonthAway; + private Writer oneMonthAwayCSV; + private Date twoMonthsAway; + private Writer twoMonthsAwayCSV; + + public Expiring(AuthzTrans trans) throws APIException, IOException, OrganizationException { + super(trans.env()); + trans.info().log("Starting Connection Process"); + + TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); + try { + TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE); + try { + session = cluster.connect(); + } finally { + tt.done(); + } + + // Load Cred. We don't follow Visitor, because we have to gather up everything into Identity Anyway + Cred.load(trans, session); + + minOwners=1; + + // Create Intermediate Output + writerList = new ArrayList(); + logDir = new File(logDir()); + logDir.mkdirs(); + + GregorianCalendar gc = new GregorianCalendar(); + now = gc.getTime(); + gc.add(GregorianCalendar.WEEK_OF_MONTH, -2); + twoWeeksPast = gc.getTime(); + File file = new File(logDir,"Expired"+Chrono.dateOnlyStamp(now)+".csv"); + twoWeeksPastCSV = new CSV(file).writer(); + writerList.add(twoWeeksPastCSV); + + gc.add(GregorianCalendar.WEEK_OF_MONTH, 2+2); + twoWeeksAway = gc.getTime(); + file = new File(logDir,"TwoWeeksAway"+Chrono.dateOnlyStamp(now)+".csv"); + twoWeeksAwayCSV = new CSV(file).writer(); + writerList.add(twoWeeksAwayCSV); + + gc.add(GregorianCalendar.WEEK_OF_MONTH, -2); + gc.add(GregorianCalendar.MONTH, 1); + oneMonthAway = gc.getTime(); + file = new File(logDir,"OneMonthAway"+Chrono.dateOnlyStamp(now)+".csv"); + oneMonthAwayCSV = new CSV(file).writer(); + writerList.add(oneMonthAwayCSV); + + gc.add(GregorianCalendar.MONTH, 1); + twoMonthsAway = gc.getTime(); + file = new File(logDir,"TwoMonthsAway"+Chrono.dateOnlyStamp(now)+".csv"); + twoMonthsAwayCSV = new CSV(file).writer(); + writerList.add(twoMonthsAwayCSV); + } finally { + tt0.done(); + } + } + + @Override + protected void run(AuthzTrans trans) { + try { + File file = new File(logDir, "AllOwnersExpired" + Chrono.dateOnlyStamp(now) + ".csv"); + final CSV ownerCSV = new CSV(file); + + Map> owners = new TreeMap>(); + trans.info().log("Process UserRoles"); + UserRole.load(trans, session, UserRole.v2_0_11, new Visitor() { + @Override + public void visit(UserRole ur) { + // Cannot just delete owners, unless there is at least one left. Process later + if ("owner".equals(ur.rname())) { + Set urs = owners.get(ur.role()); + if (urs == null) { + urs = new HashSet(); + owners.put(ur.role(), urs); + } + urs.add(ur); + } else { + writeAnalysis(ur); + } + } + }); + + // Now Process Owners, one owner Role at a time, ensuring one is left, + // preferably + // a good one. If so, process the others as normal. Otherwise, write + // ExpiredOwners + // report + if (!owners.values().isEmpty()) { + // Lazy Create file + CSV.Writer expOwner = null; + try { + for (Set sur : owners.values()) { + int goodOwners = 0; + for (UserRole ur : sur) { + if (ur.expires().after(now)) { + ++goodOwners; + } + } + + for (UserRole ur : sur) { + if (goodOwners >= minOwners) { + writeAnalysis(ur); + } else { + if (expOwner == null) { + expOwner = ownerCSV.writer(); + } + expOwner.row(ur.role(), ur.user(), ur.expires()); + } + } + } + } finally { + expOwner.close(); + } + } + + trans.info().log("Checking for Expired Credentials"); + for (Cred cred : Cred.data.values()) { + List linst = cred.instances; + if(linst!=null) { + Instance lastBath = null; + for(Instance inst : linst) { + if(inst.expires.before(twoWeeksPast)) { + cred.row(twoWeeksPastCSV,inst); + } else if(inst.expires.after(now)){ + if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) { + if(lastBath==null || lastBath.expires.before(inst.expires)) { + lastBath = inst; + } + } else if(inst.type==CredDAO.CERT_SHA256_RSA) { + writeAnalysis(cred, inst); + } + } + } + writeAnalysis(cred, lastBath); + } + } + + trans.info().log("Checking for Expired X509s"); + X509.load(trans, session, new Visitor() { + @Override + public void visit(X509 x509) { + try { + for(Certificate cert : Factory.toX509Certificate(x509.x509)) { + writeAnalysis(x509, (X509Certificate)cert); + } + } catch (CertificateException | IOException e) { + trans.error().log(e, "Error Decrypting X509"); + } + + } + }); + } catch (FileNotFoundException e) { + trans.info().log(e); + } + } + + + protected void writeAnalysis(UserRole ur) { + if(ur.expires().before(twoWeeksPast)) { + ur.row(twoWeeksPastCSV); + } else { + if(ur.expires().after(now) && ur.expires().before(twoWeeksAway)) { + ur.row(twoWeeksAwayCSV); + } else { + if(ur.expires().before(oneMonthAway)) { + ur.row(oneMonthAwayCSV); + } else { + if(ur.expires().before(twoMonthsAway)) { + ur.row(twoMonthsAwayCSV); + } + } + } + } + } + + protected void writeAnalysis(Cred cred, Instance inst) { + if(inst!=null) { + if(inst.expires.after(now) && inst.expires.before(twoWeeksAway)) { + cred.row(twoWeeksAwayCSV, inst); + } else { + if(inst.expires.before(oneMonthAway)) { + cred.row(oneMonthAwayCSV, inst); + } else { + if(inst.expires.before(twoMonthsAway)) { + cred.row(twoMonthsAwayCSV, inst); + } + } + } + } + } + + protected void writeAnalysis(X509 x509, X509Certificate x509Cert) throws IOException { + if(x509Cert!=null) { + if(twoWeeksPast.after(x509Cert.getNotAfter())) { + x509.row(twoWeeksPastCSV,x509Cert); + } + } + } + + @Override + protected void _close(AuthzTrans trans) { + session.close(); + for(CSV.Writer cw : writerList) { + cw.close(); + } + } + +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/ExpiringNext.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/ExpiringNext.java deleted file mode 100644 index d34f9248..00000000 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/reports/ExpiringNext.java +++ /dev/null @@ -1,143 +0,0 @@ -/** - * ============LICENSE_START==================================================== - * org.onap.aaf - * =========================================================================== - * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - * =========================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END==================================================== - * - */ - -package org.onap.aaf.auth.reports; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.List; - -import org.onap.aaf.auth.Batch; -import org.onap.aaf.auth.dao.cass.CredDAO; -import org.onap.aaf.auth.env.AuthzTrans; -import org.onap.aaf.auth.helpers.Cred; -import org.onap.aaf.auth.helpers.UserRole; -import org.onap.aaf.auth.helpers.Cred.Instance; -import org.onap.aaf.auth.org.OrganizationException; -import org.onap.aaf.misc.env.APIException; -import org.onap.aaf.misc.env.Env; -import org.onap.aaf.misc.env.TimeTaken; -import org.onap.aaf.misc.env.util.Chrono; - -public class ExpiringNext extends Batch { - - public ExpiringNext(AuthzTrans trans) throws APIException, IOException, OrganizationException { - super(trans.env()); - trans.info().log("Starting Connection Process"); - - TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); - try { - TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE); - try { - session = cluster.connect(); - } finally { - tt.done(); - } - - UserRole.load(trans, session, UserRole.v2_0_11); - Cred.load(trans, session); - } finally { - tt0.done(); - } - } - - @Override - protected void run(AuthzTrans trans) { - GregorianCalendar gc = new GregorianCalendar(); - Date now = gc.getTime(); - gc.add(GregorianCalendar.WEEK_OF_MONTH, 2); - Date twoWeeks = gc.getTime(); - // Set time way off - gc.set(GregorianCalendar.YEAR, 3000); - Date earliestUR = gc.getTime(); - Date earliestCred = gc.getTime(); - // Run for Roles - List expiring = new ArrayList<>(); - - trans.info().log("Checking for Expired UserRoles"); - for (UserRole ur : UserRole.getData()) { - if (ur.expires().after(now)) { - if (ur.expires().before(twoWeeks)) { - expiring.add(Chrono.dateOnlyStamp(ur.expires()) + ":\t" + ur.user() + '\t' + ur.role()); - } - if (ur.expires().before(earliestUR)) { - earliestUR = ur.expires(); - } - } - } - - if (expiring.size()>0) { - Collections.sort(expiring,Collections.reverseOrder()); - for (String s : expiring) { - System.err.print('\t'); - System.err.println(s); - } - trans.info().printf("Earliest Expiring UR is %s\n\n", Chrono.dateOnlyStamp(earliestUR)); - } else { - trans.info().printf("No Expiring UserRoles within 2 weeks"); - } - - expiring.clear(); - - trans.info().log("Checking for Expired Credentials"); - for ( Cred creds : Cred.data.values()) { - Instance lastInstance=null; - for (Instance inst : creds.instances) { - if (inst.type==CredDAO.BASIC_AUTH || inst.type==CredDAO.BASIC_AUTH_SHA256) { - if (lastInstance == null || inst.expires.after(lastInstance.expires)) { - lastInstance = inst; - } - } - } - if (lastInstance!=null) { - if (lastInstance.expires.after(now)) { - if (lastInstance.expires.before(twoWeeks)) { - expiring.add(Chrono.dateOnlyStamp(lastInstance.expires) + ": \t" + creds.id); - } - } - if (lastInstance.expires.before(earliestCred)) { - earliestCred = lastInstance.expires; - } - } - } - - if (expiring.size()>0) { - Collections.sort(expiring,Collections.reverseOrder()); - for (String s : expiring) { - System.err.print('\t'); - System.err.println(s); - } - trans.info().printf("Earliest Expiring Cred is %s\n\n", Chrono.dateOnlyStamp(earliestCred)); - } else { - trans.info().printf("No Expiring Creds within 2 weeks"); - } - - } - - @Override - protected void _close(AuthzTrans trans) { - session.close(); - } - -} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Expiring.java deleted file mode 100644 index e9f0e726..00000000 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Expiring.java +++ /dev/null @@ -1,506 +0,0 @@ -/** - * ============LICENSE_START==================================================== - * org.onap.aaf - * =========================================================================== - * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - * =========================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END==================================================== - * - */ - -package org.onap.aaf.auth.update; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.List; -import java.util.UUID; - -import org.onap.aaf.auth.Batch; -import org.onap.aaf.auth.BatchPrincipal; -import org.onap.aaf.auth.actions.Action; -import org.onap.aaf.auth.actions.ActionDAO; -import org.onap.aaf.auth.actions.CacheTouch; -import org.onap.aaf.auth.actions.CredDelete; -import org.onap.aaf.auth.actions.CredPrint; -import org.onap.aaf.auth.actions.Email; -import org.onap.aaf.auth.actions.Message; -import org.onap.aaf.auth.actions.URDelete; -import org.onap.aaf.auth.actions.URFutureApprove; -import org.onap.aaf.auth.actions.URFutureApproveExec; -import org.onap.aaf.auth.actions.URPrint; -import org.onap.aaf.auth.dao.cass.ApprovalDAO; -import org.onap.aaf.auth.dao.cass.CredDAO; -import org.onap.aaf.auth.dao.cass.FutureDAO; -import org.onap.aaf.auth.dao.hl.Function.FUTURE_OP; -import org.onap.aaf.auth.dao.hl.Function.OP_STATUS; -import org.onap.aaf.auth.env.AuthzTrans; -import org.onap.aaf.auth.helpers.Approval; -import org.onap.aaf.auth.helpers.Cred; -import org.onap.aaf.auth.helpers.Future; -import org.onap.aaf.auth.helpers.NS; -import org.onap.aaf.auth.helpers.Role; -import org.onap.aaf.auth.helpers.UserRole; -import org.onap.aaf.auth.helpers.Cred.Instance; -import org.onap.aaf.auth.layer.Result; -import org.onap.aaf.auth.org.OrganizationException; -import org.onap.aaf.auth.org.Organization.Identity; -import org.onap.aaf.misc.env.APIException; -import org.onap.aaf.misc.env.Env; -import org.onap.aaf.misc.env.TimeTaken; -import org.onap.aaf.misc.env.util.Chrono; - -public class Expiring extends Batch { - private CredPrint crPrint; - private URFutureApprove urFutureApprove; - private URFutureApproveExec urFutureApproveExec; - private CredDelete crDelete; - private URDelete urDelete; - private final CacheTouch cacheTouch; - private final AuthzTrans noAvg; - private final ApprovalDAO apprDAO; - private final FutureDAO futureDAO; - private final PrintStream urDeleteF,urRecoverF; - private final URPrint urPrint; - private Email email; - private File deletesFile; - - public Expiring(AuthzTrans trans) throws APIException, IOException, OrganizationException { - super(trans.env()); - trans.info().log("Starting Connection Process"); - - noAvg = env.newTransNoAvg(); - noAvg.setUser(new BatchPrincipal("batch:Expiring")); - - TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); - try { - crPrint = new CredPrint("Expired:"); - - TimeTaken tt = trans.start("Connect to Cluster with DAOs", Env.REMOTE); - try { - urFutureApprove = new URFutureApprove(trans, cluster,isDryRun()); - checkOrganizationAcccess(trans, urFutureApprove.question()); - urFutureApproveExec = new URFutureApproveExec(trans, urFutureApprove); - urPrint = new URPrint("User Roles:"); - crDelete = new CredDelete(trans, urFutureApprove); - urDelete = new URDelete(trans,urFutureApprove); - cacheTouch = new CacheTouch(trans, urFutureApprove); - - // Reusing... don't destroy - apprDAO = urFutureApprove.question().approvalDAO; - futureDAO = urFutureApprove.question().futureDAO; - - TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE); - try { - session = urFutureApprove.getSession(trans); - } finally { - tt2.done(); - } - } finally { - tt.done(); - } - - File data_dir = new File(env.getProperty("aaf_data_dir")); - if (!data_dir.exists() || !data_dir.canWrite() || !data_dir.canRead()) { - throw new IOException("Cannot read/write to Data Directory "+ data_dir.getCanonicalPath() + ": EXITING!!!"); - } - UserRole.setDeleteStream( - urDeleteF = new PrintStream(new FileOutputStream(deletesFile = new File(data_dir,"UserRoleDeletes.dat"),false))); - UserRole.setRecoverStream( - urRecoverF = new PrintStream(new FileOutputStream(new File(data_dir,"UserRoleRecover.dat"),false))); - UserRole.load(trans, session, UserRole.v2_0_11); - - Cred.load(trans, session); - NS.load(trans, session,NS.v2_0_11); - Future.load(trans,session,Future.withConstruct); - Approval.load(trans,session,Approval.v2_0_17); - Role.load(trans, session); - - email = new Email(); - email.subject("AAF Expiring Process Alert (ENV: %s)",batchEnv); - email.preamble("Expiring Process Alert for %s",batchEnv); - email.signature("Sincerely,\nAAF Expiring Batch Process\n"); - String address = env.getProperty("ALERT_TO_ADDRESS"); - if (address==null) { - throw new APIException("ALERT_TO_ADDRESS property is required"); - } - email.addTo(address); - - } catch (OrganizationException e) { - throw new APIException("Error getting valid Organization",e); - } finally { - tt0.done(); - } - } - - @Override - protected void run(AuthzTrans trans) { - // Setup Date boundaries - - final GregorianCalendar gc = new GregorianCalendar(); - final Date now = gc.getTime(); - - gc.add(GregorianCalendar.MONTH, 1); - Date future = gc.getTime(); -// Date earliest = null; - - // reset - gc.setTime(now); - gc.add(GregorianCalendar.DAY_OF_MONTH, -7); // save Expired Roles for 7 days. - Date tooLate = gc.getTime(); - - TimeTaken tt; - - // Clean out Approvals UserRoles are fixed up. - String memo; - for (List la : Approval.byUser.values()) { - for (Approval a : la ) { - memo = a.getMemo(); - if (memo!=null && (memo.contains("Re-Approval") || memo.contains("Re-Validate"))) { - String role = a.getRole(); - if (role!=null) { - UserRole ur = UserRole.get(a.getUser(), a.getRole()); - Future f=null; - if (ur!=null) { - if (ur.expires().after(future)) { // no need for Approval anymore - a.delayDelete(noAvg, apprDAO, dryRun, "User Role already Extended"); - UUID tkt = a.getTicket(); - if (tkt!=null && Future.data.containsKey(tkt)) { - f = Future.data.get(a.getTicket()); - } - } - } else { - a.delayDelete(noAvg, apprDAO, dryRun, "User Role does not exist"); - UUID tkt = a.getTicket(); - if (tkt !=null && Future.data.containsKey(tkt)) { - f = Future.data.get(a.getTicket()); - } - } - if (f!=null) { - f.delayedDelete(noAvg, futureDAO, dryRun, "Approvals removed"); - } - } - } - } - } - try { - trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); - Future.resetLocalData(); - Approval.resetLocalData(); - } catch (Exception t) { - t.printStackTrace(); - } - - // Run for Expired Futures - trans.info().log("Checking for Expired Approval/Futures"); - tt = trans.start("Delete old Futures", Env.REMOTE); - trans.info().log("### Running Future Execution on ",Future.data.size(), "Items"); - // Execute any Futures waiting - for (Future f : Future.data.values()) { - if (f.memo().contains("Re-Approval") || f.memo().contains("Re-Validate")) { - List la = Approval.byTicket.get(f.id()); - if (la!=null) { - Result ruf = urFutureApproveExec.exec(noAvg,la,f); - if (ruf.isOK()) { - switch(ruf.value) { - case P: - break; - case E: - case D: - case L: - f.delayedDelete(noAvg, futureDAO, dryRun,OP_STATUS.L.desc()); - Approval.delayDelete(noAvg, apprDAO, dryRun, la,OP_STATUS.L.desc()); - break; - } - } - } - } - } - try { - trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); - Future.resetLocalData(); - Approval.resetLocalData(); - } catch (Exception t) { - t.printStackTrace(); - } - - - trans.info().log("### Remove Expired on ",Future.data.size(), "Items, or premature ones"); - // Remove Expired - String expiredBeforeNow = "Expired before " + tooLate; - String expiredAfterFuture = "Expired after " + future; - try { - for (Future f : Future.data.values()) { - if (f.expires().before(tooLate)) { - f.delayedDelete(noAvg,futureDAO,dryRun, expiredBeforeNow); - Approval.delayDelete(noAvg, apprDAO, dryRun, Approval.byTicket.get(f.id()), expiredBeforeNow); - } else if (f.expires().after(future)) { - f.delayedDelete(noAvg,futureDAO,dryRun, expiredAfterFuture); - Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), expiredAfterFuture); - } - } - try { - trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); - Future.resetLocalData(); - Approval.resetLocalData(); - } catch (Exception t) { - t.printStackTrace(); - } - } finally { - tt.done(); - } - - trans.info().log("### Checking Approvals valid (",Approval.byApprover.size(),"Items)"); - // Make sure users of Approvals are still valid - for (List lapp : Approval.byTicket.values()) { - for (Approval app : lapp) { - Future f; - if (app.getTicket()==null) { - f = null; - } else { - f = Future.data.get(app.getTicket()); - if (Future.pendingDelete(f)) { - f=null; - } - } - String msg; - if (f!=null && app.getRole()!=null && Role.byName.get(app.getRole())==null) { - f.delayedDelete(noAvg,futureDAO,dryRun,msg="Role '" + app.getRole() + "' no longer exists"); - Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), msg); - continue; - } - - switch(app.getStatus()) { - case "pending": - if (f==null) { - app.delayDelete(noAvg,apprDAO, isDryRun(), "ticketDeleted"); - continue; - } - switch(app.getType()) { - case "owner": - boolean anOwner=false; - String approle = app.getRole(); - if (approle!=null) { - Role role = Role.byName.get(approle); - if (role==null) { - app.delayDelete(noAvg, apprDAO, dryRun, "Role No Longer Exists"); - continue; - } else { - // Make sure Owner Role exists - String owner = role.ns + ".owner"; - if (Role.byName.containsKey(owner)) { - List lur = UserRole.getByRole().get(owner); - if (lur != null) { - for (UserRole ur : lur) { - if (ur.user().equals(app.getApprover())) { - anOwner = true; - break; - } - } - } - } - } - if (!anOwner) { - app.delayDelete(noAvg, apprDAO, dryRun, "No longer Owner"); - } - - } - break; - case "supervisor": - try { - Identity identity = org.getIdentity(noAvg, app.getUser()); - if (identity==null) { - if (f!=null) { - f.delayedDelete(noAvg,futureDAO,dryRun,msg = app.getUser() + " is no longer associated with " + org.getName()); - Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), msg); - } - } else { - if (!app.getApprover().equals(identity.responsibleTo().fullID())) { - if (f!=null) { - f.delayedDelete(noAvg,futureDAO,dryRun,msg = app.getApprover() + " is no longer a Supervisor of " + app.getUser()); - Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), msg); - } - } - } - } catch (OrganizationException e) { - e.printStackTrace(); - } - break; - } - break; - } - } - } - try { - trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); - Future.resetLocalData(); - Approval.resetLocalData(); - } catch (Exception t) { - t.printStackTrace(); - } - - int count = 0, deleted=0, delayedURDeletes = 0; - - // Run for User Roles - trans.info().log("Checking for Expired User Roles"); - try { - for (UserRole ur : UserRole.getData()) { - if (org.getIdentity(noAvg, ur.user())==null) { // if not part of Organization; - if (isSpecial(ur.user())) { - trans.info().log(ur.user(),"is not part of organization, but may not be deleted"); - } else { - ur.delayDelete(noAvg, "Not Part of Organization", dryRun); - ++deleted; - ++delayedURDeletes; - } - } else { - if (NS.data.get(ur.ns())==null) { - ur.delayDelete(noAvg,"Namespace " + ur.ns() + " does not exist.",dryRun); - ++delayedURDeletes; - ++deleted; - } else if (!Role.byName.containsKey(ur.role())) { - ur.delayDelete(noAvg,"Role " + ur.role() + " does not exist.",dryRun); - ++deleted; - ++delayedURDeletes; - } else if (ur.expires().before(tooLate)) { - if ("owner".equals(ur.rname())) { // don't delete Owners, even if Expired - urPrint.exec(noAvg,ur,"Owner Expired (but not deleted)"); - } else { - // In this case, when UR is expired, not dependent on other lookups, we delete straight out. - urDelete.exec(noAvg, ur,"Expired before " + tooLate); - ++deleted; - } - //trans.logAuditTrail(trans.info()); - } else if (ur.expires().before(future) && ur.expires().after(now)) { - ++count; - // Is there an Approval set already - boolean needNew = true; - if (ur.role()!=null && ur.user()!=null) { - List abm = Approval.byUser.get(ur.user()); - if (abm!=null) { - for (Approval a : abm) { - if (a.getOperation().equals(FUTURE_OP.A.name()) && ur.role().equals(a.getRole())) { - if (Future.data.get(a.getTicket())!=null) { - needNew = false; - break; - } - } - } - } - } - if (needNew) { - urFutureApprove.exec(noAvg, ur,""); - } - } - } - } - } catch (OrganizationException e) { - env.info().log(e,"Exiting ..."); - } finally { - env.info().log("Found",count,"user roles expiring before",future); - env.info().log("deleting",deleted,"user roles expiring before",tooLate); - } - - // Actualize UR Deletes, or send Email - if (UserRole.sizeForDeletion()>0) { - count+=UserRole.sizeForDeletion(); - double onePercent = 0.01; - if (((double)UserRole.sizeForDeletion())/UserRole.getData().size() > onePercent) { - Message msg = new Message(); - try { - msg.line("Found %d of %d UserRoles marked for Deletion in file %s", - delayedURDeletes,UserRole.getData().size(),deletesFile.getCanonicalPath()); - } catch (IOException e) { - msg.line("Found %d of %d UserRoles marked for Deletion.\n", - delayedURDeletes); - } - msg.line("Review the File. If data is ok, Use ExpiringP2 BatchProcess to complete the deletions"); - - email.msg(msg); - email.exec(trans, org, "Email Support"); - } else { - urDeleteF.flush(); - try { - BufferedReader br = new BufferedReader(new FileReader(deletesFile)); - try { - ExpiringP2.deleteURs(noAvg, br, urDelete, null /* don't touch Cache here*/); - } finally { - br.close(); - } - } catch (IOException io) { - noAvg.error().log(io); - } - } - } - if (count>0) { - String str = String.format("%d UserRoles modified or deleted", count); - cacheTouch.exec(trans, "user_role", str); - } - - // Run for Creds - trans.info().log("Checking for Expired Credentials"); - System.out.flush(); - count = 0; - try { - CredDAO.Data crd = new CredDAO.Data(); - Date last = null; - for ( Cred creds : Cred.data.values()) { - crd.id = creds.id; - for (int type : creds.types()) { - crd.type = type; - for ( Instance inst : creds.instances) { - if (inst.expires.before(tooLate)) { - crd.expires = inst.expires; - crDelete.exec(noAvg, crd,"Expired before " + tooLate); - } else if (last==null || inst.expires.after(last)) { - last = inst.expires; - } - } - if (last!=null) { - if (last.before(future)) { - crd.expires = last; - crPrint.exec(noAvg, crd,""); - ++count; - } - } - } - } - } finally { - String str = String.format("Found %d current creds expiring before %s", count, Chrono.dateOnlyStamp(future)); - if (count>0) { - cacheTouch.exec(trans, "cred", str); - } - } - - } - - @Override - protected void _close(AuthzTrans trans) { - aspr.info("End " + this.getClass().getSimpleName() + " processing" ); - for (Action action : new Action[] {crDelete}) { - if (action instanceof ActionDAO) { - ((ActionDAO)action).close(trans); - } - } - session.close(); - urDeleteF.close(); - urRecoverF.close(); - } - -} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringOrig.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringOrig.java new file mode 100644 index 00000000..36d08b7a --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringOrig.java @@ -0,0 +1,506 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ + +package org.onap.aaf.auth.update; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.List; +import java.util.UUID; + +import org.onap.aaf.auth.Batch; +import org.onap.aaf.auth.BatchPrincipal; +import org.onap.aaf.auth.actions.Action; +import org.onap.aaf.auth.actions.ActionDAO; +import org.onap.aaf.auth.actions.CacheTouch; +import org.onap.aaf.auth.actions.CredDelete; +import org.onap.aaf.auth.actions.CredPrint; +import org.onap.aaf.auth.actions.Email; +import org.onap.aaf.auth.actions.Message; +import org.onap.aaf.auth.actions.URDelete; +import org.onap.aaf.auth.actions.URFutureApprove; +import org.onap.aaf.auth.actions.URFutureApproveExec; +import org.onap.aaf.auth.actions.URPrint; +import org.onap.aaf.auth.dao.cass.ApprovalDAO; +import org.onap.aaf.auth.dao.cass.CredDAO; +import org.onap.aaf.auth.dao.cass.FutureDAO; +import org.onap.aaf.auth.dao.hl.Function.FUTURE_OP; +import org.onap.aaf.auth.dao.hl.Function.OP_STATUS; +import org.onap.aaf.auth.env.AuthzTrans; +import org.onap.aaf.auth.helpers.Approval; +import org.onap.aaf.auth.helpers.Cred; +import org.onap.aaf.auth.helpers.Future; +import org.onap.aaf.auth.helpers.NS; +import org.onap.aaf.auth.helpers.Role; +import org.onap.aaf.auth.helpers.UserRole; +import org.onap.aaf.auth.helpers.Cred.Instance; +import org.onap.aaf.auth.layer.Result; +import org.onap.aaf.auth.org.OrganizationException; +import org.onap.aaf.auth.org.Organization.Identity; +import org.onap.aaf.misc.env.APIException; +import org.onap.aaf.misc.env.Env; +import org.onap.aaf.misc.env.TimeTaken; +import org.onap.aaf.misc.env.util.Chrono; + +public class ExpiringOrig extends Batch { + private CredPrint crPrint; + private URFutureApprove urFutureApprove; + private URFutureApproveExec urFutureApproveExec; + private CredDelete crDelete; + private URDelete urDelete; + private final CacheTouch cacheTouch; + private final AuthzTrans noAvg; + private final ApprovalDAO apprDAO; + private final FutureDAO futureDAO; + private final PrintStream urDeleteF,urRecoverF; + private final URPrint urPrint; + private Email email; + private File deletesFile; + + public ExpiringOrig(AuthzTrans trans) throws APIException, IOException, OrganizationException { + super(trans.env()); + trans.info().log("Starting Connection Process"); + + noAvg = env.newTransNoAvg(); + noAvg.setUser(new BatchPrincipal("batch:Expiring")); + + TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); + try { + crPrint = new CredPrint("Expired:"); + + TimeTaken tt = trans.start("Connect to Cluster with DAOs", Env.REMOTE); + try { + urFutureApprove = new URFutureApprove(trans, cluster,isDryRun()); + checkOrganizationAcccess(trans, urFutureApprove.question()); + urFutureApproveExec = new URFutureApproveExec(trans, urFutureApprove); + urPrint = new URPrint("User Roles:"); + crDelete = new CredDelete(trans, urFutureApprove); + urDelete = new URDelete(trans,urFutureApprove); + cacheTouch = new CacheTouch(trans, urFutureApprove); + + // Reusing... don't destroy + apprDAO = urFutureApprove.question().approvalDAO; + futureDAO = urFutureApprove.question().futureDAO; + + TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE); + try { + session = urFutureApprove.getSession(trans); + } finally { + tt2.done(); + } + } finally { + tt.done(); + } + + File data_dir = new File(env.getProperty("aaf_data_dir")); + if (!data_dir.exists() || !data_dir.canWrite() || !data_dir.canRead()) { + throw new IOException("Cannot read/write to Data Directory "+ data_dir.getCanonicalPath() + ": EXITING!!!"); + } + UserRole.setDeleteStream( + urDeleteF = new PrintStream(new FileOutputStream(deletesFile = new File(data_dir,"UserRoleDeletes.dat"),false))); + UserRole.setRecoverStream( + urRecoverF = new PrintStream(new FileOutputStream(new File(data_dir,"UserRoleRecover.dat"),false))); + UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor()); + + Cred.load(trans, session); + NS.load(trans, session,NS.v2_0_11); + Future.load(trans,session,Future.withConstruct); + Approval.load(trans,session,Approval.v2_0_17); + Role.load(trans, session); + + email = new Email(); + email.subject("AAF Expiring Process Alert (ENV: %s)",batchEnv); + email.preamble("Expiring Process Alert for %s",batchEnv); + email.signature("Sincerely,\nAAF Expiring Batch Process\n"); + String address = env.getProperty("ALERT_TO_ADDRESS"); + if (address==null) { + throw new APIException("ALERT_TO_ADDRESS property is required"); + } + email.addTo(address); + + } catch (OrganizationException e) { + throw new APIException("Error getting valid Organization",e); + } finally { + tt0.done(); + } + } + + @Override + protected void run(AuthzTrans trans) { + // Setup Date boundaries + + final GregorianCalendar gc = new GregorianCalendar(); + final Date now = gc.getTime(); + + gc.add(GregorianCalendar.MONTH, 1); + Date future = gc.getTime(); +// Date earliest = null; + + // reset + gc.setTime(now); + gc.add(GregorianCalendar.DAY_OF_MONTH, -7); // save Expired Roles for 7 days. + Date tooLate = gc.getTime(); + + TimeTaken tt; + + // Clean out Approvals UserRoles are fixed up. + String memo; + for (List la : Approval.byUser.values()) { + for (Approval a : la ) { + memo = a.getMemo(); + if (memo!=null && (memo.contains("Re-Approval") || memo.contains("Re-Validate"))) { + String role = a.getRole(); + if (role!=null) { + UserRole ur = UserRole.get(a.getUser(), a.getRole()); + Future f=null; + if (ur!=null) { + if (ur.expires().after(future)) { // no need for Approval anymore + a.delayDelete(noAvg, apprDAO, dryRun, "User Role already Extended"); + UUID tkt = a.getTicket(); + if (tkt!=null && Future.data.containsKey(tkt)) { + f = Future.data.get(a.getTicket()); + } + } + } else { + a.delayDelete(noAvg, apprDAO, dryRun, "User Role does not exist"); + UUID tkt = a.getTicket(); + if (tkt !=null && Future.data.containsKey(tkt)) { + f = Future.data.get(a.getTicket()); + } + } + if (f!=null) { + f.delayedDelete(noAvg, futureDAO, dryRun, "Approvals removed"); + } + } + } + } + } + try { + trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); + Future.resetLocalData(); + Approval.resetLocalData(); + } catch (Exception t) { + t.printStackTrace(); + } + + // Run for Expired Futures + trans.info().log("Checking for Expired Approval/Futures"); + tt = trans.start("Delete old Futures", Env.REMOTE); + trans.info().log("### Running Future Execution on ",Future.data.size(), "Items"); + // Execute any Futures waiting + for (Future f : Future.data.values()) { + if (f.memo().contains("Re-Approval") || f.memo().contains("Re-Validate")) { + List la = Approval.byTicket.get(f.id()); + if (la!=null) { + Result ruf = urFutureApproveExec.exec(noAvg,la,f); + if (ruf.isOK()) { + switch(ruf.value) { + case P: + break; + case E: + case D: + case L: + f.delayedDelete(noAvg, futureDAO, dryRun,OP_STATUS.L.desc()); + Approval.delayDelete(noAvg, apprDAO, dryRun, la,OP_STATUS.L.desc()); + break; + } + } + } + } + } + try { + trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); + Future.resetLocalData(); + Approval.resetLocalData(); + } catch (Exception t) { + t.printStackTrace(); + } + + + trans.info().log("### Remove Expired on ",Future.data.size(), "Items, or premature ones"); + // Remove Expired + String expiredBeforeNow = "Expired before " + tooLate; + String expiredAfterFuture = "Expired after " + future; + try { + for (Future f : Future.data.values()) { + if (f.expires().before(tooLate)) { + f.delayedDelete(noAvg,futureDAO,dryRun, expiredBeforeNow); + Approval.delayDelete(noAvg, apprDAO, dryRun, Approval.byTicket.get(f.id()), expiredBeforeNow); + } else if (f.expires().after(future)) { + f.delayedDelete(noAvg,futureDAO,dryRun, expiredAfterFuture); + Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), expiredAfterFuture); + } + } + try { + trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); + Future.resetLocalData(); + Approval.resetLocalData(); + } catch (Exception t) { + t.printStackTrace(); + } + } finally { + tt.done(); + } + + trans.info().log("### Checking Approvals valid (",Approval.byApprover.size(),"Items)"); + // Make sure users of Approvals are still valid + for (List lapp : Approval.byTicket.values()) { + for (Approval app : lapp) { + Future f; + if (app.getTicket()==null) { + f = null; + } else { + f = Future.data.get(app.getTicket()); + if (Future.pendingDelete(f)) { + f=null; + } + } + String msg; + if (f!=null && app.getRole()!=null && Role.byName.get(app.getRole())==null) { + f.delayedDelete(noAvg,futureDAO,dryRun,msg="Role '" + app.getRole() + "' no longer exists"); + Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), msg); + continue; + } + + switch(app.getStatus()) { + case "pending": + if (f==null) { + app.delayDelete(noAvg,apprDAO, isDryRun(), "ticketDeleted"); + continue; + } + switch(app.getType()) { + case "owner": + boolean anOwner=false; + String approle = app.getRole(); + if (approle!=null) { + Role role = Role.byName.get(approle); + if (role==null) { + app.delayDelete(noAvg, apprDAO, dryRun, "Role No Longer Exists"); + continue; + } else { + // Make sure Owner Role exists + String owner = role.ns + ".owner"; + if (Role.byName.containsKey(owner)) { + List lur = UserRole.getByRole().get(owner); + if (lur != null) { + for (UserRole ur : lur) { + if (ur.user().equals(app.getApprover())) { + anOwner = true; + break; + } + } + } + } + } + if (!anOwner) { + app.delayDelete(noAvg, apprDAO, dryRun, "No longer Owner"); + } + + } + break; + case "supervisor": + try { + Identity identity = org.getIdentity(noAvg, app.getUser()); + if (identity==null) { + if (f!=null) { + f.delayedDelete(noAvg,futureDAO,dryRun,msg = app.getUser() + " is no longer associated with " + org.getName()); + Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), msg); + } + } else { + if (!app.getApprover().equals(identity.responsibleTo().fullID())) { + if (f!=null) { + f.delayedDelete(noAvg,futureDAO,dryRun,msg = app.getApprover() + " is no longer a Supervisor of " + app.getUser()); + Approval.delayDelete(noAvg,apprDAO,dryRun, Approval.byTicket.get(f.id()), msg); + } + } + } + } catch (OrganizationException e) { + e.printStackTrace(); + } + break; + } + break; + } + } + } + try { + trans.info().log("### Removed",Future.sizeForDeletion(),"Future and",Approval.sizeForDeletion(),"Approvals"); + Future.resetLocalData(); + Approval.resetLocalData(); + } catch (Exception t) { + t.printStackTrace(); + } + + int count = 0, deleted=0, delayedURDeletes = 0; + + // Run for User Roles + trans.info().log("Checking for Expired User Roles"); + try { + for (UserRole ur : UserRole.getData()) { + if (org.getIdentity(noAvg, ur.user())==null) { // if not part of Organization; + if (isSpecial(ur.user())) { + trans.info().log(ur.user(),"is not part of organization, but may not be deleted"); + } else { + ur.delayDelete(noAvg, "Not Part of Organization", dryRun); + ++deleted; + ++delayedURDeletes; + } + } else { + if (NS.data.get(ur.ns())==null) { + ur.delayDelete(noAvg,"Namespace " + ur.ns() + " does not exist.",dryRun); + ++delayedURDeletes; + ++deleted; + } else if (!Role.byName.containsKey(ur.role())) { + ur.delayDelete(noAvg,"Role " + ur.role() + " does not exist.",dryRun); + ++deleted; + ++delayedURDeletes; + } else if (ur.expires().before(tooLate)) { + if ("owner".equals(ur.rname())) { // don't delete Owners, even if Expired + urPrint.exec(noAvg,ur,"Owner Expired (but not deleted)"); + } else { + // In this case, when UR is expired, not dependent on other lookups, we delete straight out. + urDelete.exec(noAvg, ur,"Expired before " + tooLate); + ++deleted; + } + //trans.logAuditTrail(trans.info()); + } else if (ur.expires().before(future) && ur.expires().after(now)) { + ++count; + // Is there an Approval set already + boolean needNew = true; + if (ur.role()!=null && ur.user()!=null) { + List abm = Approval.byUser.get(ur.user()); + if (abm!=null) { + for (Approval a : abm) { + if (a.getOperation().equals(FUTURE_OP.A.name()) && ur.role().equals(a.getRole())) { + if (Future.data.get(a.getTicket())!=null) { + needNew = false; + break; + } + } + } + } + } + if (needNew) { + urFutureApprove.exec(noAvg, ur,""); + } + } + } + } + } catch (OrganizationException e) { + env.info().log(e,"Exiting ..."); + } finally { + env.info().log("Found",count,"user roles expiring before",future); + env.info().log("deleting",deleted,"user roles expiring before",tooLate); + } + + // Actualize UR Deletes, or send Email + if (UserRole.sizeForDeletion()>0) { + count+=UserRole.sizeForDeletion(); + double onePercent = 0.01; + if (((double)UserRole.sizeForDeletion())/UserRole.getData().size() > onePercent) { + Message msg = new Message(); + try { + msg.line("Found %d of %d UserRoles marked for Deletion in file %s", + delayedURDeletes,UserRole.getData().size(),deletesFile.getCanonicalPath()); + } catch (IOException e) { + msg.line("Found %d of %d UserRoles marked for Deletion.\n", + delayedURDeletes); + } + msg.line("Review the File. If data is ok, Use ExpiringP2 BatchProcess to complete the deletions"); + + email.msg(msg); + email.exec(trans, org, "Email Support"); + } else { + urDeleteF.flush(); + try { + BufferedReader br = new BufferedReader(new FileReader(deletesFile)); + try { + ExpiringP2.deleteURs(noAvg, br, urDelete, null /* don't touch Cache here*/); + } finally { + br.close(); + } + } catch (IOException io) { + noAvg.error().log(io); + } + } + } + if (count>0) { + String str = String.format("%d UserRoles modified or deleted", count); + cacheTouch.exec(trans, "user_role", str); + } + + // Run for Creds + trans.info().log("Checking for Expired Credentials"); + System.out.flush(); + count = 0; + try { + CredDAO.Data crd = new CredDAO.Data(); + Date last = null; + for ( Cred creds : Cred.data.values()) { + crd.id = creds.id; + for (int type : creds.types()) { + crd.type = type; + for ( Instance inst : creds.instances) { + if (inst.expires.before(tooLate)) { + crd.expires = inst.expires; + crDelete.exec(noAvg, crd,"Expired before " + tooLate); + } else if (last==null || inst.expires.after(last)) { + last = inst.expires; + } + } + if (last!=null) { + if (last.before(future)) { + crd.expires = last; + crPrint.exec(noAvg, crd,""); + ++count; + } + } + } + } + } finally { + String str = String.format("Found %d current creds expiring before %s", count, Chrono.dateOnlyStamp(future)); + if (count>0) { + cacheTouch.exec(trans, "cred", str); + } + } + + } + + @Override + protected void _close(AuthzTrans trans) { + trans.info().log("End",this.getClass().getSimpleName(),"processing" ); + for (Action action : new Action[] {crDelete}) { + if (action instanceof ActionDAO) { + ((ActionDAO)action).close(trans); + } + } + session.close(); + urDeleteF.close(); + urRecoverF.close(); + } + +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java index cab612eb..dae62573 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/ExpiringP2.java @@ -141,7 +141,7 @@ public class ExpiringP2 extends Batch { @Override protected void _close(AuthzTrans trans) { - aspr.info("End " + this.getClass().getSimpleName() + " processing" ); + trans.info().log("End",this.getClass().getSimpleName(),"processing" ); for (Action action : new Action[] {urDelete,cacheTouch}) { if (action instanceof ActionDAO) { ((ActionDAO)action).close(trans); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java index b4631f66..cbc3ef38 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/NotifyCredExpiring.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.TreeMap; import org.onap.aaf.auth.Batch; import org.onap.aaf.auth.BatchPrincipal; @@ -43,21 +44,19 @@ import org.onap.aaf.auth.dao.hl.Question; import org.onap.aaf.auth.env.AuthzTrans; import org.onap.aaf.auth.helpers.Cred; import org.onap.aaf.auth.helpers.Notification; -import org.onap.aaf.auth.helpers.UserRole; import org.onap.aaf.auth.helpers.Notification.TYPE; +import org.onap.aaf.auth.helpers.UserRole; import org.onap.aaf.auth.layer.Result; import org.onap.aaf.auth.org.EmailWarnings; import org.onap.aaf.auth.org.Organization; +import org.onap.aaf.auth.org.Organization.Identity; import org.onap.aaf.auth.org.OrganizationException; import org.onap.aaf.auth.org.OrganizationFactory; -import org.onap.aaf.auth.org.Organization.Identity; import org.onap.aaf.misc.env.APIException; import org.onap.aaf.misc.env.Env; import org.onap.aaf.misc.env.TimeTaken; import org.onap.aaf.misc.env.util.Chrono; -import java.util.TreeMap; - public class NotifyCredExpiring extends Batch { @@ -109,7 +108,7 @@ public class NotifyCredExpiring extends Batch { Cred.load(trans, session,CredDAO.BASIC_AUTH, CredDAO.BASIC_AUTH_SHA256); Notification.load(trans, session, Notification.v2_0_18); - UserRole.load(trans, session, UserRole.v2_0_11); + UserRole.load(trans, session, UserRole.v2_0_11, new UserRole.DataLoadVisitor()); ps = new PrintStream(new FileOutputStream(logDir() + "/email"+Chrono.dateOnlyStamp()+".log",true)); ps.printf("### Approval Notify %s for %s%s\n",Chrono.dateTime(),batchEnv,dryRun?", DryRun":""); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java new file mode 100644 index 00000000..22961f61 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Remove.java @@ -0,0 +1,123 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ + +package org.onap.aaf.auth.update; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.onap.aaf.auth.Batch; +import org.onap.aaf.auth.BatchPrincipal; +import org.onap.aaf.auth.actions.CacheTouch; +import org.onap.aaf.auth.env.AuthzTrans; +import org.onap.aaf.auth.helpers.CQLBatch; +import org.onap.aaf.auth.helpers.Cred; +import org.onap.aaf.auth.helpers.UserRole; +import org.onap.aaf.auth.helpers.X509; +import org.onap.aaf.auth.org.OrganizationException; +import org.onap.aaf.cadi.CadiException; +import org.onap.aaf.cadi.client.Holder; +import org.onap.aaf.cadi.util.CSV; +import org.onap.aaf.misc.env.APIException; +import org.onap.aaf.misc.env.Env; +import org.onap.aaf.misc.env.TimeTaken; +import org.onap.aaf.misc.env.util.Chrono; + +public class Remove extends Batch { + private final AuthzTrans noAvg; + private CacheTouch cacheTouch; + private CQLBatch cqlBatch; + + public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException { + super(trans.env()); + trans.info().log("Starting Connection Process"); + + noAvg = env.newTransNoAvg(); + noAvg.setUser(new BatchPrincipal("batch:ExpiringP2")); + + TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); + try { + cacheTouch = new CacheTouch(trans, cluster, dryRun); + TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE); + try { + session = cacheTouch.getSession(trans); + } finally { + tt2.done(); + } + cqlBatch = new CQLBatch(session); + + + } finally { + tt0.done(); + } + } + + @Override + protected void run(AuthzTrans trans) { + final int maxBatch = 50; + + // Create Intermediate Output + File logDir = new File(logDir()); + File expired = new File(logDir,"Expired"+Chrono.dateOnlyStamp()+".csv"); + CSV expiredCSV = new CSV(expired); + try { + final StringBuilder sb = cqlBatch.begin(); + final Holder hi = new Holder(0); + expiredCSV.visit(new CSV.Visitor() { + @Override + public void visit(List row) throws IOException, CadiException { + int i = hi.get(); + if(i>=maxBatch) { + cqlBatch.execute(dryRun); + hi.set(0); + cqlBatch.begin(); + i=0; + } + switch(row.get(0)) { + case "ur": + hi.set(++i); + UserRole.row(sb,row); + break; + case "cred": + hi.set(++i); + Cred.row(sb,row); + break; + case "x509": + hi.set(++i); + X509.row(sb,row); + break; + } + } + }); + cqlBatch.execute(dryRun); + } catch (IOException | CadiException e) { + e.printStackTrace(); + } + } + + @Override + protected void _close(AuthzTrans trans) { + session.close(); + cacheTouch.close(trans); + } + +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java new file mode 100644 index 00000000..f308e226 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/update/Upload.java @@ -0,0 +1,302 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + */ + +package org.onap.aaf.auth.update; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.onap.aaf.auth.Batch; +import org.onap.aaf.auth.env.AuthzTrans; +import org.onap.aaf.auth.org.OrganizationException; +import org.onap.aaf.misc.env.APIException; +import org.onap.aaf.misc.env.Env; +import org.onap.aaf.misc.env.TimeTaken; + +import com.datastax.driver.core.ResultSet; + +public class Upload extends Batch { + public Upload(AuthzTrans trans) throws APIException, IOException, OrganizationException { + super(trans.env()); + trans.info().log("Starting Connection Process"); + + TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); + try { + TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE); + try { + session = cluster.connect(); + } finally { + tt.done(); + } + + } finally { + tt0.done(); + } + } + + private static final int BATCH_LENGTH = 100; + + int count,batchCnt; + + @Override + protected void run(AuthzTrans trans) { + String line; + StringBuilder sb = new StringBuilder(); + StringBuilder query = new StringBuilder(); + List array = new ArrayList(); + for(String feed : args()) { + File file = new File(feed + ".dat"); + TimeTaken tt = trans.start(file.getAbsolutePath(), Env.SUB); + System.out.println("#### Running " + feed + ".dat Feed ####"); + try { + + if(file.exists()) { + count=batchCnt=0; + try { + BufferedReader br = new BufferedReader(new FileReader(file)); + try { + while((line=br.readLine())!=null) { + if(query.length()==0) { + query.append("BEGIN BATCH\n"); + } + // Split into fields, first turning Escaped values into something we can convert back from + char c=0; + boolean inQuote = false; + int fldcnt = 0; + + for(int i=0;i array) { + String rv; + switch(feed) { + case "approval": + rv = String.format(APPR_INS_FMT,array.toArray()); + break; + case "artifact": + rv = String.format(ARTI_INS_FMT,array.toArray()); + break; + case "cred": + rv = String.format(CRED_INS_FMT,array.toArray()); + break; + case "ns": + rv = String.format(NS_INS_FMT,array.toArray()); + break; + case "role": + rv = String.format(ROLE_INS_FMT,array.toArray()); + break; + case "perm": + rv = String.format(PERM_INS_FMT,array.toArray()); + break; + case "x509": + rv = String.format(X509_INS_FMT,array.toArray()); + break; + default: + rv = ""; + } + array.clear(); + return rv; + } + + private void addField(String feed, int fldcnt, List array, StringBuilder sb) { + Boolean[] ba; + switch(feed) { + case "approval": + ba = APPR_QUOTES; + break; + case "artifact": + ba = ARTI_QUOTES; + break; + case "cred": + ba = CRED_QUOTES; + break; + case "ns": + ba = NS_QUOTES; + break; + case "role": + ba = ROLE_QUOTES; + break; + case "perm": + ba = PERM_QUOTES; + break; + case "x509": + ba = X509_QUOTES; + break; + default: + ba = null; + } + if(ba!=null) { + if(sb.toString().length()==0) { + array.add("null"); + } else { + if(ba[fldcnt]) { + String s = null; + if(sb.indexOf("'")>=0) { + s = sb.toString().replace("'","''"); + } + if(sb.indexOf("\\n")>=0) { + if(s==null) { + s = sb.toString().replace("\\n","\n"); + } else { + s = s.replace("\\n","\n"); + } + } + if(sb.indexOf("\\t")>=0) { + if(s==null) { + s = sb.toString().replace("\\t","\t"); + } else { + s = s.replace("\\t","\t"); + } + } + if(s==null) { + array.add("'" + sb + '\''); + } else { + array.add("'" + s + '\''); + } + } else { + array.add(sb.toString()); + } + } + sb.setLength(0); + } + } + + private void applyBatch(StringBuilder query) { + query.append("APPLY BATCH;"); + ResultSet rv = session.execute(query.toString()); + if(rv.wasApplied()) { + System.out.print('.'); + if((++batchCnt % 60)==0) { + System.out.println(); + } + } else { + System.out.print("Data NOT APPLIED"); + } + query.setLength(0); + } + + + @Override + protected void _close(AuthzTrans trans) { + session.close(); + } + +} + -- cgit 1.2.3-korg