diff options
| author |   Instrumental <jonathan.gathman@att.com> | 2019-02-15 19:40:04 -0600 |
|---|---|---|
| committer | Instrumental <jonathan.gathman@att.com> | 2019-02-15 20:04:29 -0600 |
| commit | 628b7105ce4d9818aac69a082e515f9275fd46fd (patch) | |
| tree | 3f1e073e6973efd06f950a70a12499976027eae4 /auth/auth-batch/src/main/java | |
| parent | 4b6435c97dc1e05ee6a1e06190e3c431f9d1a8fe (diff) | |
Batch work and client
Issue-ID: AAF-740
Change-Id: I16da4f2a87ec5d19590f0af642b91f9e2e02b246
Signed-off-by: Instrumental <jonathan.gathman@att.com>
Diffstat (limited to 'auth/auth-batch/src/main/java')
26 files changed, 1638 insertions, 1128 deletions
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/ApprovalSet.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/ApprovalSet.java index eeeef15f..b7176c26 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/ApprovalSet.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/ApprovalSet.java @@ -74,13 +74,13 @@ public class ApprovalSet { public Result<Void> write(AuthzTrans trans) { StringBuilder errs = null; - Result<FutureDAO.Data> rf = dataview.write(trans, fdd); + Result<FutureDAO.Data> rf = dataview.insert(trans, fdd); if(rf.notOK()) { errs = new StringBuilder(); errs.append(rf.errorString()); } else { for(ApprovalDAO.Data add : ladd) { - Result<ApprovalDAO.Data> af = dataview.write(trans, add); + Result<ApprovalDAO.Data> af = dataview.insert(trans, add); if(af.notOK()) { if(errs==null) { errs = new StringBuilder(); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/DataView.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/DataView.java index 73e79832..3b90f3a9 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/DataView.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/DataView.java @@ -60,9 +60,14 @@ public interface DataView { public Result<List<UserRoleDAO.Data>> ursByRole(final AuthzTrans trans, final String role); public Result<List<UserRoleDAO.Data>> ursByUser(final AuthzTrans trans, final String user); - // Writes - public Result<ApprovalDAO.Data> write(final AuthzTrans trans, final ApprovalDAO.Data add); - public Result<FutureDAO.Data> write(final AuthzTrans trans, final FutureDAO.Data add); + // Inserts + public Result<ApprovalDAO.Data> insert(final AuthzTrans trans, final ApprovalDAO.Data add); + public Result<FutureDAO.Data> insert(final AuthzTrans trans, final FutureDAO.Data add); // Deletes + public Result<ApprovalDAO.Data> delete(final AuthzTrans trans, final ApprovalDAO.Data add); + public Result<FutureDAO.Data> delete(final AuthzTrans trans, final FutureDAO.Data add); + + // Clear any buffers + public void flush(); } diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/Pending.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/Pending.java new file mode 100644 index 00000000..2e7997b4 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/Pending.java @@ -0,0 +1,108 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ +package org.onap.aaf.auth.batch.approvalsets; + +import java.text.ParseException; +import java.util.Date; +import java.util.List; + +import org.onap.aaf.cadi.util.CSV.Writer; +import org.onap.aaf.misc.env.util.Chrono; + +public class Pending { + public static final String REMIND = "remind"; + + int qty; + boolean hasNew; + Date earliest; + + /** + * Use this Constructor when there is no Last Notified Date + */ + public Pending() { + qty = 1; + hasNew = true; + earliest = null; + } + + /** + * Use this constructor to indicate when last Notified + * @param last_notified + */ + public Pending(Date last_notified) { + qty = 1; + hasNew = last_notified==null; + earliest = last_notified; + } + + /** + * Create from CSV Row + * @param row + * @throws ParseException + */ + public Pending(List<String> row) throws ParseException { + hasNew = Boolean.parseBoolean(row.get(2)); + String d = row.get(3); + if(d==null || d.isEmpty()) { + earliest = null; + } else { + earliest = Chrono.dateOnlyFmt.parse(d); + } + qty = Integer.parseInt(row.get(4)); + } + + /** + * Write CSV Row + * @param approveCW + * @param key + */ + public void row(Writer approveCW, String key) { + approveCW.row(REMIND,key,hasNew,Chrono.dateOnlyStamp(earliest),qty); + } + + public void inc() { + ++qty; + } + + public void inc(Pending value) { + qty+=value.qty; + } + + public void earliest(Date lastnotified) { + if(lastnotified==null) { + hasNew=true; + } else if (earliest==null || lastnotified.before(earliest)) { + earliest = lastnotified; + } + } + + public int qty() { + return qty; + } + + public Date earliest() { + return earliest; + } + + public boolean newApprovals() { + return hasNew; + } +}
\ No newline at end of file diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/Ticket.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/Ticket.java new file mode 100644 index 00000000..1259c87e --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/Ticket.java @@ -0,0 +1,37 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ +package org.onap.aaf.auth.batch.approvalsets; + +import java.util.HashSet; +import java.util.Set; + +import org.onap.aaf.auth.batch.helpers.Approval; +import org.onap.aaf.auth.batch.helpers.Future; + +public class Ticket { + public final Future f; + public final Set<Approval> approvals; + + public Ticket(Future future) { + this.f = future; + approvals = new HashSet<>(); + } +}
\ No newline at end of file diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/URApprovalSet.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/URApprovalSet.java index e1c75bf3..858690ac 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/URApprovalSet.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/approvalsets/URApprovalSet.java @@ -39,23 +39,24 @@ import org.onap.aaf.cadi.CadiException; import org.onap.aaf.misc.env.util.Chrono; public class URApprovalSet extends ApprovalSet { - public static final String EXTEND_STRING = "Extend access of User [%s] to Role [%s] - Expires %s"; + private boolean ownerSuperApprove; + public URApprovalSet(final AuthzTrans trans, final GregorianCalendar start, final DataView dv, final Loader<UserRoleDAO.Data> lurdd) throws IOException, CadiException { super(start, "user_role", dv); Organization org = trans.org(); UserRoleDAO.Data urdd = lurdd.load(); setConstruct(urdd.bytify()); - setMemo(String.format(EXTEND_STRING,urdd.user,urdd.role,Chrono.dateOnlyStamp(urdd.expires))); + setMemo(getMemo(urdd)); setExpires(org.expiration(null, Organization.Expiration.UserInRole)); Result<RoleDAO.Data> r = dv.roleByName(trans, urdd.role); if(r.notOKorIsEmpty()) { - throw new CadiException(String.format("Role '%s' does not exist: %s", urdd.role, r.details)); + throw new CadiException(r.errorString()); } Result<NsDAO.Data> n = dv.ns(trans, urdd.ns); if(n.notOKorIsEmpty()) { - throw new CadiException(String.format("Namespace '%s' does not exist: %s", urdd.ns,r.details)); + throw new CadiException(n.errorString()); } UserRoleDAO.Data found = null; Result<List<Data>> lur = dv.ursByRole(trans, urdd.role); @@ -68,7 +69,7 @@ public class URApprovalSet extends ApprovalSet { } } if(found==null) { - throw new CadiException(String.format("User '%s' in Role '%s' does not exist: %s", urdd.user,urdd.role,r.details)); + throw new CadiException(String.format("User '%s' in Role '%s' does not exist", urdd.user,urdd.role)); } // Primarily, Owners are responsible, unless it's owned by self @@ -87,7 +88,7 @@ public class URApprovalSet extends ApprovalSet { } } - if(isOwner) { + if(isOwner && ownerSuperApprove) { try { List<Identity> apprs = org.getApprovers(trans, urdd.user); if(apprs!=null) { @@ -108,18 +109,38 @@ public class URApprovalSet extends ApprovalSet { } } } + + public void ownerSuperApprove() { + ownerSuperApprove = true; + } - private ApprovalDAO.Data newApproval(Data urdd) throws CadiException { + private ApprovalDAO.Data newApproval(UserRoleDAO.Data urdd) throws CadiException { ApprovalDAO.Data add = new ApprovalDAO.Data(); add.id = Chrono.dateToUUID(System.currentTimeMillis()); add.ticket = fdd.id; add.user = urdd.user; add.operation = FUTURE_OP.A.name(); add.status = ApprovalDAO.PENDING; - add.memo = String.format("Re-Validate as Owner for AAF Namespace '%s' - expiring %s', ", - urdd.ns, - Chrono.dateOnlyStamp(urdd.expires)); + add.memo = getMemo(urdd); return add; } + private String getMemo(Data urdd) { + switch(urdd.rname) { + case "owner": + return String.format("Revalidate as Owner of AAF Namespace [%s] - Expires %s", + urdd.ns, + Chrono.dateOnlyStamp(urdd.expires)); + case "admin": + return String.format("Revalidate as Admin of AAF Namespace [%s] - Expires %s", + urdd.ns, + Chrono.dateOnlyStamp(urdd.expires)); + default: + return String.format("Extend access of User [%s] to Role [%s] - Expires %s", + urdd.user, + urdd.role, + Chrono.dateOnlyStamp(urdd.expires)); + } + } + } diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Approval.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Approval.java index acaf0d58..2cc6907b 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Approval.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Approval.java @@ -24,6 +24,7 @@ package org.onap.aaf.auth.batch.helpers; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.TreeMap; import java.util.UUID; @@ -50,6 +51,7 @@ public class Approval implements CacheChange.Data { public static TreeMap<String,List<Approval>> byApprover = new TreeMap<>(); public static TreeMap<String,List<Approval>> byUser = new TreeMap<>(); public static TreeMap<UUID,List<Approval>> byTicket = new TreeMap<>(); + public static List<Approval> list = new LinkedList<>(); private final static CacheChange<Approval> cache = new CacheChange<>(); public final ApprovalDAO.Data add; @@ -127,6 +129,7 @@ public class Approval implements CacheChange.Data { cw.row("approval",app.add.id,app.add.ticket,app.add.user,app.role,app.add.memo); } + public static void load(Trans trans, Session session, Creator<Approval> creator ) { trans.info().log( "query: " + creator.select() ); TimeTaken tt = trans.start("Load Notify", Env.REMOTE); @@ -147,6 +150,8 @@ public class Approval implements CacheChange.Data { ++count; try { Approval app = creator.create(row); + list.add(app); + String person = app.getApprover(); if (person!=null) { ln = byApprover.get(person); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/BatchDataView.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/BatchDataView.java index e934bda6..37def6d6 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/BatchDataView.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/BatchDataView.java @@ -24,8 +24,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.onap.aaf.auth.batch.actions.ApprovalAdd; -import org.onap.aaf.auth.batch.actions.FutureAdd; import org.onap.aaf.auth.batch.approvalsets.DataView; import org.onap.aaf.auth.dao.cass.ApprovalDAO; import org.onap.aaf.auth.dao.cass.FutureDAO; @@ -35,29 +33,29 @@ import org.onap.aaf.auth.dao.cass.UserRoleDAO; import org.onap.aaf.auth.dao.cass.UserRoleDAO.Data; import org.onap.aaf.auth.env.AuthzTrans; import org.onap.aaf.auth.layer.Result; +import org.onap.aaf.cadi.Hash; import org.onap.aaf.misc.env.APIException; import org.onap.aaf.misc.env.TimeTaken; import org.onap.aaf.misc.env.Trans; +import org.onap.aaf.misc.env.util.Chrono; -import com.datastax.driver.core.Cluster; import com.datastax.driver.core.Session; public class BatchDataView implements DataView { - private FutureAdd futureAdd; - private ApprovalAdd approvalAdd; + private static final String QUOTE_PAREN_SEMI = "');\n"; + private static final String QUOTE_COMMA = "',"; + private static final String QUOTE_COMMA_QUOTE = "','"; + private static final String COMMA_QUOTE = ",'"; + private final CQLBatchLoop cqlBatch; + private final Session session; - public BatchDataView(final AuthzTrans trans, final Cluster cluster, final boolean dryRun ) throws APIException, IOException { - futureAdd = new FutureAdd(trans, cluster, dryRun); - approvalAdd = new ApprovalAdd(trans, futureAdd); + public BatchDataView(final AuthzTrans trans, final Session session, final boolean dryRun ) throws APIException, IOException { + this.session = session; + cqlBatch = new CQLBatchLoop(new CQLBatch(trans.info(),session),50,dryRun); } public Session getSession(AuthzTrans trans) throws APIException, IOException { - TimeTaken tt = trans.start("Get Session", Trans.SUB); - try { - return futureAdd.getSession(trans); - } finally { - tt.done(); - } + return session; } public Result<NsDAO.Data> ns(AuthzTrans trans, String id) { @@ -114,13 +112,73 @@ public class BatchDataView implements DataView { } @Override - public Result<FutureDAO.Data> write(AuthzTrans trans, FutureDAO.Data fdd) { - return futureAdd.exec(trans, fdd, null); + public Result<FutureDAO.Data> delete(AuthzTrans trans, FutureDAO.Data fdd) { + cqlBatch.preLoop(); + StringBuilder sb = cqlBatch.inc(); + sb.append("DELETE from authz.future WHERE id = "); + sb.append(fdd.id.toString()); + return Result.ok(fdd); + } + + @Override + public Result<ApprovalDAO.Data> delete(AuthzTrans trans, ApprovalDAO.Data add) { + cqlBatch.preLoop(); + StringBuilder sb = cqlBatch.inc(); + sb.append("DELETE from authz.approval WHERE id = "); + sb.append(add.id.toString()); + return Result.ok(add); } + @Override - public Result<ApprovalDAO.Data> write(AuthzTrans trans, ApprovalDAO.Data add) { - return approvalAdd.exec(trans, add, null); + public Result<ApprovalDAO.Data> insert(AuthzTrans trans, ApprovalDAO.Data add) { + cqlBatch.preLoop(); + StringBuilder sb = cqlBatch.inc(); + sb.append("INSERT INTO authz.approval (id,approver,last_notified,memo,operation,status,ticket,type,user) VALUES ("); + sb.append(add.id.toString()); + sb.append(COMMA_QUOTE); + sb.append(add.approver); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(Chrono.utcStamp(add.last_notified)); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(add.memo.replace("'", "''")); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(add.operation); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(add.status); + sb.append(QUOTE_COMMA); + sb.append(add.ticket.toString()); + sb.append(COMMA_QUOTE); + sb.append(add.type); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(add.user); + sb.append(QUOTE_PAREN_SEMI); + return Result.ok(add); } + @Override + public Result<FutureDAO.Data> insert(AuthzTrans trans, FutureDAO.Data fdd) { + cqlBatch.preLoop(); + StringBuilder sb = cqlBatch.inc(); + sb.append("INSERT INTO authz.future (id,construct,expires,memo,start,target) VALUES ("); + sb.append(fdd.id.toString()); + sb.append(','); + fdd.construct.hasArray(); + sb.append(Hash.toHex(fdd.construct.array())); + sb.append(COMMA_QUOTE); + sb.append(Chrono.utcStamp(fdd.expires)); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(fdd.memo.replace("'", "''")); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(Chrono.utcStamp(fdd.expires)); + sb.append(QUOTE_COMMA_QUOTE); + sb.append(fdd.target); + sb.append(QUOTE_PAREN_SEMI); + return Result.ok(fdd); + } + + @Override + public void flush() { + cqlBatch.flush(); + } } diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatchLoop.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatchLoop.java new file mode 100644 index 00000000..ca264d14 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/CQLBatchLoop.java @@ -0,0 +1,69 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + */ + +package org.onap.aaf.auth.batch.helpers; + +public class CQLBatchLoop { + + private final CQLBatch cqlBatch; + private final int maxBatch; + private final StringBuilder sb; + private final boolean dryRun; + private int i; + + public CQLBatchLoop(CQLBatch cb, int max, boolean dryRun) { + cqlBatch = cb; + i=0; + maxBatch = max; + sb = cqlBatch.begin(); + this.dryRun = dryRun; + } + + /** + * Put at the first part of your Loop Logic... It checks if you have enough lines to + * push a batch. + */ + public void preLoop() { + if(i<0) { + cqlBatch.begin(); + } else if(i>=maxBatch) { + cqlBatch.execute(dryRun); + cqlBatch.begin(); + i=0; + } + } + + /** + * Assume this is another line in the Batch + * @return + */ + public StringBuilder inc() { + ++i; + return sb; + } + + /** + * Close up when done. However, can go back to "preLoop" safely. + */ + public void flush() { + cqlBatch.execute(dryRun); + i=-1; + } +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java index 8a5dfea1..8db2b47a 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java @@ -63,25 +63,46 @@ public class Cred { public final Date expires,written; public final Integer other; public final String tag; - public final Integer attn; - public final String notes; + public List<Note> notes; - public Instance(int type, Date expires, Integer other, long written, String tag, int attn, String notes) { + public Instance(int type, Date expires, Integer other, long written, String tag) { this.type = type; this.expires = expires; this.other = other; this.written = new Date(written); this.tag = tag; - this.attn = attn; - this.notes = notes; + } + + /** + * Usually returns Null... + * @return + */ + public List<Note> notes() { + return notes; + } + + public void addNote(int level, String note) { + if(notes==null) { + notes=new ArrayList<>(); + } + notes.add(new Note(level,note)); } public String toString() { - return expires.toString() + ": " + type + ' ' + notes; + return expires.toString() + ": " + type + ' ' + tag; } } + public static class Note { + public final int level; + public final String note; + + public Note(int level, String note) { + this.level = level; + this.note = note; + } + } public Date last(final int ... types) { Date last = null; for (Instance i : instances) { @@ -114,12 +135,12 @@ public class Cred { } public static void load(Trans trans, Session session, int ... types ) { - load(trans, session,"select id, type, expires, other, writetime(cred), tag, attn, notes from authz.cred;",types); + load(trans, session,"select id, type, expires, other, writetime(cred), tag from authz.cred;",types); } public static void loadOneNS(Trans trans, Session session, String ns,int ... types ) { - load(trans, session,"select id, type, expires, other, writetime(cred), tag, attn, notes from authz.cred WHERE ns='" + ns + "';"); + load(trans, session,"select id, type, expires, other, writetime(cred), tag from authz.cred WHERE ns='" + ns + "';"); } private static void load(Trans trans, Session session, String query, int ...types) { @@ -157,7 +178,7 @@ public class Cred { } } add(row.getString(0), row.getInt(1),row.getTimestamp(2),row.getInt(3),row.getLong(4), - row.getString(5),row.getInt(6),row.getString(7)); + row.getString(5)); } } finally { tt.done(); @@ -173,16 +194,14 @@ public class Cred { final Date timestamp, final int other, final long written, - final String tag, - final int attn, - final String notes + final String tag ) { Cred cred = data.get(id); if (cred==null) { cred = new Cred(id); data.put(id, cred); } - cred.instances.add(new Instance(type, timestamp, other, written/1000,tag,attn,notes)); + cred.instances.add(new Instance(type, timestamp, other, written/1000,tag)); List<Cred> lscd = byNS.get(cred.ns); if (lscd==null) { @@ -289,7 +308,12 @@ public class Cred { public void row(final CSV.Writer csvw, final Instance inst) { csvw.row("cred",id,ns,Integer.toString(inst.type),Chrono.dateOnlyStamp(inst.expires), - inst.expires.getTime(),inst.tag,inst.attn,inst.notes); + inst.expires.getTime(),inst.tag); + } + + public void row(final CSV.Writer csvw, final Instance inst, final String reason) { + csvw.row("cred",id,ns,Integer.toString(inst.type),Chrono.dateOnlyStamp(inst.expires), + inst.expires.getTime(),inst.tag,reason); } @@ -341,7 +365,12 @@ public class Cred { public static String histMemo(String fmt, String orgName, List<String> row) { - return String.format(fmt, row.get(1),orgName,row.get(4)); + String reason; + if(row.size()>5) { // Reason included + reason = row.get(5); + } else { + reason = String.format(fmt, row.get(1),orgName,row.get(4)); + } + return reason; } - }
\ No newline at end of file diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java index b06cbce9..73bff6e6 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/ExpireRange.java @@ -34,10 +34,14 @@ import java.util.Set; import org.onap.aaf.cadi.Access; public class ExpireRange { + public static final String ONE_MONTH = "OneMonth"; + public static final String TWO_MONTH = "TwoMonth"; + public static final String TWO_WEEK = "TwoWeek"; + public static final String ONE_WEEK = "OneWeek"; private static final String AAF_BATCH_RANGE = "aaf_batch_range."; public Map<String,List<Range>> ranges; public final Date now; - public String rangeOneMonth = "OneMonth"; + private Range delRange; public ExpireRange(final Access access) { @@ -55,14 +59,14 @@ public class ExpireRange { lcred.add(delRange); lx509.add(delRange); - lcred.add(new Range("CredOneWeek",3,1,0,0,GregorianCalendar.WEEK_OF_MONTH,1)); - lcred.add(new Range("CredTwoWeek",2,1,GregorianCalendar.WEEK_OF_MONTH,1,GregorianCalendar.WEEK_OF_MONTH,2)); - lcred.add(new Range(rangeOneMonth,1,7,GregorianCalendar.WEEK_OF_MONTH,2,GregorianCalendar.MONTH,1)); - lcred.add(new Range("TwoMonth",1,0,GregorianCalendar.MONTH,1,GregorianCalendar.MONTH,2)); + lcred.add(new Range(ONE_WEEK,3,1,0,0,GregorianCalendar.WEEK_OF_MONTH,1)); + lcred.add(new Range(TWO_WEEK,2,1,GregorianCalendar.WEEK_OF_MONTH,1,GregorianCalendar.WEEK_OF_MONTH,2)); + lcred.add(new Range(ONE_MONTH,1,7,GregorianCalendar.WEEK_OF_MONTH,2,GregorianCalendar.MONTH,1)); + lcred.add(new Range(TWO_MONTH,1,0,GregorianCalendar.MONTH,1,GregorianCalendar.MONTH,2)); - lur.add(new Range(rangeOneMonth,1,7,GregorianCalendar.WEEK_OF_MONTH,2,GregorianCalendar.MONTH,1)); + lur.add(new Range(ONE_MONTH,1,7,GregorianCalendar.WEEK_OF_MONTH,2,GregorianCalendar.MONTH,1)); - lx509.add(new Range(rangeOneMonth,1,7,GregorianCalendar.WEEK_OF_MONTH,2,GregorianCalendar.MONTH,1)); + lx509.add(new Range(ONE_MONTH,1,7,GregorianCalendar.WEEK_OF_MONTH,2,GregorianCalendar.MONTH,1)); } } diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/MiscID.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/MiscID.java deleted file mode 100644 index 4d46c20b..00000000 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/MiscID.java +++ /dev/null @@ -1,194 +0,0 @@ -/** - * ============LICENSE_START==================================================== - * org.onap.aaf - * =========================================================================== - * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - * =========================================================================== - * Modifications Copyright (C) 2019 IBM. - * =========================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END==================================================== - * - */ - -package org.onap.aaf.auth.batch.helpers; - -import java.util.Map; -import java.util.TreeMap; - -import org.onap.aaf.auth.batch.BatchException; -import org.onap.aaf.misc.env.Env; -import org.onap.aaf.misc.env.TimeTaken; -import org.onap.aaf.misc.env.Trans; - -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SimpleStatement; -import com.datastax.driver.core.Statement; - -public class MiscID { - public static final TreeMap<String,MiscID> data = new TreeMap<>(); - /* - Sample Record - aad890|mj9030|20040902|20120207 - - **** Field Definitions **** - MISCID - AT&T Miscellaneous ID - Non-User ID (Types: Internal Mechanized ID, External Mechanized ID, Datagate ID, Customer ID, Vendor ID, Exchange Mail ID, CLEC ID, Specialized ID, Training ID) - SPONSOR_ATTUID - ATTUID of MiscID Sponsor (Owner) - CREATE_DATE - Date when MiscID was created - LAST_RENEWAL_DATE - Date when MiscID Sponsorship was last renewed - */ - public String id; - public String sponsor; - public String created; - public String renewal; - public static String SELECT_QUERY = "SELECT "; - - private static final String FIELD_STRING = "id,created,sponsor,renewal"; - - /** - * Load a Row of Strings (from CSV file). - * - * Be CAREFUL that the Row lists match the Fields above!!! If this changes, change - * 1) This Object - * 2) DB "suits.cql" - * 3) Alter existing Tables - * @param row - * @throws BatchException - */ - public void set(String[] row ) throws BatchException { - if (row.length<4) { - throw new BatchException("Row of MiscID_XRef is too short"); - } - id = row[0]; - sponsor = row[1]; - created = row[2]; - renewal = row[3]; - } - - public void set(Row row) { - id = row.getString(0); - sponsor = row.getString(1); - created = row.getString(2); - renewal = row.getString(3); - } - - - public static void load(Trans trans, Session session ) { - load(trans, session,SELECT_QUERY + FIELD_STRING + " FROM authz.miscid;",data); - } - - public static void load(Trans trans, Session session, Map<String,MiscID> map ) { - load(trans, session,SELECT_QUERY + FIELD_STRING + " FROM authz.miscid;",map); - } - - public static void loadOne(Trans trans, Session session, String id ) { - load(trans, session,SELECT_QUERY + FIELD_STRING + " FROM authz.miscid WHERE id ='" + id + "';", data); - } - - public static void load(Trans trans, Session session, String query, Map<String,MiscID> map) { - trans.info().log( "query: " + query ); - TimeTaken tt = trans.start("Read MiscID", Env.REMOTE); - - ResultSet results; - try { - Statement stmt = new SimpleStatement( query ); - results = session.execute(stmt); - } finally { - tt.done(); - } - int count = 0; - try { - tt = trans.start("Load Map", Env.SUB); - try { - for ( Row row : results.all()) { - MiscID miscID = new MiscID(); - miscID.set(row); - data.put(miscID.id,miscID); - ++count; - } - } finally { - tt.done(); - } - } finally { - trans.info().log("Found",count,"miscID records"); - } - } - - /* (non-Javadoc) - * @see java.lang.Object#hashCode() - */ - @Override - public int hashCode() { - return id.hashCode(); - } - - /* (non-Javadoc) - * @see java.lang.Object#equals(java.lang.Object) - */ - @Override - public boolean equals(Object obj) { - if (null!=obj && obj instanceof MiscID) { - return id.equals(((MiscID)obj).id); - } - return false; - } - - public StringBuilder insertStmt() { - StringBuilder sb = new StringBuilder("INSERT INTO authz.miscid ("); - sb.append(FIELD_STRING); - sb.append(") VALUES ('"); - sb.append(id); - sb.append("','"); - sb.append(sponsor); - sb.append("','"); - sb.append(created); - sb.append("','"); - sb.append(renewal); - sb.append("')"); - return sb; - } - - public StringBuilder updateStmt(MiscID source) { - StringBuilder sb = null; - if (id.equals(source.id)) { - sb = addField(sb,"sponser",sponsor,source.sponsor); - sb = addField(sb,"created",created,source.created); - sb = addField(sb,"renewal",renewal,source.renewal); - } - if (sb!=null) { - sb.append(" WHERE id='"); - sb.append(id); - sb.append('\''); - } - return sb; - } - - private StringBuilder addField(StringBuilder sb, String name, String a, String b) { - if (!a.equals(b)) { - if (sb==null) { - sb = new StringBuilder("UPDATE authz.miscid SET "); - } else { - sb.append(','); - } - sb.append(name); - sb.append("='"); - sb.append(b); - sb.append('\''); - } - return sb; - } - - -}
\ No newline at end of file diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java index 0b6eb7b1..55dd1e7c 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/UserRole.java @@ -47,7 +47,10 @@ import com.datastax.driver.core.Statement; public class UserRole implements Cloneable, CacheChange.Data { - private static final String SEPARATOR = "\",\""; + public static final String UR = "ur"; + public static final String APPROVE_UR = "ur"; + + private static final String SEPARATOR = "\",\""; // CACHE Calling private static final String LOG_FMT = "%s UserRole - %s: %s-%s (%s, %s) expiring %s"; @@ -308,17 +311,21 @@ public class UserRole implements Cloneable, CacheChange.Data { cache.resetLocalData(); } - public void row(final CSV.Writer csvw) { - csvw.row("ur",user(),ns(),rname(),Chrono.dateOnlyStamp(expires()),expires().getTime()); + public void row(final CSV.Writer csvw, String tag) { + csvw.row(tag,user(),role(),ns(),rname(),Chrono.dateOnlyStamp(expires()),expires().getTime()); + } + + public void row(final CSV.Writer csvw, String tag, String reason) { + csvw.row(tag,user(),role(),ns(),rname(),Chrono.dateOnlyStamp(expires()),expires().getTime(),reason); } public static Data row(List<String> row) { Data data = new Data(); data.user = row.get(1); - data.ns = row.get(2); - data.rname = row.get(3); - data.role = data.ns + '.' + data.rname; - data.expires = new Date(Long.parseLong(row.get(5))); + data.role = row.get(2); + data.ns = row.get(3); + data.rname = row.get(4); + data.expires = new Date(Long.parseLong(row.get(6))); return data; } @@ -327,8 +334,6 @@ public class UserRole implements Cloneable, CacheChange.Data { sb.append(row.get(1)); sb.append("' AND role='"); sb.append(row.get(2)); - sb.append('.'); - sb.append(row.get(3)); sb.append("';\n"); } @@ -339,16 +344,21 @@ public class UserRole implements Cloneable, CacheChange.Data { sb.append(row.get(1)); sb.append("' AND role='"); sb.append(row.get(2)); - sb.append('.'); - sb.append(row.get(3)); sb.append("';\n"); } public static String histMemo(String fmt, List<String> row) { - return String.format(fmt, row.get(1),row.get(2)+'.'+row.get(3), row.get(4)); + String reason; + if(row.size()>7) { // Reason included + reason = String.format("%s removed from %s because %s", + row.get(1),row.get(2),row.get(7)); + } else { + reason = String.format(fmt, row.get(1),row.get(2), row.get(5)); + } + return reason; } public static String histSubject(List<String> row) { - return row.get(1) + '|' + row.get(2)+'.'+row.get(3); + return row.get(1) + '|' + row.get(2); } }
\ No newline at end of file diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java index 3cbf90fa..39f017cb 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/X509.java @@ -112,6 +112,10 @@ public class X509 { cw.row("x509",ca,Hash.toHex(serial.array()),Chrono.dateOnlyStamp(x509Cert.getNotAfter()),x500); } + public void row(CSV.Writer cw, X509Certificate x509Cert,String reason) { + cw.row("x509",ca,Hash.toHex(serial.array()),Chrono.dateOnlyStamp(x509Cert.getNotAfter()),x500,reason); + } + public static void row(StringBuilder sb, List<String> row) { sb.append("DELETE from authz.x509 WHERE ca='"); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Analyze.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Analyze.java new file mode 100644 index 00000000..35020836 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Analyze.java @@ -0,0 +1,530 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * + * Modifications Copyright (C) 2019 IBM. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ + +package org.onap.aaf.auth.batch.reports; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; + +import org.onap.aaf.auth.batch.Batch; +import org.onap.aaf.auth.batch.approvalsets.Pending; +import org.onap.aaf.auth.batch.approvalsets.Ticket; +import org.onap.aaf.auth.batch.helpers.Approval; +import org.onap.aaf.auth.batch.helpers.Cred; +import org.onap.aaf.auth.batch.helpers.Cred.Instance; +import org.onap.aaf.auth.batch.helpers.ExpireRange; +import org.onap.aaf.auth.batch.helpers.ExpireRange.Range; +import org.onap.aaf.auth.batch.helpers.Future; +import org.onap.aaf.auth.batch.helpers.Role; +import org.onap.aaf.auth.batch.helpers.UserRole; +import org.onap.aaf.auth.batch.helpers.X509; +import org.onap.aaf.auth.dao.cass.CredDAO; +import org.onap.aaf.auth.dao.cass.UserRoleDAO; +import org.onap.aaf.auth.env.AuthzTrans; +import org.onap.aaf.auth.org.Organization.Identity; +import org.onap.aaf.auth.org.OrganizationException; +import org.onap.aaf.cadi.configure.Factory; +import org.onap.aaf.cadi.util.CSV; +import org.onap.aaf.misc.env.APIException; +import org.onap.aaf.misc.env.Env; +import org.onap.aaf.misc.env.TimeTaken; +import org.onap.aaf.misc.env.Trans; +import org.onap.aaf.misc.env.util.Chrono; + + +public class Analyze extends Batch { + private static final int unknown=0; + private static final int owner=1; + private static final int supervisor=2; + private static final int total=0; + private static final int pending=1; + private static final int approved=2; + + + private static final String APPROVALS = "Approvals"; + private static final String EXTEND = "Extend"; + private static final String EXPIRED_OWNERS = "ExpiredOwners"; + private static final String CSV = ".csv"; + private static final String INFO = "info"; + private int minOwners; + private Map<String, CSV.Writer> writerList; + private ExpireRange expireRange; + private Date deleteDate; + private CSV.Writer deleteCW; + private CSV.Writer approveCW; + private CSV.Writer extendCW; + + public Analyze(AuthzTrans trans) throws APIException, IOException, OrganizationException { + super(trans.env()); + trans.info().log("Starting Connection Process"); + + TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); + try { + TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE); + try { + session = cluster.connect(); + } finally { + tt.done(); + } + + // Load Cred. We don't follow Visitor, because we have to gather up everything into Identity Anyway + Cred.load(trans, session); + + minOwners=1; + + // Create Intermediate Output + writerList = new HashMap<>(); + + expireRange = new ExpireRange(trans.env().access()); + String sdate = Chrono.dateOnlyStamp(expireRange.now); + for( List<Range> lr : expireRange.ranges.values()) { + for(Range r : lr ) { + if(writerList.get(r.name())==null) { + File file = new File(logDir(),r.name() + sdate +CSV); + CSV csv = new CSV(env.access(),file); + CSV.Writer cw = csv.writer(false); + cw.row(INFO,r.name(),Chrono.dateOnlyStamp(expireRange.now),r.reportingLevel()); + writerList.put(r.name(),cw); + if("Delete".equals(r.name())) { + deleteDate = r.getEnd(); + deleteCW = cw; + } + trans.init().log("Creating File:",file.getAbsolutePath()); + } + } + } + + // Setup New Approvals file + File file = new File(logDir(),APPROVALS + sdate +CSV); + CSV approveCSV = new CSV(env.access(),file); + approveCW = approveCSV.writer(); + approveCW.row(INFO,APPROVALS,Chrono.dateOnlyStamp(expireRange.now),1); + writerList.put(APPROVALS,approveCW); + + // Setup Extend Approvals file + file = new File(logDir(),EXTEND + sdate +CSV); + CSV extendCSV = new CSV(env.access(),file); + extendCW = extendCSV.writer(); + extendCW.row(INFO,EXTEND,Chrono.dateOnlyStamp(expireRange.now),1); + writerList.put(EXTEND,extendCW); + + // Load full data of the following + Approval.load(trans, session, Approval.v2_0_17); + Role.load(trans, session); + } finally { + tt0.done(); + } + } + + @Override + protected void run(AuthzTrans trans) { + AuthzTrans noAvg = trans.env().newTransNoAvg(); + + //////////////////// + final Map<UUID,Ticket> goodTickets = new TreeMap<>(); + TimeTaken tt = trans.start("Analyze Expired Futures",Trans.SUB); + try { + Future.load(noAvg, session, Future.withConstruct, fut -> { + List<Approval> appls = Approval.byTicket.get(fut.id()); + if(fut.expires().before(expireRange.now)) { + deleteCW.comment("Future %s expired", fut.id()); + Future.row(deleteCW,fut); + if(appls!=null) { + for(Approval a : appls) { + Approval.row(deleteCW, a); + } + } + } else if(appls==null) { // Orphaned Future (no Approvals) + deleteCW.comment("Future is Orphaned"); + Future.row(deleteCW,fut); + } else { + goodTickets.put(fut.fdd.id, new Ticket(fut)); + } + }); + } finally { + tt.done(); + } + + tt = trans.start("Connect Approvals with Futures",Trans.SUB); + try { + for(Approval appr : Approval.list) { + Ticket ticket=null; + UUID ticketID = appr.getTicket(); + if(ticketID!=null) { + ticket = goodTickets.get(appr.getTicket()); + } + if(ticket == null) { // Orphaned Approvals, no Futures + deleteCW.comment("Approval is Orphaned"); + Approval.row(deleteCW, appr); + } else { + ticket.approvals.add(appr); // add to found Ticket + } + } + } finally { + tt.done(); + } + + /* Run through all Futures, and see if + * 1) they have been executed (no longer valid) + * 2) The current Approvals indicate they can proceed + */ + Map<String,Pending> pendingApprs = new HashMap<>(); + Map<String,Pending> pendingTemp = new HashMap<>(); + + tt = trans.start("Analyze Good Tickets",Trans.SUB); + try { + for(Ticket ticket : goodTickets.values()) { + pendingTemp.clear(); + switch(ticket.f.target()) { + case "user_role": + int state[][] = new int[3][3]; + int type; + + for(Approval appr : ticket.approvals) { + switch(appr.getType()) { + case "owner": + type=owner; + break; + case "supervisor": + type=supervisor; + break; + default: + type=0; + } + ++state[type][total]; // count per type + switch(appr.getStatus()) { + case "pending": + ++state[type][pending]; + Pending n = pendingTemp.get(appr.getApprover()); + if(n==null) { + pendingTemp.put(appr.getApprover(),new Pending(appr.getLast_notified())); + } else { + n.inc(); + } + break; + case "approved": + ++state[type][approved]; + break; + default: + ++state[type][unknown]; + } + } + + // To Approve: + // Always must have at least 1 owner + if((state[owner][total]>0 && state[owner][approved]>0) && + // If there are no Supervisors, that's ok + (state[supervisor][total]==0 || + // But if there is a Supervisor, they must have approved + (state[supervisor][approved]>0))) { + UserRoleDAO.Data urdd = new UserRoleDAO.Data(); + try { + urdd.reconstitute(ticket.f.fdd.construct); + if(urdd.expires.before(ticket.f.expires())) { + extendCW.row("extend_ur",urdd.user,urdd.role,ticket.f.expires()); + } + } catch (IOException e) { + trans.error().log("Could not reconstitute UserRole"); + } + } else { // Load all the Pending. + for(Entry<String, Pending> es : pendingTemp.entrySet()) { + Pending p = pendingApprs.get(es.getKey()); + if(p==null) { + pendingApprs.put(es.getKey(), es.getValue()); + } else { + p.inc(es.getValue()); + } + } + } + break; + } + } + } finally { + tt.done(); + } + + /** + * Decide to Notify about Approvals, based on activity/last Notified + */ + tt = trans.start("Analyze Approval Reminders", Trans.SUB); + try { + GregorianCalendar gc = new GregorianCalendar(); + gc.add(GregorianCalendar.DAY_OF_WEEK, 5); + Date remind = gc.getTime(); + + for(Entry<String, Pending> es : pendingApprs.entrySet()) { + Pending p = es.getValue(); + if(p.earliest() == null || p.earliest().after(remind)) { + p.row(approveCW,es.getKey()); + } + } + } finally { + tt.done(); + } + + // clear out Approval Intermediates + goodTickets.clear(); + pendingTemp = null; + pendingApprs = null; + + /** + Run through User Roles. + Owners are treated specially in next section. + Regular roles are checked against Date Ranges. If match Date Range, write out to appropriate file. + */ + try { + tt = trans.start("Analyze UserRoles, storing Owners",Trans.SUB); + Set<String> specialCommented = new HashSet<>(); + Map<String, Set<UserRole>> owners = new TreeMap<String, Set<UserRole>>(); + try { + UserRole.load(noAvg, session, UserRole.v2_0_11, ur -> { + Identity identity; + try { + identity = trans.org().getIdentity(noAvg,ur.user()); + if(identity==null) { + // Candidate for Delete, but not Users if Special + String id = ur.user(); + for(String s : specialDomains) { + if(id.endsWith(s)) { + if(!specialCommented.contains(id)) { + deleteCW.comment("ID %s is part of special Domain %s (UR Org Check)", id,s); + specialCommented.add(id); + } + return; + } + } + if(specialNames.contains(id)) { + if(!specialCommented.contains(id)) { + deleteCW.comment("ID %s is a special ID (UR Org Check)", id); + specialCommented.add(id); + } + return; + } + ur.row(deleteCW, UserRole.UR,"Not in Organization"); + return; + } else if(Role.byName.get(ur.role())==null) { + ur.row(deleteCW, UserRole.UR,String.format("Role %s does not exist", ur.role())); + return; + } + // Cannot just delete owners, unless there is at least one left. Process later + if ("owner".equals(ur.rname())) { + Set<UserRole> urs = owners.get(ur.role()); + if (urs == null) { + urs = new HashSet<UserRole>(); + owners.put(ur.role(), urs); + } + urs.add(ur); + } else { + Range r = writeAnalysis(noAvg,ur); + if(r!=null) { + Approval existing = findApproval(ur); + if(existing==null) { + ur.row(approveCW,UserRole.APPROVE_UR); + } + } + } + } catch (OrganizationException e) { + noAvg.error().log(e); + } + }); + } finally { + tt.done(); + } + + /** + Now Process Owners, one owner Role at a time, ensuring one is left, + preferably a good one. If so, process the others as normal. + + Otherwise, write to ExpiredOwners Report + */ + tt = trans.start("Analyze Owners Separately",Trans.SUB); + try { + if (!owners.values().isEmpty()) { + File file = new File(logDir(), EXPIRED_OWNERS + Chrono.dateOnlyStamp(expireRange.now) + CSV); + final CSV ownerCSV = new CSV(env.access(),file); + CSV.Writer expOwner = ownerCSV.writer(); + expOwner.row(INFO,EXPIRED_OWNERS,Chrono.dateOnlyStamp(expireRange.now),2); + + try { + for (Set<UserRole> sur : owners.values()) { + int goodOwners = 0; + for (UserRole ur : sur) { + if (ur.expires().after(expireRange.now)) { + ++goodOwners; + } + } + + for (UserRole ur : sur) { + if (goodOwners >= minOwners) { + Range r = writeAnalysis(noAvg, ur); + if(r!=null) { + Approval existing = findApproval(ur); + if(existing==null) { + ur.row(approveCW,UserRole.APPROVE_UR); + } + } + } else { + expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires())); + Approval existing = findApproval(ur); + if(existing==null) { + ur.row(approveCW,UserRole.APPROVE_UR); + } + } + } + } + } finally { + if(expOwner!=null) { + expOwner.close(); + } + } + } + } finally { + tt.done(); + } + + /** + * Check for Expired Credentials + * + * + */ + tt = trans.start("Analyze Expired Credentials",Trans.SUB); + try { + for (Cred cred : Cred.data.values()) { + List<Instance> linst = cred.instances; + if(linst!=null) { + Instance lastBath = null; + for(Instance inst : linst) { + // if(inst.attn>0) { + // writeAnalysis(trans, cred, inst); + // // Special Behavior: only eval the LAST Instance + // } else + // All Creds go through Life Cycle + if(deleteDate!=null && inst.expires.before(deleteDate)) { + writeAnalysis(noAvg, cred, inst); // will go to Delete + // Basic Auth has Pre-EOL notifications IF there is no Newer Credential + } else if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) { + if(lastBath==null || lastBath.expires.before(inst.expires)) { + lastBath = inst; + } + } + } + if(lastBath!=null) { + writeAnalysis(noAvg, cred, lastBath); + } + } + } + } finally { + tt.done(); + } + + //////////////////// + tt = trans.start("Analyze Expired X509s",Trans.SUB); + try { + X509.load(noAvg, session, x509 -> { + try { + for(Certificate cert : Factory.toX509Certificate(x509.x509)) { + writeAnalysis(noAvg, x509, (X509Certificate)cert); + } + } catch (CertificateException | IOException e) { + noAvg.error().log(e, "Error Decrypting X509"); + } + + }); + } finally { + tt.done(); + } + } catch (FileNotFoundException e) { + noAvg.info().log(e); + } + } + + private Approval findApproval(UserRole ur) { + Approval existing = null; + List<Approval> apprs = Approval.byUser.get(ur.user()); + if(apprs!=null) { + for(Approval appr : apprs) { + if(ur.role().equals(appr.getRole()) && + appr.getMemo().contains(Chrono.dateOnlyStamp(ur.expires()))) { + existing = appr; + } + } + } + return existing; + } + + private Range writeAnalysis(AuthzTrans trans, UserRole ur) { + Range r = expireRange.getRange("ur", ur.expires()); + if(r!=null) { + CSV.Writer cw = writerList.get(r.name()); + if(cw!=null) { + ur.row(cw,UserRole.UR); + } + } + return r; + } + + private void writeAnalysis(AuthzTrans trans, Cred cred, Instance inst) { + if(cred!=null && inst!=null) { + Range r = expireRange.getRange("cred", inst.expires); + if(r!=null) { + CSV.Writer cw = writerList.get(r.name()); + if(cw!=null) { + cred.row(cw,inst); + } + } + } + } + + private void writeAnalysis(AuthzTrans trans, X509 x509, X509Certificate x509Cert) throws IOException { + Range r = expireRange.getRange("x509", x509Cert.getNotAfter()); + if(r!=null) { + CSV.Writer cw = writerList.get(r.name()); + if(cw!=null) { + x509.row(cw,x509Cert); + } + } + } + + @Override + protected void _close(AuthzTrans trans) { + session.close(); + for(CSV.Writer cw : writerList.values()) { + cw.close(); + } + } + +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java deleted file mode 100644 index 979bcd50..00000000 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Expiring.java +++ /dev/null @@ -1,337 +0,0 @@ -/** - * ============LICENSE_START==================================================== - * org.onap.aaf - * =========================================================================== - * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - * - * Modifications Copyright (C) 2019 IBM. - * =========================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END==================================================== - * - */ - -package org.onap.aaf.auth.batch.reports; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.cert.Certificate; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; - -import org.onap.aaf.auth.batch.Batch; -import org.onap.aaf.auth.batch.helpers.Approval; -import org.onap.aaf.auth.batch.helpers.Cred; -import org.onap.aaf.auth.batch.helpers.Cred.Instance; -import org.onap.aaf.auth.batch.helpers.ExpireRange; -import org.onap.aaf.auth.batch.helpers.ExpireRange.Range; -import org.onap.aaf.auth.batch.helpers.Future; -import org.onap.aaf.auth.batch.helpers.UserRole; -import org.onap.aaf.auth.batch.helpers.X509; -import org.onap.aaf.auth.dao.cass.CredDAO; -import org.onap.aaf.auth.env.AuthzTrans; -import org.onap.aaf.auth.org.OrganizationException; -import org.onap.aaf.cadi.configure.Factory; -import org.onap.aaf.cadi.util.CSV; -import org.onap.aaf.misc.env.APIException; -import org.onap.aaf.misc.env.Env; -import org.onap.aaf.misc.env.TimeTaken; -import org.onap.aaf.misc.env.util.Chrono; - - -public class Expiring extends Batch { - - private static final String CSV = ".csv"; - private static final String INFO = "info"; - private static final String EXPIRED_OWNERS = "ExpiredOwners"; - private int minOwners; - private Map<String, CSV.Writer> writerList; - private ExpireRange expireRange; - private Date deleteDate; - private CSV.Writer deleteCW; - - public Expiring(AuthzTrans trans) throws APIException, IOException, OrganizationException { - super(trans.env()); - trans.info().log("Starting Connection Process"); - - TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); - try { - TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE); - try { - session = cluster.connect(); - } finally { - tt.done(); - } - - // Load Cred. We don't follow Visitor, because we have to gather up everything into Identity Anyway - Cred.load(trans, session); - - minOwners=1; - - // Create Intermediate Output - writerList = new HashMap<>(); - - expireRange = new ExpireRange(trans.env().access()); - String sdate = Chrono.dateOnlyStamp(expireRange.now); - for( List<Range> lr : expireRange.ranges.values()) { - for(Range r : lr ) { - if(writerList.get(r.name())==null) { - File file = new File(logDir(),r.name() + sdate +CSV); - CSV csv = new CSV(env.access(),file); - CSV.Writer cw = csv.writer(false); - cw.row(INFO,r.name(),Chrono.dateOnlyStamp(expireRange.now),r.reportingLevel()); - writerList.put(r.name(),cw); - if("Delete".equals(r.name())) { - deleteDate = r.getEnd(); - deleteCW = cw; - } - trans.init().log("Creating File:",file.getAbsolutePath()); - } - } - } - Approval.load(trans, session, Approval.v2_0_17); - } finally { - tt0.done(); - } - } - - @Override - protected void run(AuthzTrans trans) { - - //////////////////// - trans.info().log("Checking for Expired Futures"); - Future.load(trans, session, Future.v2_0_17, fut -> { - if(fut.expires().before(expireRange.now)) { - Future.row(deleteCW,fut); - List<Approval> appls = Approval.byTicket.get(fut.id()); - if(appls!=null) { - for(Approval a : appls) { - Approval.row(deleteCW, a); - } - } - } - }); - - try { - File file = new File(logDir(), EXPIRED_OWNERS + Chrono.dateOnlyStamp(expireRange.now) + CSV); - final CSV ownerCSV = new CSV(env.access(),file); - - Map<String, Set<UserRole>> owners = new TreeMap<String, Set<UserRole>>(); - trans.info().log("Process UserRoles"); - - /** - Run through User Roles. - Owners are treated specially in next section. - Regular roles are checked against Date Ranges. If match Date Range, write out to appropriate file. - */ - UserRole.load(trans, session, UserRole.v2_0_11, ur -> { - // Cannot just delete owners, unless there is at least one left. Process later - if ("owner".equals(ur.rname())) { - Set<UserRole> urs = owners.get(ur.role()); - if (urs == null) { - urs = new HashSet<UserRole>(); - owners.put(ur.role(), urs); - } - urs.add(ur); - } else { - writeAnalysis(trans,ur); - } - }); - - /** - Now Process Owners, one owner Role at a time, ensuring one is left, - preferably a good one. If so, process the others as normal. - - Otherwise, write to ExpiredOwners Report - */ - if (!owners.values().isEmpty()) { - // Lazy Create file - CSV.Writer expOwner = null; - try { - for (Set<UserRole> sur : owners.values()) { - int goodOwners = 0; - for (UserRole ur : sur) { - if (ur.expires().after(expireRange.now)) { - ++goodOwners; - } - } - - for (UserRole ur : sur) { - if (goodOwners >= minOwners) { - writeAnalysis(trans, ur); - } else { - if (expOwner == null) { - expOwner = ownerCSV.writer(); - expOwner.row(INFO,EXPIRED_OWNERS,Chrono.dateOnlyStamp(expireRange.now),2); - } - expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires())); - } - } - } - } finally { - if(expOwner!=null) { - expOwner.close(); - } - } - } - - /** - * Check for Expired Credentials - * - * - */ - trans.info().log("Checking for Expired Credentials"); - for (Cred cred : Cred.data.values()) { - List<Instance> linst = cred.instances; - if(linst!=null) { - Instance lastBath = null; - for(Instance inst : linst) { - // Special Behavior: only eval the LAST Instance - if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) { - if(deleteDate!=null && inst.expires.before(deleteDate)) { - writeAnalysis(trans, cred, inst); // will go to Delete - } else if(lastBath==null || lastBath.expires.before(inst.expires)) { - lastBath = inst; - } - } else { - writeAnalysis(trans, cred, inst); - } - } - if(lastBath!=null) { - writeAnalysis(trans, cred, lastBath); - } - } - } - - //////////////////// - trans.info().log("Checking for Expired X509s"); - X509.load(trans, session, x509 -> { - try { - for(Certificate cert : Factory.toX509Certificate(x509.x509)) { - writeAnalysis(trans, x509, (X509Certificate)cert); - } - } catch (CertificateException | IOException e) { - trans.error().log(e, "Error Decrypting X509"); - } - - }); - - } catch (FileNotFoundException e) { - trans.info().log(e); - } - - //////////////////// - trans.info().log("Checking for Orphaned Approvals"); - Approval.load(trans, session, Approval.v2_0_17, appr -> { - UUID ticket = appr.add.ticket; - if(ticket==null) { - Approval.row(deleteCW,appr); - } - }); - - - } - - - private void writeAnalysis(AuthzTrans trans, UserRole ur) { - Range r = expireRange.getRange("ur", ur.expires()); - if(r!=null) { - CSV.Writer cw = writerList.get(r.name()); - if(cw!=null) { - ur.row(cw); - } - } - } - - private void writeAnalysis(AuthzTrans trans, Cred cred, Instance inst) { - if(cred!=null && inst!=null) { - Range r = expireRange.getRange("cred", inst.expires); - if(r!=null) { - CSV.Writer cw = writerList.get(r.name()); - if(cw!=null) { - cred.row(cw,inst); - } - } - } - } - - private void writeAnalysis(AuthzTrans trans, X509 x509, X509Certificate x509Cert) throws IOException { - Range r = expireRange.getRange("x509", x509Cert.getNotAfter()); - if(r!=null) { - CSV.Writer cw = writerList.get(r.name()); - if(cw!=null) { - x509.row(cw,x509Cert); - } - } - } - - /* - private String[] contacts(final AuthzTrans trans, final String ns, final int levels) { - List<UserRole> owners = UserRole.getByRole().get(ns+".owner"); - List<UserRole> current = new ArrayList<>(); - for(UserRole ur : owners) { - if(expireRange.now.before(ur.expires())) { - current.add(ur); - } - } - if(current.isEmpty()) { - trans.warn().log(ns,"has no current owners"); - current = owners; - } - - List<String> email = new ArrayList<>(); - for(UserRole ur : current) { - Identity id; - int i=0; - boolean go = true; - try { - id = org.getIdentity(trans, ur.user()); - do { - if(id!=null) { - email.add(id.email()); - if(i<levels) { - id = id.responsibleTo(); - } else { - go = false; - } - } else { - go = false; - } - } while(go); - } catch (OrganizationException e) { - trans.error().log(e); - } - } - - return email.toArray(new String[email.size()]); - } -*/ - - @Override - protected void _close(AuthzTrans trans) { - session.close(); - for(CSV.Writer cw : writerList.values()) { - cw.close(); - } - } - -} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/NotInOrg.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/NotInOrg.java index f47fae43..9cd0baee 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/NotInOrg.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/NotInOrg.java @@ -32,7 +32,6 @@ import org.onap.aaf.auth.batch.Batch; import org.onap.aaf.auth.batch.helpers.Cred; import org.onap.aaf.auth.batch.helpers.Cred.Instance; import org.onap.aaf.auth.batch.helpers.UserRole; -import org.onap.aaf.auth.batch.helpers.Visitor; import org.onap.aaf.auth.env.AuthzTrans; import org.onap.aaf.auth.org.Organization; import org.onap.aaf.auth.org.Organization.Identity; @@ -108,7 +107,7 @@ public class NotInOrg extends Batch { UserRole.load(trans, session, UserRole.v2_0_11, ur -> { try { if(!check(transNoAvg, checked, ur.user())) { - ur.row(whichWriter(transNoAvg,ur.user())); + ur.row(whichWriter(transNoAvg,ur.user()),UserRole.UR); } } catch (OrganizationException e) { trans.error().log(e, "Error Decrypting X509"); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Notify.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Notify.java index 7fd26747..189857c9 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Notify.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Notify.java @@ -19,220 +19,230 @@ * */package org.onap.aaf.auth.batch.reports; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.onap.aaf.auth.batch.Batch; -import org.onap.aaf.auth.batch.reports.bodies.NotifyBody; -import org.onap.aaf.auth.env.AuthzTrans; -import org.onap.aaf.auth.org.Mailer; -import org.onap.aaf.auth.org.Organization.Identity; -import org.onap.aaf.auth.org.OrganizationException; -import org.onap.aaf.cadi.Access; -import org.onap.aaf.cadi.CadiException; -import org.onap.aaf.cadi.client.Holder; -import org.onap.aaf.cadi.util.CSV; -import org.onap.aaf.misc.env.APIException; -import org.onap.aaf.misc.env.util.Chrono; - -public class Notify extends Batch { - private static final String HTML_CSS = "HTML_CSS"; - private final Mailer mailer; - private final String header; - private final String footer; - private List<File> notifyFile; - public final String guiURL; - private int maxEmails; - private int indent; - - public Notify(AuthzTrans trans) throws APIException, IOException, OrganizationException { - super(trans.env()); - String mailerCls = env.getProperty("MAILER"); - String mailFrom = env.getProperty("MAIL_FROM"); - String header_html = env.getProperty("HEADER_HTML"); - String footer_html = env.getProperty("FOOTER_HTML"); - String maxEmails = env.getProperty("MAX_EMAIL"); - guiURL = env.getProperty("GUI_URL"); - this.maxEmails = maxEmails==null?1:Integer.parseInt(maxEmails); - if(mailerCls==null || mailFrom==null || guiURL==null || header_html==null || footer_html==null) { - throw new APIException("Notify requires MAILER, MAILER_FROM, GUI_URL, HEADER_HTML and FOOTER_HTML properties"); - } - try { - Class<?> mailc = Class.forName(mailerCls); - Constructor<?> mailcst = mailc.getConstructor(Access.class); - mailer = (Mailer)mailcst.newInstance(env.access()); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new APIException("Unable to construct " + mailerCls,e); - } - - String line; - StringBuilder sb = new StringBuilder(); - BufferedReader br = new BufferedReader(new FileReader(header_html)); - try { - while((line=br.readLine())!=null) { - sb.append(line); - sb.append('\n'); - } - String html_css = env.getProperty(HTML_CSS); - int hc = sb.indexOf(HTML_CSS); - if(hc!=0 && html_css!=null) { - header = sb.replace(hc,hc+HTML_CSS.length(), html_css).toString(); - } else { - header = sb.toString(); - } - } finally { - br.close(); - } - - - - // Establish index from header - int lastTag = header.lastIndexOf('<'); - if(lastTag>0) { - int prevCR = header.lastIndexOf('\n',lastTag); - if(prevCR>0) { - indent = lastTag-prevCR; - } else { - indent = 6; //arbitrary - } - } - - - sb.setLength(0); - br = new BufferedReader(new FileReader(footer_html)); - try { - while((line=br.readLine())!=null) { - sb.append(line); - sb.append('\n'); - } - footer = sb.toString(); - } finally { - br.close(); - } - - // Class Load possible data - NotifyBody.load(env.access()); - - // Create Intermediate Output - File logDir = logDir(); - notifyFile = new ArrayList<>(); - if(args().length>0) { - for(int i=0;i<args().length;++i) { - notifyFile.add(new File(logDir, args()[i])); - } - } else { - String fmt = "%s"+Chrono.dateOnlyStamp()+".csv"; - File file; - for(NotifyBody nb : NotifyBody.getAll()) { - file = new File(logDir,String.format(fmt, nb.name())); - if(file.exists()) { - trans.info().printf("Processing %s",file.getCanonicalPath()); - notifyFile.add(file); - } else { - trans.info().printf("No Files found for %s",nb.name()); - } - } - } - } - - @Override - protected void run(AuthzTrans trans) { - List<String> toList = new ArrayList<>(); - List<String> ccList = new ArrayList<>(); - AuthzTrans noAvg = trans.env().newTransNoAvg(); - String subject = "Test Notify"; - boolean urgent = false; - - - - final Notify notify = this; - final Holder<List<String>> info = new Holder<>(null); - final Set<String> errorSet = new HashSet<>(); - - try { - EMAILTYPE: - for(File f : notifyFile) { - CSV csv = new CSV(env.access(),f); - try { - csv.visit(new CSV.Visitor() { - @Override - public void visit(List<String> row) throws IOException, CadiException { - if("info".equals(row.get(0))) { - info.set(row); - } - if(info.get()==null) { - throw new CadiException("First line of Feed MUST contain 'info' record"); - } - String key = row.get(0)+'|'+info.get().get(1); - NotifyBody body = NotifyBody.get(key); - if(body==null) { - errorSet.add("No NotifyBody defined for " + key); - } else { - body.store(row); - } - } - }); - } catch (IOException | CadiException e) { - e.printStackTrace(); - } - - // now create Notification - for(NotifyBody nb : NotifyBody.getAll()) { - for(String id : nb.users()) { - toList.clear(); - ccList.clear(); - try { - Identity identity = trans.org().getIdentity(noAvg, id); - if(!identity.isPerson()) { - identity = identity.responsibleTo(); - } - for(int i=1;i<nb.escalation();++i) { - if(identity != null) { - if(i==1) { - toList.add(identity.email()); - } else { - identity=identity.responsibleTo(); - ccList.add(identity.email()); - } - } - } - - StringBuilder content = new StringBuilder(); - content.append(String.format(header,version,Identity.mixedCase(identity.firstName()))); - - nb.body(noAvg, content, indent, notify, id); - content.append(footer); - - if(!mailer.sendEmail(noAvg, dryRun, toList, ccList, subject,content.toString(), urgent)) { - trans.error().log("Mailer failed to send Mail"); - } - if(maxEmails>0 && mailer.count()>=maxEmails) { - break EMAILTYPE; - } - } catch (OrganizationException e) { - trans.error().log(e); - } - } - } - - } - } finally { - for(String s : errorSet) { - trans.audit().log(s); - } - } - } - - @Override - protected void _close(AuthzTrans trans) { - } - -} + import java.io.BufferedReader; + import java.io.File; + import java.io.FileReader; + import java.io.IOException; + import java.lang.reflect.Constructor; + import java.lang.reflect.InvocationTargetException; + import java.util.ArrayList; + import java.util.HashSet; + import java.util.List; + import java.util.Set; + + import org.onap.aaf.auth.batch.Batch; + import org.onap.aaf.auth.batch.reports.bodies.NotifyBody; + import org.onap.aaf.auth.env.AuthzTrans; + import org.onap.aaf.auth.org.Mailer; + import org.onap.aaf.auth.org.Organization.Identity; + import org.onap.aaf.auth.org.OrganizationException; + import org.onap.aaf.cadi.Access; + import org.onap.aaf.cadi.CadiException; + import org.onap.aaf.cadi.client.Holder; + import org.onap.aaf.cadi.util.CSV; + import org.onap.aaf.misc.env.APIException; + import org.onap.aaf.misc.env.util.Chrono; + + public class Notify extends Batch { + private static final String HTML_CSS = "HTML_CSS"; + private final Mailer mailer; + private final String header; + private final String footer; + private Set<File> notifyFile; + public final String guiURL; + private int maxEmails; + private int indent; + + public Notify(AuthzTrans trans) throws APIException, IOException, OrganizationException { + super(trans.env()); + String mailerCls = env.getProperty("MAILER"); + String mailFrom = env.getProperty("MAIL_FROM"); + String header_html = env.getProperty("HEADER_HTML"); + String footer_html = env.getProperty("FOOTER_HTML"); + String maxEmails = env.getProperty("MAX_EMAIL"); + guiURL = env.getProperty("GUI_URL"); + this.maxEmails = maxEmails==null?1:Integer.parseInt(maxEmails); + if(mailerCls==null || mailFrom==null || guiURL==null || header_html==null || footer_html==null) { + throw new APIException("Notify requires MAILER, MAILER_FROM, GUI_URL, HEADER_HTML and FOOTER_HTML properties"); + } + try { + Class<?> mailc = Class.forName(mailerCls); + Constructor<?> mailcst = mailc.getConstructor(Access.class); + mailer = (Mailer)mailcst.newInstance(env.access()); + } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + throw new APIException("Unable to construct " + mailerCls,e); + } + + String line; + StringBuilder sb = new StringBuilder(); + BufferedReader br = new BufferedReader(new FileReader(header_html)); + try { + while((line=br.readLine())!=null) { + sb.append(line); + sb.append('\n'); + } + String html_css = env.getProperty(HTML_CSS); + int hc = sb.indexOf(HTML_CSS); + if(hc!=0 && html_css!=null) { + header = sb.replace(hc,hc+HTML_CSS.length(), html_css).toString(); + } else { + header = sb.toString(); + } + } finally { + br.close(); + } + + // Establish index from header + int lastTag = header.lastIndexOf('<'); + if(lastTag>0) { + int prevCR = header.lastIndexOf('\n',lastTag); + if(prevCR>0) { + indent = lastTag-prevCR; + } else { + indent = 6; //arbitrary + } + } + + + sb.setLength(0); + br = new BufferedReader(new FileReader(footer_html)); + try { + while((line=br.readLine())!=null) { + sb.append(line); + sb.append('\n'); + } + footer = sb.toString(); + } finally { + br.close(); + } + + // Class Load possible data + NotifyBody.load(env.access()); + + // Create Intermediate Output + File logDir = logDir(); + notifyFile = new HashSet<>(); + if(args().length>0) { + for(int i=0;i<args().length;++i) { + notifyFile.add(new File(logDir, args()[i])); + } + } else { + String fmt = "%s"+Chrono.dateOnlyStamp()+".csv"; + File file; + for(NotifyBody nb : NotifyBody.getAll()) { + file = new File(logDir,String.format(fmt, nb.name())); + if(file.exists()) { + trans.info().printf("Processing '%s' in %s",nb.type(),file.getCanonicalPath()); + notifyFile.add(file); + } else { + trans.info().printf("No Files found for %s",nb.name()); + } + } + } + } + + @Override + protected void run(AuthzTrans trans) { + List<String> toList = new ArrayList<>(); + List<String> ccList = new ArrayList<>(); + AuthzTrans noAvg = trans.env().newTransNoAvg(); + String subject = "Test Notify"; + boolean urgent = false; + + + + final Notify notify = this; + final Holder<List<String>> info = new Holder<>(null); + final Set<String> errorSet = new HashSet<>(); + + try { + for(File f : notifyFile) { + CSV csv = new CSV(env.access(),f); + try { + csv.visit(new CSV.Visitor() { + @Override + public void visit(List<String> row) throws IOException, CadiException { + if("info".equals(row.get(0))) { + info.set(row); + } + if(info.get()==null) { + throw new CadiException("First line of Feed MUST contain 'info' record"); + } + String key = row.get(0)+'|'+info.get().get(1); + NotifyBody body = NotifyBody.get(key); + if(body==null) { + errorSet.add("No NotifyBody defined for " + key); + } else { + body.store(row); + } + } + }); + } catch (IOException | CadiException e) { + e.printStackTrace(); + } + + } + + // now create Notification + for(NotifyBody nb : NotifyBody.getAll()) { + String run = nb.type()+nb.name(); + String test = dryRun?run:null; + ONE_EMAIL: + for(String id : nb.users()) { + + toList.clear(); + ccList.clear(); + try { + Identity identity = trans.org().getIdentity(noAvg, id); + if(identity==null) { + trans.warn().printf("%s is invalid for this Organization. Skipping notification.",id); + } else { + if(!identity.isPerson()) { + identity = identity.responsibleTo(); + } + for(int i=1;i<nb.escalation();++i) { + if(identity != null) { + if(i==1) { + toList.add(identity.email()); + } else { + identity=identity.responsibleTo(); + ccList.add(identity.email()); + } + } + } + + StringBuilder content = new StringBuilder(); + content.append(String.format(header,version,Identity.mixedCase(identity.firstName()))); + + nb.body(noAvg, content, indent, notify, id); + content.append(footer); + + if(mailer.sendEmail(noAvg, test, toList, ccList, subject,content.toString(), urgent)) { + nb.inc(); + } else { + trans.error().log("Mailer failed to send Mail"); + } + if(maxEmails>0 && nb.count()>=maxEmails) { + break ONE_EMAIL; + } + } + } catch (OrganizationException e) { + trans.error().log(e); + } + } + trans.info().printf("Emailed %d for %s",nb.count(),run); + } + + + } finally { + for(String s : errorSet) { + trans.audit().log(s); + } + } + } + + @Override + protected void _close(AuthzTrans trans) { + } + + } diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/PrepExtend.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/PrepExtend.java index d0eab007..47a1b600 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/PrepExtend.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/PrepExtend.java @@ -133,7 +133,7 @@ public class PrepExtend extends Batch { */ UserRole.load(trans, session, UserRole.v2_0_11, ur -> { if(from.before(ur.expires()) && to.after(ur.expires())) { - ur.row(cw); + ur.row(cw,UserRole.UR); } }); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyBody.java index 453c2f2f..b36cf648 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyBody.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyBody.java @@ -50,6 +50,7 @@ public abstract class NotifyBody { private final String type; private String date; private int escalation; + private int count; public NotifyBody(final String type, final String name) { rows = new TreeMap<>(); @@ -57,6 +58,7 @@ public abstract class NotifyBody { this.type = type; date=""; escalation = 1; + count = 0; } public void store(List<String> row) { @@ -87,6 +89,10 @@ public abstract class NotifyBody { return name; } + public String type() { + return type; + } + public String date() { return date; } @@ -183,7 +189,6 @@ public abstract class NotifyBody { } } } - protected void println(StringBuilder sb, int indent, Object ... objs) { for(int i=0;i<indent;++i) { @@ -195,12 +200,24 @@ public abstract class NotifyBody { sb.append('\n'); } - protected void printCell(StringBuilder sb, int indent, String current, String prev) { + protected String printCell(StringBuilder sb, int indent, String current, String prev) { if(current.equals(prev)) { println(sb,indent,DUPL); } else { - println(sb,indent,"<td>",current,"</td>"); + printCell(sb,indent,current); } + return current; // use to set prev... + } + + protected void printCell(StringBuilder sb, int indent, String current) { + println(sb,indent,"<td>",current,"</td>"); + } + + public synchronized void inc() { + ++count; + } + + public int count() { + return count; } - } diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyCredBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyCredBody.java index 2369582d..e06be053 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyCredBody.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyCredBody.java @@ -21,11 +21,13 @@ package org.onap.aaf.auth.batch.reports.bodies; import java.io.IOException; +import java.util.GregorianCalendar; import java.util.List; import org.onap.aaf.auth.batch.reports.Notify; import org.onap.aaf.auth.env.AuthzTrans; import org.onap.aaf.cadi.Access; +import org.onap.aaf.misc.env.util.Chrono; public abstract class NotifyCredBody extends NotifyBody { @@ -35,32 +37,40 @@ public abstract class NotifyCredBody extends NotifyBody { // Default explanation = "The following Credentials are expiring on the dates shown. " - + "Failure to act before the expiration date will cause your App's Authentications to fail."; + + "Failure to act before the expiration date will cause your App's " + + "Authentications to fail." + + "<h3>Instructions for 'Password':</h3><ul>" + + "<li>Click on the Fully Qualified ID to ADD a new Password</li>" + + "<li><b>REMEMBER!</b> You are not finished until you <ol>" + + "<li><b>CHANGE <i>ALL</i></b> the configurations on <b><i>ALL</i></b> your processes!!</li>" + + "<li><b>BOUNCE</b> them</li></ol>" + + "<li>IF there is a WARNING, click the link for more information</li>" + + "</ul>"; } @Override public boolean body(AuthzTrans trans, StringBuilder sb, int indent, Notify n, String id) { println(sb,indent,explanation); - println(sb,indent,"<br><br>"); println(sb,indent,"<table>"); indent+=2; println(sb,indent,"<tr>"); indent+=2; println(sb,indent,"<th>Fully Qualified ID</th>"); + println(sb,indent,"<th>Unique ID</th>"); println(sb,indent,"<th>Type</th>"); - println(sb,indent,"<th>Details</th>"); println(sb,indent,"<th>Expires</th>"); - println(sb,indent,"<th>Cred Detail Page</th>"); + println(sb,indent,"<th>Warnings</th>"); indent-=2; println(sb,indent,"</tr>"); - String theid, type, info, gui, expires, notes; - String p_theid=null, p_type=null, p_gui=null, p_expires=null; + String theid, type, info, expires, warnings; + GregorianCalendar gc = new GregorianCalendar(); for(List<String> row : rows.get(id)) { theid=row.get(1); switch(row.get(3)) { case "1": case "2": type = "Password"; + break; case "200": type = "x509 (Certificate)"; break; @@ -68,27 +78,22 @@ public abstract class NotifyCredBody extends NotifyBody { type = "Unknown, see AAF GUI"; break; } - gui = "<a href=\""+n.guiURL+"/creddetail?ns="+row.get(2)+"\">"+row.get(2)+"</a>"; - expires = row.get(4); + theid = "<a href=\""+n.guiURL+"/creddetail?ns="+row.get(2)+"\">"+theid+"</a>"; + gc.setTimeInMillis(Long.parseLong(row.get(5))); + expires = Chrono.niceUTCStamp(gc); info = row.get(6); - notes = row.get(8); - if(notes!=null && !notes.isEmpty()) { - info += "<br>" + notes; - } + //TODO get Warnings + warnings = ""; println(sb,indent,"<tr>"); indent+=2; - printCell(sb,indent,theid,p_theid); - printCell(sb,indent,type,p_type); - printCell(sb,indent,info,null); - printCell(sb,indent,expires,p_expires); - printCell(sb,indent,gui,p_gui); + printCell(sb,indent,theid); + printCell(sb,indent,info); + printCell(sb,indent,type); + printCell(sb,indent,expires); + printCell(sb,indent,warnings); indent-=2; println(sb,indent,"</tr>"); - p_theid=theid; - p_type=type; - p_gui=gui; - p_expires=expires; } indent-=2; println(sb,indent,"</table>"); diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyURBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyURBody.java new file mode 100644 index 00000000..e2c04d7f --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/NotifyURBody.java @@ -0,0 +1,104 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ +package org.onap.aaf.auth.batch.reports.bodies; + +import java.io.IOException; +import java.util.List; + +import org.onap.aaf.auth.batch.reports.Notify; +import org.onap.aaf.auth.env.AuthzTrans; +import org.onap.aaf.auth.org.Organization.Identity; +import org.onap.aaf.auth.org.OrganizationException; +import org.onap.aaf.cadi.Access; + +public abstract class NotifyURBody extends NotifyBody { + + private final String explanation; + public NotifyURBody(Access access, String name) throws IOException { + super("ur",name); + + // Default + explanation = "The Roles for the IDs listed will expire on the dates shown. If " + + "allowed to expire, the ID will no longer have access to the Permissions " + + "associated with that Role."; + } + + @Override + public boolean body(AuthzTrans trans, StringBuilder sb, int indent, Notify n, String id) { + String fullname = "n/a"; + String kind = "Name"; + try { + Identity identity = trans.org().getIdentity(trans, id); + if(identity==null) { + trans.warn().printf("Cannot find %s in Organization",id); + } else { + fullname = identity.fullName(); + if(!identity.isPerson()) { + if((identity = identity.responsibleTo())!=null) { + kind = "AppID Sponsor"; + fullname = identity.fullName(); + } + } + } + } catch (OrganizationException e) { + trans.error().log(e); + fullname = "n/a"; + } + println(sb,indent,explanation); + println(sb,indent,"<table>"); + indent+=2; + println(sb,indent,"<tr>"); + indent+=2; + println(sb,indent,"<th>"+kind+"</th>"); + println(sb,indent,"<th>Fully Qualified ID</th>"); + println(sb,indent,"<th>Role</th>"); + println(sb,indent,"<th>Expires</th>"); + indent-=2; + println(sb,indent,"</tr>"); + + String name = null; + String fqi = null; + for(List<String> row : rows.get(id)) { + println(sb,indent,"<tr>"); + indent+=2; + name = printCell(sb,indent,fullname,name); + fqi = printCell(sb,indent,row.get(1),fqi); + printCell(sb,indent,row.get(2)+'.'+row.get(3)); + printCell(sb,indent,row.get(4)); + indent-=2; + println(sb,indent,"</tr>"); + } + indent-=2; + println(sb,indent,"</table>"); + + return true; + } + + @Override + public String user(List<String> row) { + if( (row != null) && row.size()>1) { + return row.get(1); + } + return null; + } + + +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/OneMonthNotifyCredBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/OneMonthNotifyCredBody.java new file mode 100644 index 00000000..c3ed4f69 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/OneMonthNotifyCredBody.java @@ -0,0 +1,32 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ +package org.onap.aaf.auth.batch.reports.bodies; + +import java.io.IOException; + +import org.onap.aaf.auth.batch.helpers.ExpireRange; +import org.onap.aaf.cadi.Access; + +public class OneMonthNotifyCredBody extends NotifyCredBody { + public OneMonthNotifyCredBody(Access access) throws IOException { + super(access, ExpireRange.ONE_MONTH); + } +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/OneMonthNotifyURBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/OneMonthNotifyURBody.java new file mode 100644 index 00000000..8e4ea8b6 --- /dev/null +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/OneMonthNotifyURBody.java @@ -0,0 +1,32 @@ +/** + * ============LICENSE_START==================================================== + * org.onap.aaf + * =========================================================================== + * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + * =========================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END==================================================== + * + */ +package org.onap.aaf.auth.batch.reports.bodies; + +import java.io.IOException; + +import org.onap.aaf.auth.batch.helpers.ExpireRange; +import org.onap.aaf.cadi.Access; + +public class OneMonthNotifyURBody extends NotifyURBody { + public OneMonthNotifyURBody(Access access) throws IOException { + super(access, ExpireRange.ONE_MONTH); + } +} diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/TwoWeeksNotifyCredBody.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/TwoWeeksNotifyCredBody.java index 97f09ac2..e8a55c91 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/TwoWeeksNotifyCredBody.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/bodies/TwoWeeksNotifyCredBody.java @@ -22,10 +22,11 @@ package org.onap.aaf.auth.batch.reports.bodies; import java.io.IOException; +import org.onap.aaf.auth.batch.helpers.ExpireRange; import org.onap.aaf.cadi.Access; public class TwoWeeksNotifyCredBody extends NotifyCredBody { public TwoWeeksNotifyCredBody(Access access) throws IOException { - super(access, "CredTwoWeek"); + super(access, ExpireRange.TWO_WEEK); } } diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Approvals.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Approvals.java index 341a072e..36fd6274 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Approvals.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Approvals.java @@ -23,17 +23,19 @@ package org.onap.aaf.auth.batch.update; import java.io.File; import java.io.IOException; +import java.text.ParseException; import java.util.ArrayList; import java.util.GregorianCalendar; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import org.onap.aaf.auth.batch.Batch; import org.onap.aaf.auth.batch.BatchPrincipal; import org.onap.aaf.auth.batch.approvalsets.ApprovalSet; +import org.onap.aaf.auth.batch.approvalsets.Pending; import org.onap.aaf.auth.batch.approvalsets.URApprovalSet; -import org.onap.aaf.auth.batch.helpers.Approval; import org.onap.aaf.auth.batch.helpers.BatchDataView; -import org.onap.aaf.auth.batch.helpers.Future; import org.onap.aaf.auth.batch.helpers.NS; import org.onap.aaf.auth.batch.helpers.Role; import org.onap.aaf.auth.batch.helpers.UserRole; @@ -42,159 +44,135 @@ import org.onap.aaf.auth.env.AuthzTrans; import org.onap.aaf.auth.layer.Result; import org.onap.aaf.auth.org.OrganizationException; import org.onap.aaf.cadi.CadiException; +import org.onap.aaf.cadi.client.Holder; import org.onap.aaf.cadi.util.CSV; import org.onap.aaf.misc.env.APIException; +import org.onap.aaf.misc.env.TimeTaken; +import org.onap.aaf.misc.env.Trans; import org.onap.aaf.misc.env.util.Chrono; public class Approvals extends Batch { private final AuthzTrans noAvg; private BatchDataView dataview; + private List<CSV> csvList; + private GregorianCalendar now; public Approvals(AuthzTrans trans) throws APIException, IOException, OrganizationException { super(trans.env()); noAvg = env.newTransNoAvg(); noAvg.setUser(new BatchPrincipal("batch:Approvals")); - - dataview = new BatchDataView(noAvg,cluster,dryRun); - - session = dataview.getSession(trans); - - Approval.load(trans, session, Approval.v2_0_17); - Future.load(trans, session, Future.v2_0_17); - Role.load(trans, session); + session = cluster.connect(); + dataview = new BatchDataView(noAvg,session,dryRun); NS.load(trans, session, NS.v2_0_11); + Role.load(trans, session); UserRole.load(trans, session, UserRole.v2_0_11); - } - @Override - protected void run(AuthzTrans trans) { - // Create Intermediate Output - final GregorianCalendar now = new GregorianCalendar(); + now = new GregorianCalendar(); - List<File> approveFiles = new ArrayList<>(); + csvList = new ArrayList<>(); + File f; if(args().length>0) { for(int i=0;i<args().length;++i) { - approveFiles.add(new File(logDir(), args()[i])); + f = new File(logDir(), args()[i]); + if(f.exists()) { + csvList.add(new CSV(env.access(),f).processAll()); + } else { + trans.error().printf("CSV File %s does not exist",f.getAbsolutePath()); + } } } else { - approveFiles.add(new File(logDir(),"OneMonth"+Chrono.dateOnlyStamp()+".csv")); + f = new File(logDir(), "Approvals"+Chrono.dateOnlyStamp()+".csv"); + if(f.exists()) { + csvList.add(new CSV(env.access(),f).processAll()); + } else { + trans.error().printf("CSV File %s does not exist",f.getAbsolutePath()); + } } - for(File f : approveFiles) { - trans.init().log("Processing File:",f.getAbsolutePath()); - } -// GregorianCalendar gc = new GregorianCalendar(); -// Date now = gc.getTime(); -// String today = Chrono.dateOnlyStamp(now); - for(File f : approveFiles) { - trans.info().log("Processing ",f.getAbsolutePath(),"for Approvals"); - if(f.exists()) { - CSV approveCSV = new CSV(env.access(),f).processAll(); - try { - approveCSV.visit(row -> { - switch(row.get(0)) { - case "ur": - UserRoleDAO.Data urdd = UserRole.row(row); - List<Approval> apvs = Approval.byUser.get(urdd.user); - - System.out.println(row); - if(apvs==null) { - // Create an Approval - ApprovalSet uras = new URApprovalSet(noAvg, now, dataview, () -> { - return urdd; - }); - Result<Void> rw = uras.write(noAvg); - if(rw.notOK()) { - System.out.println(rw.errorString()); - } + } + + @Override + protected void run(AuthzTrans trans) { + Map<String,Pending> mpending = new TreeMap<>(); + Holder<Integer> count = new Holder<>(0); + for(CSV approveCSV : csvList) { + TimeTaken tt = trans.start("Load Analyzed Reminders",Trans.SUB,approveCSV.name()); + try { + approveCSV.visit(row -> { + switch(row.get(0)) { + case Pending.REMIND: + try { + Pending p = new Pending(row); + Pending mp = mpending.get(row.get(1)); + if(mp==null) { + mpending.put(row.get(1), p); } else { - // Check that Existing Approval is still valid - for(Approval a : apvs) { - Future ticket = Future.data.get(a.add.ticket); - if(ticket==null) { - // Orphaned Approval - delete - } else { - - } - } + mp.inc(p); // FYI, unlikely } - break; - default: - System.out.println(row); - //noAvg.debug().printf("Ignoring %s",type); - } - }); - } catch (IOException | CadiException e) { - e.printStackTrace(); - // .... but continue with next row - } - - /* - List<Approval> pending = new ArrayList<>(); - boolean isOwner,isSupervisor; - for (Entry<String, List<Approval>> es : Approval.byApprover.entrySet()) { - isOwner = isSupervisor = false; - String approver = es.getKey(); - if (approver.indexOf('@')<0) { - approver += org.getRealm(); - } - Date latestNotify=null, soonestExpire=null; - GregorianCalendar latest=new GregorianCalendar(); - GregorianCalendar soonest=new GregorianCalendar(); - pending.clear(); - - for (Approval app : es.getValue()) { - Future f = app.getTicket()==null?null:Future.data.get(app.getTicket()); - if (f==null) { // only Ticketed Approvals are valid.. the others are records. - // Approvals without Tickets are no longer valid. - if ("pending".equals(app.getStatus())) { - app.setStatus("lapsed"); - app.update(noAvg,apprDAO,dryRun); // obeys dryRun - } - } else { - if ((soonestExpire==null && f.expires()!=null) || (soonestExpire!=null && f.expires()!=null && soonestExpire.before(f.expires()))) { - soonestExpire=f.expires(); - } - - if ("pending".equals(app.getStatus())) { - if (!isOwner) { - isOwner = "owner".equals(app.getType()); - } - if (!isSupervisor) { - isSupervisor = "supervisor".equals(app.getType()); - } + count.set(count.get()+1); + } catch (ParseException e) { + trans.error().log(e); + } + break; + } + }); + } catch (IOException | CadiException e) { + e.printStackTrace(); + // .... but continue with next row + } finally { + tt.done(); + } + } + trans.info().printf("Processed %d Reminder Rows", count.get()); - if ((latestNotify==null && app.getLast_notified()!=null) ||(latestNotify!=null && app.getLast_notified()!=null && latestNotify.before(app.getLast_notified()))) { - latestNotify=app.getLast_notified(); - } - pending.add(app); - } - } - } + count.set(0); + for(CSV approveCSV : csvList) { + TimeTaken tt = trans.start("Processing %s's UserRoles",Trans.SUB,approveCSV.name()); + try { + approveCSV.visit(row -> { + switch(row.get(0)) { + case UserRole.APPROVE_UR: + UserRoleDAO.Data urdd = UserRole.row(row); + // Create an Approval + ApprovalSet uras = new URApprovalSet(noAvg, now, dataview, () -> { + return urdd; + }); + Result<Void> rw = uras.write(noAvg); + if(rw.isOK()) { + Pending p = new Pending(); + Pending mp = mpending.get(urdd.user); + if(mp==null) { + mpending.put(urdd.user, p); + } else { + mp.inc(p); + } + count.set(count.get()+1); + } else { + trans.error().log(rw.errorString()); + } + break; + } + }); + dataview.flush(); + } catch (IOException | CadiException e) { + e.printStackTrace(); + // .... but continue with next row + } finally { + tt.done(); + } + trans.info().printf("Processed %d UserRoles", count.get()); - if (!pending.isEmpty()) { - boolean go = false; - if (latestNotify==null) { // never notified... make it so - go=true; - } else { - if (!today.equals(Chrono.dateOnlyStamp(latest))) { // already notified today - latest.setTime(latestNotify); - soonest.setTime(soonestExpire); - int year; - int days = soonest.get(GregorianCalendar.DAY_OF_YEAR)-latest.get(GregorianCalendar.DAY_OF_YEAR); - days+=((year=soonest.get(GregorianCalendar.YEAR))-latest.get(GregorianCalendar.YEAR))*365 + - (soonest.isLeapYear(year)?1:0); - if (days<7) { // If Expirations get within a Week (or expired), notify everytime. - go = true; - } - } - } - } - */ + count.set(0); + tt = trans.start("Notify for Pending", Trans.SUB); + try { + + } finally { + tt.done(); } - } + trans.info().printf("Created %d Notifications", count.get()); + } } @Override diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java index bcc8591a..dad03ce5 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java @@ -33,6 +33,7 @@ import org.onap.aaf.auth.batch.Batch; import org.onap.aaf.auth.batch.BatchPrincipal; import org.onap.aaf.auth.batch.helpers.Approval; import org.onap.aaf.auth.batch.helpers.CQLBatch; +import org.onap.aaf.auth.batch.helpers.CQLBatchLoop; import org.onap.aaf.auth.batch.helpers.Cred; import org.onap.aaf.auth.batch.helpers.Future; import org.onap.aaf.auth.batch.helpers.UserRole; @@ -53,168 +54,150 @@ import org.onap.aaf.misc.env.TimeTaken; import org.onap.aaf.misc.env.util.Chrono; public class Remove extends Batch { - private final AuthzTrans noAvg; - private HistoryDAO historyDAO; + private final AuthzTrans noAvg; + private HistoryDAO historyDAO; private CQLBatch cqlBatch; - public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException { - super(trans.env()); - trans.info().log("Starting Connection Process"); - - noAvg = env.newTransNoAvg(); - noAvg.setUser(new BatchPrincipal("Remove")); - - TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); - try { - historyDAO = new HistoryDAO(trans, cluster, CassAccess.KEYSPACE); - TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE); - try { - session = historyDAO.getSession(trans); - } finally { - tt2.done(); - } - cqlBatch = new CQLBatch(noAvg.info(),session); - - - } finally { - tt0.done(); - } - } - - @Override - protected void run(AuthzTrans trans) { - final int maxBatch = 25; - - // Create Intermediate Output - File logDir = logDir(); - - List<File> remove = new ArrayList<>(); - if(args().length>0) { - for(int i=0;i<args().length;++i) { - remove.add(new File(logDir, args()[i])); - } - } else { - remove.add(new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv")); - } - - for(File f : remove) { - trans.init().log("Processing File:",f.getAbsolutePath()); - } - - final Holder<Boolean> ur = new Holder<>(false); - final Holder<Boolean> cred = new Holder<>(false); - final Holder<Boolean> x509 = new Holder<>(false); - final Holder<String> memoFmt = new Holder<String>(""); - final HistoryDAO.Data hdd = new HistoryDAO.Data(); - final String orgName = trans.org().getName(); - - hdd.action="delete"; - hdd.reconstruct = ByteBuffer.allocate(0); - hdd.user = noAvg.user(); - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); - hdd.yr_mon = Integer.parseInt(sdf.format(new Date())); - - try { - for(File f : remove) { - trans.info().log("Processing ",f.getAbsolutePath(),"for Deletions"); - if(f.exists()) { - CSV removeCSV = new CSV(env.access(),f); - - try { - final StringBuilder sb = cqlBatch.begin(); - final Holder<Integer> hi = new Holder<Integer>(0); - removeCSV.visit(new CSV.Visitor() { - @Override - public void visit(List<String> row) throws IOException, CadiException { - int i = hi.get(); - if(i>=maxBatch) { - cqlBatch.execute(dryRun); - hi.set(0); - cqlBatch.begin(); - i=0; - } - switch(row.get(0)) { - case "info": - switch(row.get(1)) { - case "Delete": - memoFmt.set("%s expired from %s on %s"); - break; - case "NotInOrgDelete": - memoFmt.set("Identity %s was removed from %s on %s"); - break; - - } - break; - case "ur": - if(!ur.get()) { - ur.set(true); - } - hi.set(++i); - UserRole.batchDelete(sb,row); - hdd.target=UserRoleDAO.TABLE; - hdd.subject=UserRole.histSubject(row); - hdd.memo=UserRole.histMemo(memoFmt.get(), row); - historyDAO.createBatch(sb, hdd); - break; - case "cred": - if(!cred.get()) { - cred.set(true); - } - hi.set(++i); - Cred.batchDelete(sb,row); - hdd.target=CredDAO.TABLE; - hdd.subject=Cred.histSubject(row); - hdd.memo=Cred.histMemo(memoFmt.get(), orgName,row); - historyDAO.createBatch(sb, hdd); - break; - case "x509": - if(!x509.get()) { - x509.set(true); - } - hi.set(++i); - X509.row(sb,row); - hdd.target=CertDAO.TABLE; - hdd.subject=X509.histSubject(row); - hdd.memo=X509.histMemo(memoFmt.get(),row); - historyDAO.createBatch(sb, hdd); - break; - case "future": - // Not cached - hi.set(++i); - Future.deleteByIDBatch(sb,row.get(1)); - break; - case "approval": - // Not cached - hi.set(++i); - Approval.deleteByIDBatch(sb,row.get(1)); - break; - } + public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException { + super(trans.env()); + trans.info().log("Starting Connection Process"); + + noAvg = env.newTransNoAvg(); + noAvg.setUser(new BatchPrincipal("Remove")); + + TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); + try { + historyDAO = new HistoryDAO(trans, cluster, CassAccess.KEYSPACE); + TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE); + try { + session = historyDAO.getSession(trans); + } finally { + tt2.done(); + } + cqlBatch = new CQLBatch(noAvg.info(),session); + + + } finally { + tt0.done(); + } + } + + @Override + protected void run(AuthzTrans trans) { + + // Create Intermediate Output + File logDir = logDir(); + + List<File> remove = new ArrayList<>(); + if(args().length>0) { + for(int i=0;i<args().length;++i) { + remove.add(new File(logDir, args()[i])); + } + } else { + remove.add(new File(logDir,"Delete"+Chrono.dateOnlyStamp()+".csv")); + } + + for(File f : remove) { + trans.init().log("Processing File:",f.getAbsolutePath()); + } + + final Holder<Boolean> ur = new Holder<>(false); + final Holder<Boolean> cred = new Holder<>(false); + final Holder<Boolean> x509 = new Holder<>(false); + final Holder<String> memoFmt = new Holder<String>(""); + final HistoryDAO.Data hdd = new HistoryDAO.Data(); + final String orgName = trans.org().getName(); + + hdd.action="delete"; + hdd.reconstruct = ByteBuffer.allocate(0); + hdd.user = noAvg.user(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); + hdd.yr_mon = Integer.parseInt(sdf.format(new Date())); + + try { + final CQLBatchLoop cbl = new CQLBatchLoop(cqlBatch,50,dryRun); + for(File f : remove) { + trans.info().log("Processing ",f.getAbsolutePath(),"for Deletions"); + if(f.exists()) { + CSV removeCSV = new CSV(env.access(),f); + try { + removeCSV.visit( row -> { + cbl.preLoop(); + switch(row.get(0)) { + case "info": + switch(row.get(1)) { + case "Delete": + memoFmt.set("%s expired from %s on %s"); + break; + case "NotInOrgDelete": + memoFmt.set("Identity %s was removed from %s on %s"); + break; + } + break; + case "ur": + if(!ur.get()) { + ur.set(true); + } + UserRole.batchDelete(cbl.inc(),row); + hdd.target=UserRoleDAO.TABLE; + hdd.subject=UserRole.histSubject(row); + hdd.memo=UserRole.histMemo(memoFmt.get(), row); + historyDAO.createBatch(cbl.inc(), hdd); + break; + case "cred": + if(!cred.get()) { + cred.set(true); + } + Cred.batchDelete(cbl.inc(),row); + hdd.target=CredDAO.TABLE; + hdd.subject=Cred.histSubject(row); + hdd.memo=Cred.histMemo(memoFmt.get(), orgName,row); + historyDAO.createBatch(cbl.inc(), hdd); + break; + case "x509": + if(!x509.get()) { + x509.set(true); + } + X509.row(cbl.inc(),row); + hdd.target=CertDAO.TABLE; + hdd.subject=X509.histSubject(row); + hdd.memo=X509.histMemo(memoFmt.get(),row); + historyDAO.createBatch(cbl.inc(), hdd); + break; + case "future": + // Not cached + Future.deleteByIDBatch(cbl.inc(),row.get(1)); + break; + case "approval": + // Not cached + Approval.deleteByIDBatch(cbl.inc(),row.get(1)); + break; } }); - cqlBatch.execute(dryRun); + cbl.flush(); } catch (IOException | CadiException e) { e.printStackTrace(); } - } else { - trans.error().log("File",f.getAbsolutePath(),"does not exist."); - } - } - } finally { - if(ur.get()) { - cqlBatch.touch(UserRoleDAO.TABLE, 0, UserRoleDAO.CACHE_SEG, dryRun); - } - if(cred.get()) { - cqlBatch.touch(CredDAO.TABLE, 0, CredDAO.CACHE_SEG, dryRun); - } - if(x509.get()) { - cqlBatch.touch(CertDAO.TABLE, 0, CertDAO.CACHE_SEG, dryRun); - } - } - } - - @Override - protected void _close(AuthzTrans trans) { - session.close(); - } + } else { + trans.error().log("File",f.getAbsolutePath(),"does not exist."); + } + } + } finally { + if(ur.get()) { + cqlBatch.touch(UserRoleDAO.TABLE, 0, UserRoleDAO.CACHE_SEG, dryRun); + } + if(cred.get()) { + cqlBatch.touch(CredDAO.TABLE, 0, CredDAO.CACHE_SEG, dryRun); + } + if(x509.get()) { + cqlBatch.touch(CertDAO.TABLE, 0, CertDAO.CACHE_SEG, dryRun); + } + } + } + + @Override + protected void _close(AuthzTrans trans) { + session.close(); + } } |