Merge "Commit 8 for Create Optimized Sched API"

3 files changed, 686 insertions, 20 deletions

diff --git a/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/CmsoOptimizerClient.java b/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/CmsoOptimizerClient.java
new file mode 100644
index 0000000..d81c293
--- /dev/null
+++ b/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/CmsoOptimizerClient.java
@@ -0,0 +1,401 @@
+/*
+ * Copyright © 2017-2019 AT&T Intellectual Property. Modifications Copyright © 2018 IBM.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ * Unless otherwise specified, all documentation contained herein is licensed under the Creative
+ * Commons License, Attribution 4.0 Intl. (the "License"); you may not use this documentation except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * https://creativecommons.org/licenses/by/4.0/
+ *
+ * Unless required by applicable law or agreed to in writing, documentation distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onap.optf.cmso.optimizer;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import javax.ws.rs.ProcessingException;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.ResponseProcessingException;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.onap.observations.Mdc;
+import org.onap.observations.Observation;
+import org.onap.optf.cmso.common.BasicAuthenticatorFilter;
+import org.onap.optf.cmso.common.CMSStatusEnum;
+import org.onap.optf.cmso.common.LogMessages;
+import org.onap.optf.cmso.common.PropertiesManagement;
+import org.onap.optf.cmso.filters.CmsoClientFilters;
+import org.onap.optf.cmso.model.DomainData;
+import org.onap.optf.cmso.model.Schedule;
+import org.onap.optf.cmso.model.dao.ScheduleDAO;
+import org.onap.optf.cmso.optimizer.model.OptimizerRequest;
+import org.onap.optf.cmso.optimizer.model.OptimizerResponse;
+import org.onap.optf.cmso.service.rs.models.HealthCheckComponent;
+import org.onap.optf.cmso.service.rs.models.v2.NameValue;
+import org.onap.optf.cmso.service.rs.models.v2.SchedulingData;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * The Class CmsoOptimizerClient.
+ */
+@Component
+public class CmsoOptimizerClient {
+    private static EELFLogger debug = EELFManager.getInstance().getDebugLogger();
+
+    @Autowired
+    ScheduleDAO scheduleDAO;
+
+    @Autowired
+    Environment env;
+
+    @Autowired
+    PropertiesManagement pm;
+
+    @Autowired
+    CmsoOptimizerHandler optimizerHandler;
+
+    /**
+     * Schedule optimization.
+     *
+     * @param uuid - non empty
+     */
+    public void scheduleOptimization(UUID uuid) {
+        Map<String, String> mdcSave = Mdc.save();
+        try {
+            // Ensure that only one cmso is requsting this call to optimizer
+            Schedule schedule = scheduleDAO.lockOne(uuid);
+            if (schedule.getStatus().equals(CMSStatusEnum.PendingSchedule.toString())) {
+                scheduleNewOptimization(schedule);
+            }
+            if (schedule.getStatus().equals(CMSStatusEnum.OptimizationInProgress.toString())) {
+                pollOptimizer(schedule);
+            }
+            return;
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+        } finally {
+            Mdc.restore(mdcSave);
+        }
+    }
+
+    /**
+     * Schedule New Optimization.
+     *
+     * @param schedule - schedule
+     */
+    public void scheduleNewOptimization(Schedule schedule) {
+        try {
+            Integer maxAttempts = env.getProperty("cmso.optimizer.maxAttempts", Integer.class, 20);
+            //
+            // Only 'successfully' process one schedule per invocation
+            // If a schedule attemp fails and it could be because of the data in the
+            // message, try the next one. We don't want bad data to
+            //
+            if (schedule.getOptimizerAttemptsToSchedule() >= maxAttempts) {
+                schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+                schedule.setOptimizerMessage("Maximum number of attempts exceeded " + maxAttempts);
+                updateScheduleStatus(schedule);
+                return;
+            }
+            OptimizerRequest cmReq = null;
+            try {
+                cmReq = buildRequestMessage(schedule);
+                if (cmReq == null) {
+                    return;
+                }
+            } catch (Exception e) {
+                Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+                schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+                schedule.setOptimizerMessage("Unexpected exception: " + e.getMessage());
+                updateScheduleStatus(schedule);
+                return;
+            }
+            initiateOptimization(schedule, cmReq);
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+        }
+    }
+
+    private void initiateOptimization(Schedule schedule, OptimizerRequest cmReq) {
+        try {
+            String optimizerurl = env.getProperty("cmso.optimizer.request.url");
+            String username = env.getProperty("mechid.user");
+            String password = pm.getProperty("mechid.pass", "");
+            // This service will call optimizer for each PendingSchedule
+            // If the request is successfully scheduled in optimizer, status will be
+            // updated to OptimizationInProgress.
+            Client client = ClientBuilder.newClient();
+            client.register(new BasicAuthenticatorFilter(username, password));
+            client.register(new CmsoClientFilters());
+            WebTarget optimizerTarget = client.target(optimizerurl);
+            Invocation.Builder invocationBuilder = optimizerTarget.request(MediaType.APPLICATION_JSON);
+            try {
+                //
+                // First, push OptimizationInProgress to the DB (flush()) assuming a 202 status,
+                // in case the optimizer callback is received prior to the
+                // commit of this transaction.
+                // optimizer Callback will throw an error if it receives a response in the incorrect
+                // state.
+                //
+                schedule.setOptimizerTransactionId(cmReq.getRequestId());
+                schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+                schedule.setStatus(CMSStatusEnum.OptimizationInProgress.toString());
+                updateScheduleStatus(schedule);
+                debug.debug("optimizer url / user: " + optimizerurl + " / " + username);
+                debug.debug("optimizer Request: " + new ObjectMapper().writeValueAsString(cmReq));
+                Observation.report(LogMessages.OPTIMIZER_REQUEST, "Begin", schedule.getScheduleId(), optimizerurl);
+                Response response = invocationBuilder.post(Entity.json(cmReq));
+                Observation.report(LogMessages.OPTIMIZER_REQUEST, "End", schedule.getScheduleId(), optimizerurl);
+                switch (response.getStatus()) {
+                    case 202:
+                        debug.debug("Successfully scheduled optimization: " + schedule.getScheduleId());
+                        // Scheduled with optimizer
+                        break;
+                    case 400: // Bad request
+                    {
+                        schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+                        schedule.setOptimizerStatus("HTTP Status: " + response.getStatus());
+                        String message = response.readEntity(String.class);
+                        schedule.setOptimizerMessage(message);
+                        schedule.setStatus(CMSStatusEnum.ScheduleFailed.toString());
+                        // Need to understand the cause of this error. May be teh same as optimizer
+                        // down.
+                        int tries = schedule.getOptimizerAttemptsToSchedule();
+                        tries++;
+                        schedule.setOptimizerAttemptsToSchedule(tries);
+                        updateScheduleStatus(schedule);
+                        Observation.report(LogMessages.OPTIMIZER_EXCEPTION, message);
+                    }
+                        break;
+                    case 500:
+                    default: {
+                        String message = response.readEntity(String.class);
+                        // SHould probably track the number of retries.
+                        schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+                        int tries = schedule.getOptimizerAttemptsToSchedule();
+                        tries++;
+                        schedule.setStatus(CMSStatusEnum.ScheduleFailed.toString());
+                        schedule.setOptimizerAttemptsToSchedule(tries);
+                        schedule.setOptimizerMessage(message);
+                        updateScheduleStatus(schedule);
+                        /// Got processing error response
+                        // may be transient, wait for next cycle.
+                        Observation.report(LogMessages.OPTIMIZER_EXCEPTION, message);
+                        // Wait until next cycle and try again.
+                    }
+                }
+            } catch (ResponseProcessingException e) {
+                schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+                schedule.setOptimizerStatus("Failed to parse optimizer response");
+                schedule.setStatus(CMSStatusEnum.ScheduleFailed.toString());
+                // Need to understand the cause of this error. May be teh same as optimizer down.
+                int tries = schedule.getOptimizerAttemptsToSchedule();
+                tries++;
+                schedule.setOptimizerAttemptsToSchedule(tries);
+                updateScheduleStatus(schedule);
+                // Getting invalid response from optimizer.
+                // May be data related.
+                Observation.report(LogMessages.OPTIMIZER_EXCEPTION, e, e.getMessage());
+            } catch (ProcessingException e) {
+                // Don't track number of retries on IO error (optimizer is down)
+                schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+                schedule.setStatus(CMSStatusEnum.PendingSchedule.toString());
+                updateScheduleStatus(schedule);
+                /// Cannot connect to optimizer
+                Observation.report(LogMessages.OPTIMIZER_EXCEPTION, e, e.getMessage());
+                // Wait until next cycle
+            }
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+        }
+
+    }
+
+    /**
+     * Poll the optimizer.
+     *
+     * @param schedule - schedule
+     */
+    public void pollOptimizer(Schedule schedule) {
+        try {
+            String optimizerurl = env.getProperty("cmso.optimizer.status.url");
+            String username = env.getProperty("mechid.user");
+            String password = pm.getProperty("mechid.pass", "");
+            Long timeout = env.getProperty("cmso.optimizer.request.timeout.secs", Long.class);
+            if (timeout == null) {
+                timeout = 3600L;
+            }
+            if (!optimizerurl.endsWith("/")) {
+                optimizerurl += "/";
+            }
+            Long now = System.currentTimeMillis();
+            if (now > schedule.getOptimizerDateTimeMillis() + (timeout * 1000)) {
+                schedule.setStatus(CMSStatusEnum.ScheduleFailed.toString());
+                updateScheduleStatus(schedule);
+                return;
+            }
+
+            optimizerurl += schedule.getOptimizerTransactionId();
+            // This service will call optimizer for each PendingSchedule
+            // If the request is successfully scheduled in optimizer, status will be
+            // updated to OptimizationInProgress.
+            Client client = ClientBuilder.newClient();
+            client.register(new BasicAuthenticatorFilter(username, password));
+            client.register(new CmsoClientFilters());
+            WebTarget optimizerTarget = client.target(optimizerurl);
+            Invocation.Builder invocationBuilder = optimizerTarget.request(MediaType.APPLICATION_JSON);
+            debug.debug("optimizer url / user: " + optimizerurl + " / " + username);
+            Observation.report(LogMessages.OPTIMIZER_REQUEST, "Begin", schedule.getScheduleId(), optimizerurl);
+            Response response = invocationBuilder.get();
+            Observation.report(LogMessages.OPTIMIZER_REQUEST, "End", schedule.getScheduleId(), optimizerurl);
+            switch (response.getStatus()) {
+                case 200:
+
+                    String optimizerResponseString = response.readEntity(String.class);
+                    ObjectMapper om = new ObjectMapper();
+                    OptimizerResponse optimizerResponse =
+                                    om.readValue(optimizerResponseString, OptimizerResponse.class);
+                    debug.debug("Successfully retrieved optimization: " + schedule.getScheduleId());
+                    optimizerHandler.handleOptimizerResponse(optimizerResponse, schedule);
+                    break;
+                default: // Bad request
+                {
+                    schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+                    schedule.setOptimizerStatus("HTTP Status: " + response.getStatus());
+                    String message = response.readEntity(String.class);
+                    schedule.setOptimizerMessage(message);
+                    schedule.setStatus(CMSStatusEnum.ScheduleFailed.toString());
+                    updateScheduleStatus(schedule);
+                    Observation.report(LogMessages.OPTIMIZER_EXCEPTION, message);
+                }
+            }
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+            schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+            schedule.setOptimizerMessage(e.getMessage());
+            schedule.setStatus(CMSStatusEnum.ScheduleFailed.toString());
+            updateScheduleStatus(schedule);
+        }
+    }
+
+    private OptimizerRequest buildRequestMessage(Schedule schedule) {
+        String request = schedule.getScheduleInfo();
+        ObjectMapper om = new ObjectMapper();
+        try {
+            SchedulingData info = om.readValue(request, SchedulingData.class);
+            OptimizerRequest orequest = new OptimizerRequest();
+            orequest.setChangeWindows(info.getChangeWindows());
+            orequest.setPolicies(info.getPolicies());
+            orequest.setRequestId(schedule.getScheduleId());
+            orequest.setCommonData(marshallCommonData(schedule));
+            orequest.setElements(info.getElements());
+            orequest.setAdditionalDuration(info.getAdditionalDurationInSeconds());
+            orequest.setNormalDuration(info.getNormalDurationInSeconds());
+            orequest.setConcurrencyLimit(info.getConcurrencyLimit());
+            return orequest;
+        } catch (Exception e) {
+            // Parsing should work here because this was a toString on the original object.
+            // We may have an issue when upgrading....
+            // Perhaps We create ChangeManagementSchedulingInfoV1, ...V2, etc.
+            // ANd try them one after another....
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, "Unable to parse message. Format changed?");
+            schedule.setOptimizerStatus("Failed to parse optimizer request");
+            schedule.setOptimizerDateTimeMillis(System.currentTimeMillis());
+            schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+            scheduleDAO.save(schedule);
+        }
+        return null;
+    }
+
+    private List<NameValue> marshallCommonData(Schedule schedule) {
+        List<NameValue> nvList = new ArrayList<>();
+        List<DomainData> ddList = schedule.getDomainData();
+        for (DomainData dd : ddList) {
+            NameValue nv = new NameValue();
+            nv.setName(dd.getName());
+            // TODO: handle other than String values
+            nv.setValue(dd.getValue());
+            nvList.add(nv);
+        }
+        return nvList;
+    }
+
+    /**
+     * Update schedule status.
+     *
+     * @param schedule the schedule
+     */
+    @Transactional
+    public void updateScheduleStatus(Schedule schedule) {
+        scheduleDAO.save(schedule);
+    }
+
+    /**
+     * Health check.
+     * @return
+     */
+    public HealthCheckComponent healthCheck() {
+        Map<String, String> mdcSave = Mdc.save();
+        HealthCheckComponent hcc = new HealthCheckComponent();
+        hcc.setName("OPtimizer Interface");
+        String optimizerurl = env.getProperty("cmso.optimizer.health.url");
+        String username = env.getProperty("mechid.user");
+        String password = pm.getProperty("mechid.pass", "");
+        hcc.setUrl(optimizerurl);
+        try {
+            Client client = ClientBuilder.newClient();
+            client.register(new BasicAuthenticatorFilter(username, password));
+            client.register(new CmsoClientFilters());
+
+            WebTarget optimizerTarget = client.target(optimizerurl);
+            Invocation.Builder invocationBuilder = optimizerTarget.request(MediaType.APPLICATION_JSON);
+            debug.debug("Optimizer url / user: " + optimizerurl + " / " + username);
+            Response response = invocationBuilder.get();
+            Observation.report(LogMessages.OPTIMIZER_REQUEST, "End", "healthcheck", optimizerurl);
+            String message = response.getStatus() + ":" + response.readEntity(String.class);
+            switch (response.getStatus()) {
+                case 200:
+                    debug.debug("Successful optimizer healthcheck");
+                    hcc.setHealthy(true);
+                    break;
+                case 400:
+                default:
+                    hcc.setStatus(message);
+                    break;
+            }
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e.toString());
+            hcc.setStatus(e.toString());
+        } finally {
+            Mdc.restore(mdcSave);
+        }
+        return hcc;
+    }
+}
diff --git a/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/CmsoOptimizerHandler.java b/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/CmsoOptimizerHandler.java
new file mode 100644
index 0000000..ea78df1
--- /dev/null
+++ b/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/CmsoOptimizerHandler.java
@@ -0,0 +1,262 @@
+/*******************************************************************************
+ * Copyright © 2019 AT&T Intellectual Property.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ * Unless otherwise specified, all documentation contained herein is licensed under the Creative
+ * Commons License, Attribution 4.0 Intl. (the "License"); you may not use this documentation except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * https://creativecommons.org/licenses/by/4.0/
+ *
+ * Unless required by applicable law or agreed to in writing, documentation distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.onap.optf.cmso.optimizer;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.core.Response.Status;
+import org.onap.observations.Observation;
+import org.onap.optf.cmso.common.CMSStatusEnum;
+import org.onap.optf.cmso.common.LogMessages;
+import org.onap.optf.cmso.common.exceptions.CMSException;
+import org.onap.optf.cmso.model.ChangeManagementGroup;
+import org.onap.optf.cmso.model.ChangeManagementSchedule;
+import org.onap.optf.cmso.model.Schedule;
+import org.onap.optf.cmso.model.dao.ChangeManagementChangeWindowDAO;
+import org.onap.optf.cmso.model.dao.ChangeManagementDetailDAO;
+import org.onap.optf.cmso.model.dao.ChangeManagementGroupDAO;
+import org.onap.optf.cmso.model.dao.ChangeManagementScheduleDAO;
+import org.onap.optf.cmso.model.dao.ScheduleDAO;
+import org.onap.optf.cmso.optimizer.model.OptimizerResponse;
+import org.onap.optf.cmso.optimizer.model.OptimizerScheduleInfo;
+import org.onap.optf.cmso.optimizer.model.ScheduledElement;
+import org.onap.optf.cmso.optimizer.model.UnScheduledElement;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+/**
+ * The Class CmsoOptimizerHandler.
+ */
+@Component
+public class CmsoOptimizerHandler {
+    private static EELFLogger debug = EELFManager.getInstance().getDebugLogger();
+
+    @Autowired
+    Environment env;
+
+    @Autowired
+    ChangeManagementScheduleDAO cmScheduleDAO;
+
+    @Autowired
+    ScheduleDAO scheduleDAO;
+
+    @Autowired
+    ChangeManagementGroupDAO cmGroupDAO;
+
+    @Autowired
+    ChangeManagementChangeWindowDAO cmChangeWindowDAO;
+
+    @Autowired
+    ChangeManagementDetailDAO cmDetailsDAO;
+
+    /**
+     * Handle optimizer response.
+     *
+     * @param response the response
+     * @param schedule the schedule
+     */
+    public void handleOptimizerResponse(OptimizerResponse response, Schedule schedule) {
+        try {
+            // Note that transaction ID and schedule ID are currently the same value.
+
+            String id = response.getRequestId();
+            CMSStatusEnum status = CMSStatusEnum.PendingApproval.fromString(schedule.getStatus());
+            debug.debug("Status at time of optimizer status is " + status.toString() + " for " + id);
+            switch (status) {
+                // PendingSchedule may be a valid status in the cases where SNIRO async call
+                // returns before
+                // We have committed the OptimizationInProgress status
+                // The dispatch logic ensures that we only every dispatch once.
+                case OptimizationInProgress:
+                    processResponse(response, schedule);
+                    scheduleDAO.save(schedule);
+                    break;
+                default:
+                    throw new CMSException(Status.PRECONDITION_FAILED, LogMessages.OPTIMIZER_CALLBACK_STATE_ERROR,
+                                    CMSStatusEnum.OptimizationInProgress.toString(), schedule.getStatus().toString());
+            }
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+        }
+    }
+
+    private void processResponse(OptimizerResponse response, Schedule schedule) {
+        try {
+            schedule.setOptimizerReturnDateTimeMillis(System.currentTimeMillis());
+            schedule.setOptimizerStatus(response.getStatus().toString());
+            schedule.setOptimizerMessage(response.getErrorMessage());
+            switch (response.getStatus()) {
+                case COMPLETED:
+                    saveSchedules(response, schedule);
+                    break;
+                case FAILED:
+                    schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+                    break;
+                case PENDING_OPTIMIZER:
+                case PENDING_TICKETS:
+                case PENDING_TOPOLOGY:
+                    // Leave status as In progress
+                    break;
+                default:
+                    break;
+            }
+            scheduleDAO.save(schedule);
+        } catch (CMSException e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+            schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+            schedule.setOptimizerStatus(e.getStatus().toString());
+            schedule.setOptimizerMessage(e.getLocalizedMessage());
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+            schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+            schedule.setOptimizerStatus("Exception");
+            schedule.setOptimizerMessage(e.getLocalizedMessage());
+        }
+    }
+
+    private void saveSchedules(OptimizerResponse response, Schedule schedule) throws CMSException {
+
+        // TODO: Persist the list of schedules in the DB
+
+        // For Dublin we choose the best schedule.
+        // and only request Accept on that one.
+        // FOr the future, the user will get to choose one of the persisted schedules
+        List<OptimizerScheduleInfo> schedules = response.getSchedules();
+
+        OptimizerScheduleInfo osi = chooseSchedule(schedules);
+        if (osi == null) {
+            schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+            schedule.setOptimizerMessage("No schedules returned for COMPLETED status");
+            return;
+        }
+        if (osi.getScheduledElements().size() == 0) {
+            schedule.setStatus(CMSStatusEnum.OptimizationFailed.toString());
+            schedule.setOptimizerMessage("No elements scheduled for COMPLETED status");
+            return;
+        }
+
+        List<ChangeManagementGroup> groups = cmGroupDAO.findBySchedulesID(schedule.getUuid());
+        Map<String, ChangeManagementGroup> updatedGroups = new HashMap<>();
+
+        for (ScheduledElement element : osi.getScheduledElements()) {
+            updateGroup(element, groups, updatedGroups);
+            String groupId = element.getGroupId();
+            String vnfName = element.getElementId();
+            ChangeManagementSchedule cms =
+                            cmScheduleDAO.findOneByScheduleUUIDGroupIdAndVnfName(schedule.getUuid(), groupId, vnfName);
+            cms.setStartTimeMillis(element.getStartTime().getTime());
+            cms.setFinishTimeMillis(element.getEndTime().getTime());
+            cms.setStatus(CMSStatusEnum.PendingApproval.toString());
+            cmScheduleDAO.save(cms);
+        }
+        if (osi.getUnScheduledElements() != null) {
+            for (UnScheduledElement element : osi.getUnScheduledElements()) {
+                String groupId = element.getGroupId();
+                String vnfName = element.getElementId();
+                ChangeManagementSchedule cms = cmScheduleDAO.findOneByScheduleUUIDGroupIdAndVnfName(schedule.getUuid(),
+                                groupId, vnfName);
+                cms.setStatus(CMSStatusEnum.NotScheduled.toString());
+                cmScheduleDAO.save(cms);
+
+            }
+        }
+
+        // Save any changes to the groups
+        for (ChangeManagementGroup cmg : updatedGroups.values()) {
+            cmGroupDAO.save(cmg);
+        }
+        schedule.setStatus(CMSStatusEnum.PendingApproval.toString());
+    }
+
+    private void updateGroup(ScheduledElement element, List<ChangeManagementGroup> groups,
+                    Map<String, ChangeManagementGroup> updatedGroups) {
+
+        // For Dublin the contents of CMG are not functional
+        // since were are doing individual scheduling.
+        // We log the not found exception, but do not
+        // throw it at this time.
+        try {
+            ChangeManagementGroup cmg = updatedGroups.get(element.getGroupId());
+            if (cmg == null) {
+                for (ChangeManagementGroup group : groups) {
+                    if (group.getGroupId().equals(element.getGroupId())) {
+                        cmg = group;
+                        break;
+                    }
+                }
+            }
+            if (cmg == null) {
+                throw new CMSException(Status.INTERNAL_SERVER_ERROR, LogMessages.MISSING_VALID_GROUP_FOR_ELEMENT,
+                                element.getElementId());
+            }
+            Long elementStartTime = element.getStartTime().getTime();
+            Long elementFinishTime = element.getEndTime().getTime();
+            if (cmg.getStartTimeMillis() == null || cmg.getStartTimeMillis() > elementStartTime) {
+                cmg.setStartTimeMillis(elementStartTime);
+                updatedGroups.put(cmg.getGroupId(), cmg);
+            }
+            if (cmg.getFinishTime() == null || cmg.getFinishTimeMillis() < elementFinishTime) {
+                cmg.setFinishTimeMillis(elementFinishTime);
+                updatedGroups.put(cmg.getGroupId(), cmg);
+            }
+            if (cmg.getLastInstanceStartTimeMillis() == null
+                            || cmg.getLastInstanceStartTimeMillis() < elementStartTime) {
+                cmg.setLastInstanceStartTimeMillis(elementStartTime);
+                updatedGroups.put(cmg.getGroupId(), cmg);
+            }
+        } catch (Exception e) {
+            Observation.report(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());
+        }
+    }
+
+    private OptimizerScheduleInfo chooseSchedule(List<OptimizerScheduleInfo> schedules) {
+        // The most scheduled elements is the priority
+        //
+        OptimizerScheduleInfo chosenOne = null;
+        for (OptimizerScheduleInfo osi : schedules) {
+            if (chosenOne == null || osi.getScheduledElements().size() > chosenOne.getScheduledElements().size()) {
+                chosenOne = osi;
+            } else {
+                // Same number of scheduled elements.
+                // What is the policy.
+                if (betterSchedule(osi, chosenOne)) {
+                    chosenOne = osi;
+                }
+            }
+        }
+        return chosenOne;
+    }
+
+    private boolean betterSchedule(OptimizerScheduleInfo osi, OptimizerScheduleInfo chosenOne) {
+        // TODO Create a more sophisticated choosing process -
+        return true;
+    }
+}
diff --git a/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/OptimizerQuartzJob.java b/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/OptimizerQuartzJob.java
index 12d56f3..48d6e73 100644
--- a/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/OptimizerQuartzJob.java
+++ b/cmso-service/src/main/java/org/onap/optf/cmso/optimizer/OptimizerQuartzJob.java
@@ -1,27 +1,27 @@
 /*

  * Copyright © 2017-2019 AT&T Intellectual Property.

  * Modifications Copyright © 2018 IBM.

- * 

+ *

  * Licensed under the Apache License, Version 2.0 (the "License");

  * you may not use this file except in compliance with the License.

  * You may obtain a copy of the License at

- * 

+ *

  *         http://www.apache.org/licenses/LICENSE-2.0

- * 

+ *

  * Unless required by applicable law or agreed to in writing, software

  * distributed under the License is distributed on an "AS IS" BASIS,

  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  * See the License for the specific language governing permissions and

  * limitations under the License.

- * 

- * 

+ *

+ *

  * Unless otherwise specified, all documentation contained herein is licensed

  * under the Creative Commons License, Attribution 4.0 Intl. (the "License");

  * you may not use this documentation except in compliance with the License.

  * You may obtain a copy of the License at

- * 

+ *

  *         https://creativecommons.org/licenses/by/4.0/

- * 

+ *

  * Unless required by applicable law or agreed to in writing, documentation

  * distributed under the License is distributed on an "AS IS" BASIS,

  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

@@ -31,24 +31,24 @@
 

 package org.onap.optf.cmso.optimizer;

 

+import com.att.eelf.configuration.EELFLogger;

+import com.att.eelf.configuration.EELFManager;

 import java.util.List;

 import java.util.Map;

 import java.util.UUID;

-

 import javax.ws.rs.client.Client;

 import javax.ws.rs.client.ClientBuilder;

 import javax.ws.rs.client.Invocation;

 import javax.ws.rs.client.WebTarget;

 import javax.ws.rs.core.MediaType;

 import javax.ws.rs.core.Response;

-

 import org.onap.observations.Mdc;

 import org.onap.optf.cmso.common.BasicAuthenticatorFilter;

 import org.onap.optf.cmso.common.CMSStatusEnum;

 import org.onap.optf.cmso.common.DomainsEnum;

 import org.onap.optf.cmso.common.LogMessages;

 import org.onap.optf.cmso.common.PropertiesManagement;

-import org.onap.optf.cmso.filters.CMSOClientFilters;

+import org.onap.optf.cmso.filters.CmsoClientFilters;

 import org.onap.optf.cmso.model.Schedule;

 import org.onap.optf.cmso.model.dao.ScheduleDAO;

 import org.quartz.DisallowConcurrentExecution;

@@ -60,9 +60,6 @@ import org.springframework.core.env.Environment;
 import org.springframework.scheduling.quartz.QuartzJobBean;

 import org.springframework.stereotype.Component;

 

-import com.att.eelf.configuration.EELFLogger;

-import com.att.eelf.configuration.EELFManager;

-

 @Component

 @DisallowConcurrentExecution

 public class OptimizerQuartzJob extends QuartzJobBean {

@@ -91,8 +88,8 @@ public class OptimizerQuartzJob extends QuartzJobBean {
         // return;

 

         try {

-            // This job will look at the schedules waiting to go to SNIRO

-            // (PendingSchedule),

+            // This job will look at the schedules waiting to go to Optimizer or waiting on response from optimizer

+            // (PendingSchedule, PendingOptimizer),

             // schedule the request and update the status to PendingSchedule

             // and update the state to OptimizationInProgress

             List<Schedule> schedules = scheduleDAO.findByDomainStatus(DomainsEnum.ChangeManagement.toString(),

@@ -100,6 +97,12 @@ public class OptimizerQuartzJob extends QuartzJobBean {
             for (Schedule s : schedules) {

                 scheduleOptimization(s);

             }

+            List<Schedule> inProgressSchedules = scheduleDAO.findByDomainStatus(DomainsEnum.ChangeManagement.toString(),

+                    CMSStatusEnum.OptimizationInProgress.toString());

+            for (Schedule s : inProgressSchedules)

+            {

+              scheduleOptimization(s);

+            }

 

         } catch (Exception e) {

             debug.debug(LogMessages.UNEXPECTED_EXCEPTION, e, e.getMessage());

@@ -120,7 +123,7 @@ public class OptimizerQuartzJob extends QuartzJobBean {
             String pass = pm.getProperty("mechid.pass", "");

             Client client = ClientBuilder.newClient();

             client.register(new BasicAuthenticatorFilter(user, pass));

-            client.register(new CMSOClientFilters());

+            client.register(new CmsoClientFilters());

             WebTarget target = client.target(url);

             Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON);

             Response response = null;

@@ -149,16 +152,16 @@ public class OptimizerQuartzJob extends QuartzJobBean {
      * According to the documentation I read, Quartz would queue a job without

      * waiting for the completion of the job with @DisallowConcurrentExecution to

      * complete so that there would be a backlog of triggers to process

-     * 

+     *

      * This was designed to spin though these stale triggers. When this didn't work,

      * I discovered that the behavior is that Quartz will wait for the appropriate

      * interval after @DisallowConcurrentExecution jobs complete.

-     * 

+     *

      * I tested by adding a sleep for an interval > the trigger interval

-     * 

+     *

      * QUartz appears to do what makes sense. Leaving this here in case issues

      * arise...

-     * 

+     *

      */

     @SuppressWarnings("unused")

     private boolean isStale(JobExecutionContext context) {