diff options
Diffstat (limited to 'plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli')
17 files changed, 2722 insertions, 0 deletions
diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java new file mode 100755 index 000000000..7a8b3deb0 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java @@ -0,0 +1,765 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2018 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.annotation.Nonnull; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import org.apache.commons.lang.StringUtils; + +import org.onap.ccsdk.sli.core.dblib.DbLibService; +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionManager; +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionResponse; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.MemberBuilder; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.PropertyKeys; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.resolver.HealthResolver; +import org.onap.ccsdk.sli.plugins.grtoolkit.resolver.SingleNodeHealthResolver; +import org.onap.ccsdk.sli.plugins.grtoolkit.resolver.SixNodeHealthResolver; +import org.onap.ccsdk.sli.plugins.grtoolkit.resolver.ThreeNodeHealthResolver; + +import org.json.JSONArray; +import org.json.JSONObject; + +import org.opendaylight.controller.cluster.datastore.DistributedDataStoreInterface; +import org.opendaylight.controller.md.sal.binding.api.DataBroker; +import org.opendaylight.controller.md.sal.binding.api.DataTreeChangeListener; +import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService; +import org.opendaylight.controller.sal.binding.api.BindingAwareBroker; +import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.GrToolkitService; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.Member; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.Site; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierInput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierOutput; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierOutputBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.site.health.output.SitesBuilder; +import org.opendaylight.yangtools.yang.common.RpcResult; +import org.opendaylight.yangtools.yang.common.RpcResultBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * API implementation of the {@code GrToolkitService} interface generated from + * the gr-toolkit.yang model. The RPCs contained within this class are meant to + * run in an architecture agnostic fashion, where the response is repeatable + * and predictable across any given node configuration. To facilitate this, + * health checking and failover logic has been abstracted into the + * {@code HealthResolver} classes. + * <p> + * Anyone who wishes to write a custom resolver for use with GR Toolkit should + * extend the {@code HealthResolver} class. The currently provided resolvers + * are useful references for further implementation. + * + * @author Anthony Haddox + * @see GrToolkitService + * @see HealthResolver + * @see SingleNodeHealthResolver + * @see ThreeNodeHealthResolver + * @see SixNodeHealthResolver + */ +public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataTreeChangeListener { + private static final String APP_NAME = "gr-toolkit"; + private static final String PROPERTIES_FILE = System.getenv("SDNC_CONFIG_DIR") + "/gr-toolkit.properties"; + private String akkaConfig; + private String httpProtocol; + private String siteIdentifier = System.getenv("SITE_NAME"); + private final Logger log = LoggerFactory.getLogger(GrToolkitProvider.class); + private final ExecutorService executor; + protected DataBroker dataBroker; + protected NotificationPublishService notificationService; + protected RpcProviderRegistry rpcRegistry; + protected BindingAwareBroker.RpcRegistration<GrToolkitService> rpcRegistration; + protected DbLibService dbLib; + private String member; + private ClusterActor self; + private HashMap<String, ClusterActor> memberMap; + private Properties properties; + private DistributedDataStoreInterface configDatastore; + private HealthResolver resolver; + + /** + * Constructs the provider for the GR Toolkit API. Dependencies are + * injected using the GrToolkit.xml blueprint. + * + * @param dataBroker The Data Broker + * @param notificationProviderService The Notification Service + * @param rpcProviderRegistry The RPC Registry + * @param configDatastore The Configuration Data Store provided by the controller + * @param dbLibService Reference to the controller provided DbLibService + */ + public GrToolkitProvider(DataBroker dataBroker, + NotificationPublishService notificationProviderService, + RpcProviderRegistry rpcProviderRegistry, + DistributedDataStoreInterface configDatastore, + DbLibService dbLibService) { + log.info("Creating provider for {}", APP_NAME); + this.executor = Executors.newFixedThreadPool(1); + this.dataBroker = dataBroker; + this.notificationService = notificationProviderService; + this.rpcRegistry = rpcProviderRegistry; + this.configDatastore = configDatastore; + this.dbLib = dbLibService; + initialize(); + } + + /** + * Initializes some structures necessary to hold health check information + * and perform failovers. + */ + private void initialize() { + log.info("Initializing provider for {}", APP_NAME); + createContainers(); + setProperties(); + defineMembers(); + rpcRegistration = rpcRegistry.addRpcImplementation(GrToolkitService.class, this); + log.info("Initialization complete for {}", APP_NAME); + } + + /** + * Creates the {@code Properties} object with the contents of + * gr-toolkit.properties, found at the {@code SDNC_CONFIG_DIR} directory, + * which should be set as an environment variable. If the properties file + * is not found, GR Toolkit will not function. + */ + private void setProperties() { + log.info("Loading properties from {}", PROPERTIES_FILE); + properties = new Properties(); + File propertiesFile = new File(PROPERTIES_FILE); + if(!propertiesFile.exists()) { + log.warn("setProperties(): Properties file not found."); + } else { + try(FileInputStream fileInputStream = new FileInputStream(propertiesFile)) { + properties.load(fileInputStream); + if(!properties.containsKey(PropertyKeys.SITE_IDENTIFIER)) { + properties.put(PropertyKeys.SITE_IDENTIFIER, "Unknown Site"); + } + httpProtocol = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? "https://" : "http://"; + akkaConfig = properties.getProperty(PropertyKeys.AKKA_CONF_LOCATION).trim(); + if(StringUtils.isEmpty(siteIdentifier)) { + siteIdentifier = properties.getProperty(PropertyKeys.SITE_IDENTIFIER).trim(); + } + log.info("setProperties(): Loaded properties."); + } catch(IOException e) { + log.error("setProperties(): Error loading properties.", e); + } + } + } + + /** + * Parses the akka.conf file used by the controller to define an akka + * cluster. This method requires the <i>seed-nodes</i> definition to exist + * on a single line. + */ + private void defineMembers() { + member = configDatastore.getActorUtils().getCurrentMemberName().getName(); + log.info("defineMembers(): Cluster member: {}", member); + + log.info("defineMembers(): Parsing akka.conf for cluster memberMap..."); + try { + File akkaConfigFile = new File(this.akkaConfig); + try(FileReader fileReader = new FileReader(akkaConfigFile); + BufferedReader bufferedReader = new BufferedReader(fileReader)) { + String line; + while((line = bufferedReader.readLine()) != null) { + if(line.contains("seed-nodes =")) { + parseSeedNodes(line); + break; + } + } + } + } catch(IOException e) { + log.error("defineMembers(): Couldn't load akka", e); + } catch(NullPointerException e) { + log.error("defineMembers(): akkaConfig is null. Check properties file and restart {} bundle.", APP_NAME); + log.error("defineMembers(): NullPointerException", e); + } + log.info("self:\n{}", self); + } + + /** + * Sets up the {@code InstanceIdentifier}s for Data Store transactions. + */ + private void createContainers() { + // Replace with MD-SAL write for FailoverStatus + } + + /** + * Shuts down the {@code ExecutorService} and closes the RPC Provider Registry. + */ + @Override + public void close() throws Exception { + log.info("Closing provider for {}", APP_NAME); + executor.shutdown(); + rpcRegistration.close(); + log.info("close(): Successfully closed provider for {}", APP_NAME); + } + + /** + * Listens for changes to the Data tree. + * + * @param changes Data tree changes. + */ + @Override + public void onDataTreeChanged(@Nonnull Collection changes) { + log.info("onDataTreeChanged(): No changes."); + } + + /** + * Makes a call to {@code resolver.getClusterHealth()} to determine the + * health of the akka clustered controllers. + * + * @param input request body adhering to the model for + * {@code ClusterHealthInput} + * @return response adhering to the model for {@code ClusterHealthOutput} + * @see HealthResolver + * @see ClusterHealthInput + * @see ClusterHealthOutput + */ + @Override + public ListenableFuture<RpcResult<ClusterHealthOutput>> clusterHealth(ClusterHealthInput input) { + log.info("{}:cluster-health invoked.", APP_NAME); + resolver.getClusterHealth(); + return buildClusterHealthOutput(); + } + + /** + * Makes a call to {@code resolver.getSiteHealth()} to determine the health + * of all of the application components of a site. In a multi-site config, + * this will gather the health of all sites. + * + * @param input request body adhering to the model for + * {@code SiteHealthInput} + * @return response adhering to the model for {@code SiteHealthOutput} + * @see HealthResolver + * @see SiteHealthInput + * @see SiteHealthOutput + */ + @Override + public ListenableFuture<RpcResult<SiteHealthOutput>> siteHealth(SiteHealthInput input) { + log.info("{}:site-health invoked.", APP_NAME); + List<SiteHealth> sites = resolver.getSiteHealth(); + return buildSiteHealthOutput(sites); + } + + /** + * Makes a call to {@code resolver.getDatabaseHealth()} to determine the + * health of the database(s) used by the controller. + * + * @param input request body adhering to the model for + * {@code DatabaseHealthInput} + * @return response adhering to the model for {@code DatabaseHealthOutput} + * @see HealthResolver + * @see DatabaseHealthInput + * @see DatabaseHealthOutput + */ + @Override + public ListenableFuture<RpcResult<DatabaseHealthOutput>> databaseHealth(DatabaseHealthInput input) { + log.info("{}:database-health invoked.", APP_NAME); + DatabaseHealthOutputBuilder outputBuilder = new DatabaseHealthOutputBuilder(); + DatabaseHealth health = resolver.getDatabaseHealth(); + outputBuilder.setStatus(health.getHealth().equals(Health.HEALTHY) ? "200" : "500"); + outputBuilder.setHealth(health.getHealth().toString()); + outputBuilder.setServedBy(member); + log.info("databaseHealth(): Health: {}", health.getHealth()); + return Futures.immediateFuture(RpcResultBuilder.<DatabaseHealthOutput>status(true).withResult(outputBuilder.build()).build()); + } + + /** + * Makes a call to {@code resolver.getAdminHealth()} to determine the + * health of the administrative portal(s) used by the controller. + * + * @param input request body adhering to the model for + * {@code AdminHealthInput} + * @return response adhering to the model for {@code AdminHealthOutput} + * @see HealthResolver + * @see AdminHealthInput + * @see AdminHealthOutput + */ + @Override + public ListenableFuture<RpcResult<AdminHealthOutput>> adminHealth(AdminHealthInput input) { + log.info("{}:admin-health invoked.", APP_NAME); + AdminHealthOutputBuilder outputBuilder = new AdminHealthOutputBuilder(); + AdminHealth adminHealth = resolver.getAdminHealth(); + outputBuilder.setStatus(Integer.toString(adminHealth.getStatusCode())); + outputBuilder.setHealth(adminHealth.getHealth().toString()); + outputBuilder.setServedBy(member); + log.info("adminHealth(): Status: {} | Health: {}", adminHealth.getStatusCode(), adminHealth.getHealth()); + return Futures.immediateFuture(RpcResultBuilder.<AdminHealthOutput>status(true).withResult(outputBuilder.build()).build()); + } + + /** + * Places IP Tables rules in place to drop akka communications traffic with + * one or mode nodes. This method does not not perform any checks to see if + * rules currently exist, and assumes success. + * + * @param input request body adhering to the model for + * {@code HaltAkkaTrafficInput} + * @return response adhering to the model for {@code HaltAkkaTrafficOutput} + * @see HaltAkkaTrafficInput + * @see HaltAkkaTrafficOutput + */ + @Override + public ListenableFuture<RpcResult<HaltAkkaTrafficOutput>> haltAkkaTraffic(HaltAkkaTrafficInput input) { + log.info("{}:halt-akka-traffic invoked.", APP_NAME); + HaltAkkaTrafficOutputBuilder outputBuilder = new HaltAkkaTrafficOutputBuilder(); + outputBuilder.setStatus("200"); + modifyIpTables(IpTables.ADD, input.getNodeInfo().toArray()); + outputBuilder.setServedBy(member); + + return Futures.immediateFuture(RpcResultBuilder.<HaltAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build()); + } + + /** + * Removes IP Tables rules in place to permit akka communications traffic + * with one or mode nodes. This method does not not perform any checks to + * see if rules currently exist, and assumes success. + * + * @param input request body adhering to the model for + * {@code ResumeAkkaTrafficInput} + * @return response adhering to the model for {@code ResumeAkkaTrafficOutput} + * @see ResumeAkkaTrafficInput + * @see ResumeAkkaTrafficOutput + */ + @Override + public ListenableFuture<RpcResult<ResumeAkkaTrafficOutput>> resumeAkkaTraffic(ResumeAkkaTrafficInput input) { + log.info("{}:resume-akka-traffic invoked.", APP_NAME); + ResumeAkkaTrafficOutputBuilder outputBuilder = new ResumeAkkaTrafficOutputBuilder(); + outputBuilder.setStatus("200"); + modifyIpTables(IpTables.DELETE, input.getNodeInfo().toArray()); + outputBuilder.setServedBy(member); + + return Futures.immediateFuture(RpcResultBuilder.<ResumeAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build()); + } + + /** + * Returns a canned response containing the identifier for this + * controller's site. + * + * @param input request body adhering to the model for + * {@code SiteIdentifierInput} + * @return response adhering to the model for {@code SiteIdentifierOutput} + * @see SiteIdentifierInput + * @see SiteIdentifierOutput + */ + @Override + public ListenableFuture<RpcResult<SiteIdentifierOutput>> siteIdentifier(SiteIdentifierInput input) { + log.info("{}:site-identifier invoked.", APP_NAME); + SiteIdentifierOutputBuilder outputBuilder = new SiteIdentifierOutputBuilder(); + outputBuilder.setStatus("200"); + outputBuilder.setId(siteIdentifier); + outputBuilder.setServedBy(member); + return Futures.immediateFuture(RpcResultBuilder.<SiteIdentifierOutput>status(true).withResult(outputBuilder.build()).build()); + } + + /** + * Makes a call to {@code resolver.tryFailover()} to try a failover defined + * by the active {@code HealthResolver}. + * + * @param input request body adhering to the model for + * {@code FailoverInput} + * @return response adhering to the model for {@code FailoverOutput} + * @see HealthResolver + * @see FailoverInput + * @see FailoverOutput + */ + @Override + public ListenableFuture<RpcResult<FailoverOutput>> failover(FailoverInput input) { + log.info("{}:failover invoked.", APP_NAME); + FailoverOutputBuilder outputBuilder = new FailoverOutputBuilder(); + FailoverStatus failoverStatus = resolver.tryFailover(input); + outputBuilder.setServedBy(member); + outputBuilder.setMessage(failoverStatus.getMessage()); + outputBuilder.setStatus(Integer.toString(failoverStatus.getStatusCode())); + log.info("{}:{}.", APP_NAME, failoverStatus.getMessage()); + return Futures.immediateFuture(RpcResultBuilder.<FailoverOutput>status(true).withResult(outputBuilder.build()).build()); + } + + /** + * Performs an akka traffic isolation of the active site from the standby + * site in an Active/Standby architecture. Invokes the + * {@code halt-akka-traffic} RPC against the standby site nodes using the + * information of the active site nodes. + * + * @param activeSite list of nodes in the active site + * @param standbySite list of nodes in the standby site + * @param port http or https port of the controller + * @deprecated No longer used since the refactor to use the HealthResolver + * pattern. Retained so the logic can be replicated later. + */ + @Deprecated + private void isolateSiteFromCluster(ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) { + log.info("isolateSiteFromCluster(): Halting Akka traffic..."); + for(ClusterActor actor : standbySite) { + try { + log.info("Halting Akka traffic for: {}", actor.getNode()); + // Build JSON with activeSite actor Node and actor AkkaPort + JSONObject akkaInput = new JSONObject(); + JSONObject inputBlock = new JSONObject(); + JSONArray votingStateArray = new JSONArray(); + JSONObject nodeInfo; + for(ClusterActor node : activeSite) { + nodeInfo = new JSONObject(); + nodeInfo.put("node", node.getNode()); + nodeInfo.put("port", node.getAkkaPort()); + votingStateArray.put(nodeInfo); + } + inputBlock.put("node-info", votingStateArray); + akkaInput.put("input", inputBlock); + ConnectionResponse response = ConnectionManager.getConnectionResponse(httpProtocol + actor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:halt-akka-traffic", ConnectionManager.HttpMethod.POST, akkaInput.toString(), ""); + } catch(IOException e) { + log.error("isolateSiteFromCluster(): Could not halt Akka traffic for: " + actor.getNode(), e); + } + } + } + + /** + * Invokes the down unreachable action through the Jolokia mbean API. + * + * @param activeSite list of nodes in the active site + * @param standbySite list of nodes in the standby site + * @param port http or https port of the controller + * @deprecated No longer used since the refactor to use the HealthResolver + * pattern. Retained so the logic can be replicated later. + */ + @Deprecated + private void downUnreachableNodes(ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) { + log.info("downUnreachableNodes(): Setting site unreachable..."); + JSONObject jolokiaInput = new JSONObject(); + jolokiaInput.put("type", "EXEC"); + jolokiaInput.put("mbean", "akka:type=Cluster"); + jolokiaInput.put("operation", "down"); + JSONArray arguments = new JSONArray(); + for(ClusterActor actor : activeSite) { + // Build Jolokia input + // May need to change from akka port to actor.getAkkaPort() + arguments.put("akka.tcp://opendaylight-cluster-data@" + actor.getNode() + ":" + properties.getProperty(PropertyKeys.CONTROLLER_PORT_AKKA)); + } + jolokiaInput.put("arguments", arguments); + log.debug("downUnreachableNodes(): {}", jolokiaInput); + try { + log.info("downUnreachableNodes(): Setting nodes unreachable"); + ConnectionResponse response = ConnectionManager.getConnectionResponse(httpProtocol + standbySite.get(0).getNode() + ":" + port + "/jolokia", ConnectionManager.HttpMethod.POST, jolokiaInput.toString(), ""); + } catch(IOException e) { + log.error("downUnreachableNodes(): Error setting nodes unreachable", e); + } + } + + /** + * Triggers a data backup and export sequence of MD-SAL data. Invokes the + * {@code data-export-import:schedule-export} RPC to schedule a data export + * and subsequently the {@code daexim-offsite-backup:backup-data} RPC + * against the active site to export and backup the data. Assumes the + * controllers have the org.onap.ccsdk.sli.northbound.daeximoffsitebackup + * bundle installed. + * + * @param activeSite list of nodes in the active site + * @param port http or https port of the controller + * @deprecated No longer used since the refactor to use the HealthResolver + * pattern. Retained so the logic can be replicated later. + */ + @Deprecated + private void backupMdSal(ArrayList<ClusterActor> activeSite, String port) { + log.info("backupMdSal(): Backing up data..."); + try { + log.info("backupMdSal(): Scheduling backup for: {}", activeSite.get(0).getNode()); + ConnectionResponse response = ConnectionManager.getConnectionResponse(httpProtocol + activeSite.get(0).getNode() + ":" + port + "/restconf/operations/data-export-import:schedule-export", ConnectionManager.HttpMethod.POST, "{ \"input\": { \"run-at\": \"30\" } }", ""); + } catch(IOException e) { + log.error("backupMdSal(): Error backing up MD-SAL", e); + } + for(ClusterActor actor : activeSite) { + try { + // Move data offsite + log.info("backupMdSal(): Backing up data for: {}", actor.getNode()); + ConnectionResponse response = ConnectionManager.getConnectionResponse(httpProtocol + actor.getNode() + ":" + port + "/restconf/operations/daexim-offsite-backup:backup-data", ConnectionManager.HttpMethod.POST, null, ""); + } catch(IOException e) { + log.error("backupMdSal(): Error backing up data.", e); + } + } + } + + /** + * Builds a response object for {@code clusterHealth()}. Sorts and iterates + * over the contents of the {@code memberMap}, which contains the health + * information of the cluster, and adds them to the {@code outputBuilder}. + * If the ClusterActor is healthy, according to + * {@code resolver.isControllerHealthy()}, the {@code ClusterHealthOutput} + * status has a {@code 0} appended, otherwise a {@code 1} is appended. A + * status of all zeroes denotes a healthy cluster. This status should be + * easily decoded by tools which use the output. + * + * @return future containing a completed {@code ClusterHealthOutput} + * @see ClusterActor + * @see ClusterHealthOutput + * @see HealthResolver + */ + @SuppressWarnings("unchecked") + private ListenableFuture<RpcResult<ClusterHealthOutput>> buildClusterHealthOutput() { + ClusterHealthOutputBuilder outputBuilder = new ClusterHealthOutputBuilder(); + outputBuilder.setServedBy(member); + List memberList = new ArrayList<Member>(); + StringBuilder stat = new StringBuilder(); + memberMap.values() + .stream() + .sorted(Comparator.comparingInt(member -> Integer.parseInt(member.getMember().split("-")[1]))) + .forEach(member -> { + memberList.add(new MemberBuilder(member).build()); + // 0 is a healthy controller, 1 is unhealthy. + // The list is sorted so users can decode to find unhealthy nodes + // This will also let them figure out health on a per-site basis + // Depending on any tools they use with this API + if(resolver.isControllerHealthy(member)) { + stat.append("0"); + } else { + stat.append("1"); + } + }); + outputBuilder.setStatus(stat.toString()); + outputBuilder.setMembers(memberList); + RpcResult<ClusterHealthOutput> rpcResult = RpcResultBuilder.<ClusterHealthOutput>status(true).withResult(outputBuilder.build()).build(); + return Futures.immediateFuture(rpcResult); + } + + /** + * Builds a response object for {@code siteHealth()}. Iterates over a list + * of {@code SiteHealth} objects and populates the {@code SiteHealthOutput} + * with the information. + * + * @param sites list of sites + * @return future containing a completed {@code SiteHealthOutput} + * @see SiteHealth + * @see HealthResolver + */ + @SuppressWarnings("unchecked") + private ListenableFuture<RpcResult<SiteHealthOutput>> buildSiteHealthOutput(List<SiteHealth> sites) { + SiteHealthOutputBuilder outputBuilder = new SiteHealthOutputBuilder(); + SitesBuilder siteBuilder = new SitesBuilder(); + outputBuilder.setStatus("200"); + outputBuilder.setSites((List) new ArrayList<Site>()); + + for(SiteHealth site : sites) { + siteBuilder.setHealth(site.getHealth().toString()); + siteBuilder.setRole(site.getRole()); + siteBuilder.setId(site.getId()); + outputBuilder.getSites().add(siteBuilder.build()); + log.info("buildSiteHealthOutput(): Health for {}: {}", site.getId(), site.getHealth().getHealth()); + } + + outputBuilder.setServedBy(member); + RpcResult<SiteHealthOutput> rpcResult = RpcResultBuilder.<SiteHealthOutput>status(true).withResult(outputBuilder.build()).build(); + return Futures.immediateFuture(rpcResult); + } + + /** + * Parses a line containing the akka networking information of the akka + * controller cluster. Assumes entries of the format: + * <p> + * akka.tcp://opendaylight-cluster-data@<FQDN>:<AKKA_PORT> + * <p> + * The information is stored in a {@code ClusterActor} object, and then + * added to the memberMap HashMap, with the {@code FQDN} as the key. The + * final step is a call to {@code createHealthResolver} to create the + * health resolver for the provider. + * + * @param line the line containing all of the seed nodes + * @see ClusterActor + * @see HealthResolver + */ + private void parseSeedNodes(String line) { + memberMap = new HashMap<>(); + line = line.substring(line.indexOf("[\""), line.indexOf(']')); + String[] splits = line.split(","); + + for(int ndx = 0; ndx < splits.length; ndx++) { + String nodeName = splits[ndx]; + int delimLocation = nodeName.indexOf('@'); + String port = nodeName.substring(splits[ndx].indexOf(':', delimLocation) + 1, splits[ndx].indexOf('"', splits[ndx].indexOf(':'))); + splits[ndx] = nodeName.substring(delimLocation + 1, splits[ndx].indexOf(':', delimLocation)); + log.info("parseSeedNodes(): Adding node: {}:{}", splits[ndx], port); + ClusterActor clusterActor = new ClusterActor(); + clusterActor.setNode(splits[ndx]); + clusterActor.setAkkaPort(port); + clusterActor.setMember("member-" + (ndx + 1)); + if(member.equals(clusterActor.getMember())) { + self = clusterActor; + } + memberMap.put(clusterActor.getNode(), clusterActor); + log.info("parseSeedNodes(): {}", clusterActor); + } + + createHealthResolver(); + } + + /** + * Creates the specific health resolver requested by the user, as specified + * in the gr-toolkit.properties file. If a resolver is not specified, or + * there is an issue creating the resolver, it will use a fallback resolver + * based on how many nodes are added to the memberMap HashMap. + * + * @see HealthResolver + * @see SingleNodeHealthResolver + * @see ThreeNodeHealthResolver + * @see SixNodeHealthResolver + */ + private void createHealthResolver() { + log.info("createHealthResolver(): Creating health resolver..."); + try { + Class resolverClass = null; + String userDefinedResolver = properties.getProperty(PropertyKeys.RESOLVER); + if(StringUtils.isEmpty(userDefinedResolver)) { + throw new InstantiationException(); + } + resolverClass = Class.forName(userDefinedResolver); + Class[] types = { Map.class , properties.getClass(), DbLibService.class }; + Constructor<HealthResolver> constructor = resolverClass.getConstructor(types); + Object[] parameters = { memberMap, properties, dbLib }; + resolver = constructor.newInstance(parameters); + log.info("createHealthResolver(): Created resolver from name {}", resolver.toString()); + } catch(ClassNotFoundException | InstantiationException | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { + log.warn("createHealthResolver(): Could not create user defined resolver", e); + if(memberMap.size() == 1) { + log.info("createHealthResolver(): FALLBACK: Initializing SingleNodeHealthResolver..."); + resolver = new SingleNodeHealthResolver(memberMap, properties, dbLib); + } else if(memberMap.size() == 3) { + log.info("createHealthResolver(): FALLBACK: Initializing ThreeNodeHealthResolver..."); + resolver = new ThreeNodeHealthResolver(memberMap, properties, dbLib); + } else if(memberMap.size() == 6) { + log.info("createHealthResolver(): FALLBACK: Initializing SixNodeHealthResolver..."); + resolver = new SixNodeHealthResolver(memberMap, properties, dbLib); + } + } + } + + /** + * Adds or drops IPTables rules to block or resume akka traffic for a node + * in the akka cluster. Assumes that the user or group that the controller + * is run as has the ability to run sudo /sbin/iptables without requiring a + * password. This method will run indefinitely if that assumption is not + * correct. This method does not check to see if any rules around the node + * are preexisting, so multiple uses will result in multiple additions and + * removals from IPTables. + * + * @param task the operation to be performed against IPTables + * @param nodeInfo array containing the nodes to be added or dropped from + * IPTables + */ + private void modifyIpTables(IpTables task, Object[] nodeInfo) { + log.info("modifyIpTables(): Modifying IPTables rules..."); + if(task == IpTables.ADD) { + for(Object node : nodeInfo) { + org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.halt.akka.traffic.input.NodeInfo n = + (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.halt.akka.traffic.input.NodeInfo) node; + log.info("modifyIpTables(): Isolating {}", n.getNode()); + executeCommand(String.format("sudo /sbin/iptables -A INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode())); + executeCommand(String.format("sudo /sbin/iptables -A OUTPUT -p tcp --destination-port %s -j DROP -d %s", n.getPort(), n.getNode())); + } + } else if(task == IpTables.DELETE) { + for(Object node : nodeInfo) { + org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.resume.akka.traffic.input.NodeInfo n = + (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.resume.akka.traffic.input.NodeInfo) node; + log.info("modifyIpTables(): De-isolating {}", n.getNode()); + executeCommand(String.format("sudo /sbin/iptables -D INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode())); + executeCommand(String.format("sudo /sbin/iptables -D OUTPUT -p tcp --destination-port %s -j DROP -d %s", n.getPort(), n.getNode())); + } + } + if(nodeInfo.length > 0) { + executeCommand("sudo /sbin/iptables -L"); + } + } + + /** + * Opens a shell session and executes a command. + * + * @param command the shell command to execute + */ + private void executeCommand(String command) { + log.info("executeCommand(): Executing command: {}", command); + String[] cmd = command.split(" "); + try { + Process p = Runtime.getRuntime().exec(cmd); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(p.getInputStream())); + String inputLine; + StringBuilder content = new StringBuilder(); + while((inputLine = bufferedReader.readLine()) != null) { + content.append(inputLine); + } + bufferedReader.close(); + log.info("executeCommand(): {}", content); + } catch(IOException e) { + log.error("executeCommand(): Error executing command", e); + } + } + + /** + * The IPTables operations this module can perform. + */ + enum IpTables { + ADD, + DELETE + } +}
\ No newline at end of file diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionManager.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionManager.java new file mode 100644 index 000000000..99fcd3e04 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionManager.java @@ -0,0 +1,157 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.connection; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; + +/** + * Handles the process for getting HTTP connections to resources. Has the + * ability to send JSON payloads. Only supports basic authorization when + * sending credentials. + * + * @author Anthony Haddox + * @see ConnectionResponse + */ +public interface ConnectionManager { + Logger log = LoggerFactory.getLogger(ConnectionManager.class); + int CONNECTION_TIMEOUT = 5000; // 5 second timeout + enum HttpMethod { + GET("GET"), + POST("POST"); + + private final String method; + HttpMethod(String method) { + this.method = method; + } + String getMethod() { + return method; + } + } + + /** + * Writes a JSON payload to an {@code HTTPURLConnection OutputStream}. + * + * @param input the JSON payload to send + * @param connection the {@code HTTPURLConnection} to write to + * @throws IOException if there is a problem writing to the output stream + */ + static void sendPayload(String input, HttpURLConnection connection) throws IOException { + byte[] out = input.getBytes(StandardCharsets.UTF_8); + int length = out.length; + + connection.setFixedLengthStreamingMode(length); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.connect(); + try(OutputStream os = connection.getOutputStream()) { + os.write(out); + } + } + + /** + * Gets an {@code HTTPURLConnection} to a {@code host}. + * + * @param host the host to connect to + * @return an {@code HTTPURLConnection} + * @throws IOException if a connection cannot be opened + */ + static HttpURLConnection getConnection(String host) throws IOException { + log.info("getConnection(): Getting connection to: {}", host); + URL url = new URL(host); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Connection", "keep-alive"); + connection.setRequestProperty("Proxy-Connection", "keep-alive"); + connection.setConnectTimeout(CONNECTION_TIMEOUT); + connection.setReadTimeout(CONNECTION_TIMEOUT); + return connection; + } + + /** + * Gets an {@code HTTPURLConnection} to a {@code host} and sets the + * Authorization header with the supplied credentials. Only supports basic + * authentication. + * + * @param host the host to connect to + * @param credentials the authorization credentials + * @return an {@code HTTPURLConnection} with Authorization header set + * @throws IOException if a connection cannot be opened + */ + static HttpURLConnection getConnection(String host, String credentials) throws IOException { + String auth = "Basic " + javax.xml.bind.DatatypeConverter.printBase64Binary(credentials.getBytes()); + HttpURLConnection connection = getConnection(host); + connection.addRequestProperty("Authorization", auth); + credentials = null; + auth = null; + return connection; + } + + /** + * Opens a connection to a path, sends a payload (if supplied with one), + * and returns the response. + * @param path the host to connect to + * @param method the {@code HttpMethod} to use + * @param input the payload to send + * @param credentials the credentials to use + * @return a {@code ConnectionResponse} containing the response body and + * status code of the operation + * @throws IOException if a connection cannot be opened or if the payload + * cannot be sent + * @see HttpMethod + */ + static ConnectionResponse getConnectionResponse(String path, HttpMethod method, String input, String credentials) throws IOException { + HttpURLConnection connection = (StringUtils.isEmpty(credentials)) ? getConnection(path) : getConnection(path, credentials); + credentials = null; + connection.setRequestMethod(method.getMethod()); + connection.setDoInput(true); + + if(!StringUtils.isEmpty(input)) { + sendPayload(input, connection); + } + + StringBuilder content = new StringBuilder(); + try(BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + String inputLine; + while((inputLine = bufferedReader.readLine()) != null) { + content.append(inputLine); + } + } finally { + connection.disconnect(); + } + + ConnectionResponse connectionResponse = new ConnectionResponse(); + connectionResponse.content = content.toString(); + connectionResponse.statusCode = connection.getResponseCode(); + log.info("getConnectionResponse(): {} response code from {}", connectionResponse.statusCode, path); + log.debug("getConnectionResponse(): Response:\n{}", connectionResponse.content); + return connectionResponse; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionResponse.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionResponse.java new file mode 100644 index 000000000..fb16d2a12 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/connection/ConnectionResponse.java @@ -0,0 +1,43 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.connection; + +/** + * A data container for HTTP connection requests. + * + * @author Anthony Haddox + * @see ConnectionManager + */ +public class ConnectionResponse { + public int statusCode; + public String content; + + public ConnectionResponse withStatusCode(int statusCode) { + this.statusCode = statusCode; + return this; + } + + @Override + public String toString() { + return "Status: " + statusCode + "\nContent: " + content; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/AdminHealth.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/AdminHealth.java new file mode 100644 index 000000000..0e88df55e --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/AdminHealth.java @@ -0,0 +1,58 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +/** + * A data container for Admin health. + * + * @author Anthony Haddox + * @see org.onap.ccsdk.sli.plugins.grtoolkit.resolver.HealthResolver + */ +public class AdminHealth { + private Health health; + private int statusCode; + + public AdminHealth(Health health) { + this.health = health; + } + + public AdminHealth(Health health, int statusCode) { + this.health = health; + this.statusCode = statusCode; + } + + public Health getHealth() { + return health; + } + + public void setHealth(Health health) { + this.health = health; + } + + public int getStatusCode() { + return statusCode; + } + + public void setStatusCode(int statusCode) { + this.statusCode = statusCode; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterActor.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterActor.java new file mode 100644 index 000000000..d039c865e --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterActor.java @@ -0,0 +1,222 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2018 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A data container with information about an actor in the Akka cluster. + * + * @author Anthony Haddox + */ +public class ClusterActor { + private String node; + private String member; + private String site; + private String akkaPort; + private boolean voting; + private boolean up; + private boolean unreachable; + private ArrayList<String> shardLeader; + private ArrayList<String> replicaShards; + private ArrayList<String> nonReplicaShards; + private HashMap<String, Integer> commits; + + public static final String SITE_1 = "Site 1"; + public static final String SITE_2 = "Site 2"; + + public ClusterActor() { + node = ""; + member = ""; + site = ""; + voting = false; + up = false; + unreachable = false; + shardLeader = new ArrayList<>(); + replicaShards = new ArrayList<>(); + nonReplicaShards = new ArrayList<>(); + commits = new HashMap<>(); + } + + public String getNode() { + return node; + } + + public void setNode(String node) { + this.node = node; + } + + public String getMember() { + return member; + } + + public void setMember(String member) { + this.member = member; + } + + public String getSite() { + return site; + } + + public void setSite(String site) { + this.site = site; + } + + public String getAkkaPort() { + return akkaPort; + } + + public void setAkkaPort(String akkaPort) { + this.akkaPort = akkaPort; + } + + public boolean isVoting() { + return voting; + } + + public void setVoting(boolean voting) { + this.voting = voting; + } + + public boolean isUp() { + return up; + } + + public void setUp(boolean up) { + this.up = up; + } + + public boolean isUnreachable() { + return unreachable; + } + + public void setUnreachable(boolean unreachable) { + this.unreachable = unreachable; + } + + public List<String> getShardLeader() { + return shardLeader; + } + + public void setShardLeader(List<String> shardLeader) { + this.shardLeader = (ArrayList<String>) shardLeader; + } + + public List<String> getReplicaShards() { + return replicaShards; + } + + public void setReplicaShards(List<String> replicaShards) { + this.replicaShards = (ArrayList<String>) replicaShards; + } + + public List<String> getNonReplicaShards() { + return nonReplicaShards; + } + + public void setNonReplicaShards(List<String> nonReplicaShards) { + this.nonReplicaShards = (ArrayList<String>) nonReplicaShards; + } + + public Map<String, Integer> getCommits() { + return commits; + } + + public void setCommits(Map<String, Integer> commits) { + this.commits = (HashMap<String, Integer>) commits; + } + + public void flush() { + shardLeader.clear(); + replicaShards.clear(); + nonReplicaShards.clear(); + commits.clear(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("[ "); + builder.append(this.member); + builder.append(" ] "); + + builder.append(this.node); + builder.append(":"); + builder.append(this.akkaPort); + builder.append(" is"); + if(up) + builder.append(" Up"); + else + builder.append(" Down"); + if(unreachable) { + builder.append(" [ UNREACHABLE ]"); + return builder.toString(); + } + + if(voting) + builder.append(" (Voting)"); + + builder.append("\n"); + + for(String l : this.shardLeader) { + builder.append("\tLeader: "); + builder.append(l); + builder.append("\n"); + } + + for(String r : this.replicaShards) { + builder.append("\tReplicating: "); + builder.append(r); + builder.append("\n"); + } + + for(String n : this.nonReplicaShards) { + builder.append("\tNot replicating: "); + builder.append(n); + builder.append("\n"); + } + + for(Map.Entry<String, Integer> entry : commits.entrySet()) { + String key = entry.getKey(); + int value = entry.getValue(); + if(value > 0) { + builder.append("\t"); + builder.append(value); + builder.append(" commits ahead of "); + builder.append(key); + builder.append("\n"); + } + else if(value < 0) { + builder.append("\t"); + builder.append(value); + builder.append(" commits behind "); + builder.append(key); + builder.append("\n"); + } + } + + return builder.toString(); + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterHealth.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterHealth.java new file mode 100644 index 000000000..52198225d --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/ClusterHealth.java @@ -0,0 +1,49 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +/** + * A data container for Akka Cluster health. + * + * @author Anthony Haddox + * @see org.onap.ccsdk.sli.plugins.grtoolkit.resolver.HealthResolver + */ +public class ClusterHealth { + private Health health; + + public ClusterHealth() { + health = Health.FAULTY; + } + + public ClusterHealth withHealth(Health h) { + this.health = h; + return this; + } + + public Health getHealth() { + return health; + } + + public void setHealth(Health health) { + this.health = health; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/DatabaseHealth.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/DatabaseHealth.java new file mode 100644 index 000000000..14c11b51a --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/DatabaseHealth.java @@ -0,0 +1,44 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +/** + * A data container for Database health. + * + * @author Anthony Haddox + * @see org.onap.ccsdk.sli.plugins.grtoolkit.resolver.HealthResolver + */ +public class DatabaseHealth { + private Health health; + + public DatabaseHealth(Health health) { + this.health = health; + } + + public Health getHealth() { + return health; + } + + public void setHealth(Health health) { + this.health = health; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/FailoverStatus.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/FailoverStatus.java new file mode 100644 index 000000000..7366ede43 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/FailoverStatus.java @@ -0,0 +1,64 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +/** + * A data container for the status of a controller-level failover. + * + * @author Anthony Haddox + * @see org.onap.ccsdk.sli.plugins.grtoolkit.resolver.HealthResolver + */ +public class FailoverStatus { + private int statusCode; + private String message; + + public FailoverStatus() { + this.statusCode = 200; + this.message = "Failover complete."; + } + + public FailoverStatus withStatusCode(int code) { + this.statusCode = code; + return this; + } + + public FailoverStatus withMessage(String message) { + this.message = message; + return this; + } + + public int getStatusCode() { + return statusCode; + } + + public void setStatusCode(int statusCode) { + this.statusCode = statusCode; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/Health.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/Health.java new file mode 100644 index 000000000..deb5cb444 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/Health.java @@ -0,0 +1,40 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +/** + * Potential health values. + * + * @author Anthony Haddox + */ +public enum Health { + HEALTHY("HEALTHY"), + FAULTY("FAULTY"); + + private final String health; + Health(String health) { + this.health = health; + } + public String getHealth() { + return health; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/MemberBuilder.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/MemberBuilder.java new file mode 100644 index 000000000..863b56674 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/MemberBuilder.java @@ -0,0 +1,88 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2018 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.cluster.health.output.MembersBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.member.CommitStatusBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.member.ReplicasBuilder; +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.member.LeaderBuilder; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Extends the {@code MembersBuilder} generated from the gr-toolkit.yang model. + * Uses information from a {@code ClusterActor} to populate the builder fields. + * + * @author Anthony Haddox + * @see ClusterActor + * @see org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.cluster.health.output.MembersBuilder + */ +public class MemberBuilder extends MembersBuilder { + public MemberBuilder(ClusterActor actor) { + super(); + this.setAddress(actor.getNode()); + this.setRole(actor.getMember()); + this.setVoting(actor.isVoting()); + this.setUp(actor.isUp()); + this.setUnreachable(actor.isUnreachable()); + populateReplicas(actor.getReplicaShards()); + populateCommits(actor.getCommits()); + populateLeader(actor.getShardLeader()); + } + + private void populateLeader(List<String> shardLeader) { + LeaderBuilder builder; + this.setLeader(new ArrayList<>()); + for(String leader : shardLeader) { + builder = new LeaderBuilder(); + builder.setShard(leader); + this.getLeader().add(builder.build()); + } + } + + private void populateCommits(Map<String, Integer> commits) { + CommitStatusBuilder builder; + this.setCommitStatus(new ArrayList<>()); + for(Map.Entry<String, Integer> entry : commits.entrySet()) { + String key = entry.getKey(); + Integer value = entry.getValue(); + if(value != 0) { + builder = new CommitStatusBuilder(); + builder.setShard(key); + builder.setDelta(value); + this.getCommitStatus().add(builder.build()); + } + } + } + + private void populateReplicas(List<String> replicaShards) { + ReplicasBuilder builder; + this.setReplicas(new ArrayList<>()); + for(String shard : replicaShards) { + builder = new ReplicasBuilder(); + builder.setShard(shard); + this.getReplicas().add(builder.build()); + } + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/PropertyKeys.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/PropertyKeys.java new file mode 100644 index 000000000..f2ad90a7f --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/PropertyKeys.java @@ -0,0 +1,40 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +public interface PropertyKeys { + String RESOLVER = "resolver"; + String SITE_IDENTIFIER = "site.identifier"; + String CONTROLLER_USE_SSL = "controller.useSsl"; + String CONTROLLER_PORT_SSL = "controller.port.ssl"; + String CONTROLLER_PORT_HTTP = "controller.port.http"; + String CONTROLLER_PORT_AKKA = "controller.port.akka"; + String CONTROLLER_CREDENTIALS = "controller.credentials"; + String AKKA_CONF_LOCATION = "akka.conf.location"; + String MBEAN_CLUSTER = "mbean.cluster"; + String MBEAN_SHARD_MANAGER = "mbean.shardManager"; + String MBEAN_SHARD_CONFIG = "mbean.shard.config"; + String ADM_USE_SSL = "adm.useSsl"; + String ADM_PORT_SSL = "adm.port.ssl"; + String ADM_PORT_HTTP = "adm.port.http"; + String ADM_FQDN = "adm.fqdn"; + String ADM_HEALTHCHECK= "adm.healthcheck"; +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/SiteHealth.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/SiteHealth.java new file mode 100644 index 000000000..8407032d5 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/data/SiteHealth.java @@ -0,0 +1,125 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.data; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + + +/** + * A data container for Site health. + * + * @author Anthony Haddox + * @see org.onap.ccsdk.sli.plugins.grtoolkit.resolver.HealthResolver + */ +public class SiteHealth { + private List<AdminHealth> adminHealth; + private List<DatabaseHealth> databaseHealth; + private List<ClusterHealth> clusterHealth; + + private Health health; + private String id; + private String role; + + public SiteHealth() { + adminHealth = new ArrayList<>(); + databaseHealth = new ArrayList<>(); + clusterHealth = new ArrayList<>(); + + // Faulty by default, it's up to the health check to affirm the health + health = Health.FAULTY; + } + + public SiteHealth withAdminHealth(AdminHealth... health) { + Collections.addAll(adminHealth, health); + return this; + } + + public SiteHealth withDatabaseHealth(DatabaseHealth... health) { + Collections.addAll(databaseHealth, health); + return this; + } + + public SiteHealth withClusterHealth(ClusterHealth... health) { + Collections.addAll(clusterHealth, health); + return this; + } + + public SiteHealth withId(String id) { + this.id = id; + return this; + } + + public SiteHealth withRole(String role) { + this.role = role; + return this; + } + + public Health getHealth() { + return health; + } + + public void setHealth(Health health) { + this.health = health; + } + + public List<AdminHealth> getAdminHealth() { + return adminHealth; + } + + public void setAdminHealth(List<AdminHealth> adminHealth) { + this.adminHealth = adminHealth; + } + + public List<DatabaseHealth> getDatabaseHealth() { + return databaseHealth; + } + + public void setDatabaseHealth(List<DatabaseHealth> databaseHealth) { + this.databaseHealth = databaseHealth; + } + + public List<ClusterHealth> getClusterHealth() { + return clusterHealth; + } + + public void setClusterHealth(List<ClusterHealth> clusterHealth) { + this.clusterHealth = clusterHealth; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getRole() { + return role; + } + + public void setRole(String role) { + this.role = role; + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/HealthResolver.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/HealthResolver.java new file mode 100644 index 000000000..5c4ed13d7 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/HealthResolver.java @@ -0,0 +1,212 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.resolver; + +import org.json.JSONException; +import org.json.JSONObject; + +import org.onap.ccsdk.sli.core.dblib.DbLibService; +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionManager; +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionResponse; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.PropertyKeys; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth; + +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverInput; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.onap.ccsdk.sli.plugins.grtoolkit.data.Health.HEALTHY; + +/** + * Abstract class for the Health Resolver system, which allows for custom logic + * to be implemented, while leaving inputs/outputs generic and architecture + * agnostic. This class provides some simple implementations of both Admin and + * Database health checking, but leaves cluster and site health determinations + * up to the implementer. Useful implementation examples can be found in the + * {@code SingleNodeHealthResolver}, {@code ThreeNodeHealthResolver}, and + * {@code SixNodeHealthResolver} classes. + * + * @author Anthony Haddox + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see SingleNodeHealthResolver + * @see ThreeNodeHealthResolver + * @see SixNodeHealthResolver + */ +public abstract class HealthResolver { + private final Logger log = LoggerFactory.getLogger(HealthResolver.class); + static final String OUTPUT = "output"; + final String httpProtocol; + final String controllerPort; + final String credentials; + final Map<String, ClusterActor> memberMap; + private DbLibService dbLib; + final ShardResolver shardResolver; + private String adminPath; + private String siteIdentifier; + + /** + * Constructs the health resolver used by the {@code GrToolkitProvider} to + * determine the health of the application components. + * + * @param map a HashMap containing all of the nodes in the akka cluster + * @param properties the properties passed ino the provider + * @param dbLib a reference to the {@code DbLibService} of the provider + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + */ + HealthResolver(Map<String, ClusterActor> map, Properties properties, DbLibService dbLib) { + log.info("Creating {}", this.getClass().getCanonicalName()); + this.memberMap = map; + this.dbLib = dbLib; + shardResolver = ShardResolver.getInstance(properties); + + String adminProtocol = "true".equals(properties.getProperty(PropertyKeys.ADM_USE_SSL)) ? "https://" : "http://"; + String adminPort = "true".equals(properties.getProperty(PropertyKeys.ADM_USE_SSL)) ? properties.getProperty(PropertyKeys.ADM_PORT_SSL) : properties.getProperty(PropertyKeys.ADM_PORT_HTTP); + adminPath = adminProtocol + properties.getProperty(PropertyKeys.ADM_FQDN) + ":" + adminPort + properties.getProperty(PropertyKeys.ADM_HEALTHCHECK); + siteIdentifier = properties.getProperty(PropertyKeys.SITE_IDENTIFIER).trim(); + + controllerPort = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL).trim() : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP).trim(); + httpProtocol = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? "https://" : "http://"; + if(siteIdentifier == null || siteIdentifier.isEmpty()) { + siteIdentifier = properties.getProperty(PropertyKeys.SITE_IDENTIFIER).trim(); + } + credentials = properties.getProperty(PropertyKeys.CONTROLLER_CREDENTIALS).trim(); + } + + public abstract ClusterHealth getClusterHealth(); + public abstract List<SiteHealth> getSiteHealth(); + public abstract FailoverStatus tryFailover(FailoverInput input); + public abstract void resolveSites(); + + /** + * Gets a connection to the admin portal. If the status code is 200, the + * admin portal is assumed to be healthy. + * + * @return an {@code AdminHealth} object with health of the admin portal + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see AdminHealth + */ + public AdminHealth getAdminHealth() { + log.info("getAdminHealth(): Requesting health check from {}", adminPath); + try { + ConnectionResponse response = ConnectionManager.getConnectionResponse(adminPath, ConnectionManager.HttpMethod.GET, null, null); + Health health = (response.statusCode == 200) ? HEALTHY : Health.FAULTY; + AdminHealth adminHealth = new AdminHealth(health, response.statusCode); + log.info("getAdminHealth(): Response: {}", response); + return adminHealth; + } catch(IOException e) { + log.error("getAdminHealth(): Problem getting ADM health.", e); + return new AdminHealth(Health.FAULTY, 500); + } + } + + /** + * Uses {@code DbLibService} to get a connection to the database. If + * {@code DbLibService} is active and the connection it returns is not read + * only, the database(s) is assumed to be healthy. + * + * @return an {@code DatabaseHealth} object with health of the database + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see DatabaseHealth + */ + public DatabaseHealth getDatabaseHealth() { + log.info("getDatabaseHealth(): Determining database health..."); + try (Connection connection = dbLib.getConnection()){ + log.debug("getDatabaseHealth(): DBLib isActive(): {}", dbLib.isActive()); + log.debug("getDatabaseHealth(): DBLib isReadOnly(): {}", connection.isReadOnly()); + log.debug("getDatabaseHealth(): DBLib isClosed(): {}", connection.isClosed()); + if(!dbLib.isActive() || connection.isClosed() || connection.isReadOnly()) { + log.warn("getDatabaseHealth(): Database is FAULTY"); + return new DatabaseHealth(Health.FAULTY); + } + log.info("getDatabaseHealth(): Database is HEALTHY"); + } catch(SQLException e) { + log.error("getDatabaseHealth(): Database is FAULTY"); + log.error("getDatabaseHealth(): Error", e); + return new DatabaseHealth(Health.FAULTY); + } + + return new DatabaseHealth(HEALTHY); + } + + /** + * Utility method to see if an input is healthy. + * + * @return true if the input is healthy + * @see Health + */ + boolean isHealthy(Health h) { + return HEALTHY == h; + } + + public String getSiteIdentifier() { + return siteIdentifier; + } + + public void setSiteIdentifier(String siteIdentifier) { + this.siteIdentifier = siteIdentifier; + } + + /** + * Used to invoke the admin-health or database-health RPC to check if that + * component is healthy. + * + * @param path the path to the admin-health or database-health RPCs + * @return true if the component is healthy + * @throws IOException if a connection cannot be obtained + */ + boolean isRemoteComponentHealthy(String path) throws IOException { + String content = ConnectionManager.getConnectionResponse(path, ConnectionManager.HttpMethod.POST, null, credentials).content; + try { + JSONObject responseJson = new JSONObject(content); + JSONObject responseValue = responseJson.getJSONObject(OUTPUT); + return HEALTHY.toString().equals(responseValue.getString("health")); + } catch(JSONException e) { + log.error("Error parsing JSON", e); + throw new IOException(); + } + } + + /** + * Checks a {@code ClusterActor} object to see if the node is healthy. + * + * @param controller the controller to check + * @return true if the controller is up and reachable + * @see ClusterActor + */ + public boolean isControllerHealthy(ClusterActor controller) { + return (controller.isUp() && ! controller.isUnreachable()); + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ShardResolver.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ShardResolver.java new file mode 100644 index 000000000..8e96bff0c --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ShardResolver.java @@ -0,0 +1,177 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.resolver; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionManager; +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionResponse; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.PropertyKeys; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Used to perform operations on the data shard information returned as JSON + * from Jolokia. + * + * @author Anthony Haddox + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + */ +public class ShardResolver { + private final Logger log = LoggerFactory.getLogger(ShardResolver.class); + private static ShardResolver _shardResolver; + + private String jolokiaClusterPath; + private String shardManagerPath; + private String shardPathTemplate; + private String credentials; + private String httpProtocol; + + private static final String VALUE = "value"; + + private ShardResolver(Properties properties) { + String port = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL).trim() : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP).trim(); + httpProtocol = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? "https://" : "http://"; + jolokiaClusterPath = ":" + port + properties.getProperty(PropertyKeys.MBEAN_CLUSTER).trim(); + shardManagerPath = ":" + port + properties.getProperty(PropertyKeys.MBEAN_SHARD_MANAGER).trim(); + shardPathTemplate = ":" + port + properties.getProperty(PropertyKeys.MBEAN_SHARD_CONFIG).trim(); + credentials = properties.getProperty(PropertyKeys.CONTROLLER_CREDENTIALS).trim(); + } + + public static ShardResolver getInstance(Properties properties) { + if (_shardResolver == null) { + _shardResolver = new ShardResolver(properties); + } + return _shardResolver; + } + + private void getMemberStatus(ClusterActor clusterActor) throws IOException { + log.info("getMemberStatus(): Getting member status for {}", clusterActor.getNode()); + ConnectionResponse response = ConnectionManager.getConnectionResponse(httpProtocol + clusterActor.getNode() + jolokiaClusterPath, ConnectionManager.HttpMethod.GET, null, credentials); + try { + JSONObject responseJson = new JSONObject(response.content); + JSONObject responseValue = responseJson.getJSONObject(VALUE); + clusterActor.setUp("Up".equals(responseValue.getString("MemberStatus"))); + clusterActor.setUnreachable(false); + } catch(JSONException e) { + log.error("getMemberStatus(): Error parsing response from {}", clusterActor.getNode(), e); + clusterActor.setUp(false); + clusterActor.setUnreachable(true); + } + } + + private void getShardStatus(ClusterActor clusterActor) throws IOException { + log.info("getShardStatus(): Getting shard status for {}", clusterActor.getNode()); + ConnectionResponse response = ConnectionManager.getConnectionResponse(httpProtocol + clusterActor.getNode() + shardManagerPath, ConnectionManager.HttpMethod.GET, null, credentials); + try { + JSONObject responseValue = new JSONObject(response.content).getJSONObject(VALUE); + JSONArray shardList = responseValue.getJSONArray("LocalShards"); + + String pattern = "-config$"; + Pattern r = Pattern.compile(pattern); + List<String> shards = new ArrayList<>(); + for(int ndx = 0; ndx < shardList.length(); ndx++) { + shards.add(shardList.getString(ndx)); + } + shards.parallelStream().forEach(shard -> { + Matcher m = r.matcher(shard); + String operationalShardName = m.replaceFirst("-operational"); + String shardConfigPath = String.format(shardPathTemplate, shard); + String shardOperationalPath = String.format(shardPathTemplate, operationalShardName).replace("Config", "Operational"); + try { + extractShardInfo(clusterActor, shard, shardConfigPath); + extractShardInfo(clusterActor, operationalShardName, shardOperationalPath); + } catch(IOException e) { + log.error("getShardStatus(): Error extracting shard info for {}", shard); + } + }); + } catch(JSONException e) { + log.error("getShardStatus(): Error parsing response from " + clusterActor.getNode(), e); + } + } + + private void extractShardInfo(ClusterActor clusterActor, String shardName, String shardPath) throws IOException { + log.info("extractShardInfo(): Extracting shard info for {}", shardName); + String shardPrefix = ""; +// String shardPrefix = clusterActor.getMember() + "-shard-"; + log.debug("extractShardInfo(): Pulling config info for {} from: {}", shardName, shardPath); + ConnectionResponse response = ConnectionManager.getConnectionResponse(httpProtocol + clusterActor.getNode() + shardPath, ConnectionManager.HttpMethod.GET, null, credentials); + log.debug("extractShardInfo(): Response: {}", response.content); + + try { + JSONObject shardValue = new JSONObject(response.content).getJSONObject(VALUE); + clusterActor.setVoting(shardValue.getBoolean("Voting")); + if(shardValue.getString("PeerAddresses").length() > 0) { + clusterActor.getReplicaShards().add(shardName.replace(shardPrefix, "")); + if(shardValue.getString("Leader").startsWith(clusterActor.getMember())) { + clusterActor.getShardLeader().add(shardName.replace(shardPrefix, "")); + } + } else { + clusterActor.getNonReplicaShards().add(shardName.replace(shardPrefix, "")); + } + JSONArray followerInfo = shardValue.getJSONArray("FollowerInfo"); + for(int followerNdx = 0; followerNdx < followerInfo.length(); followerNdx++) { + int commitIndex = shardValue.getInt("CommitIndex"); + int matchIndex = followerInfo.getJSONObject(followerNdx).getInt("matchIndex"); + if(commitIndex != -1 && matchIndex != -1) { + int commitsBehind = commitIndex - matchIndex; + clusterActor.getCommits().put(followerInfo.getJSONObject(followerNdx).getString("id"), commitsBehind); + } + } + } catch(JSONException e) { + log.error("extractShardInfo(): Error parsing response from " + clusterActor.getNode(), e); + } + } + + public void getControllerHealth(Map<String, ClusterActor> memberMap) { + memberMap.values().parallelStream().forEach(this::getControllerHealth); + } + + // Seen ConcurrentAccess issues, probably related to getting the controller health + private synchronized void getControllerHealth(ClusterActor clusterActor) { + clusterActor.flush(); + log.info("getControllerHealth(): Gathering info for {}", clusterActor.getNode()); + try { + // First flush out the old values + getMemberStatus(clusterActor); + getShardStatus(clusterActor); + } catch(IOException e) { + log.error("getControllerHealth(): Connection Error", e); + clusterActor.setUnreachable(true); + clusterActor.setUp(false); + } + log.info("getControllerHealth(): MemberInfo:\n{}", clusterActor); + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SingleNodeHealthResolver.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SingleNodeHealthResolver.java new file mode 100644 index 000000000..2799df1b0 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SingleNodeHealthResolver.java @@ -0,0 +1,160 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.resolver; + +import org.onap.ccsdk.sli.core.dblib.DbLibService; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth; + +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverInput; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Implementation of {@code HealthResolver} for a single node controller + * architecture. + * + * @author Anthony Haddox + * @see HealthResolver + */ +public class SingleNodeHealthResolver extends HealthResolver { + private final Logger log = LoggerFactory.getLogger(SingleNodeHealthResolver.class); + + /** + * Constructs the health resolver used by the {@code GrToolkitProvider} to + * determine the health of the application components. + * + * @param map a HashMap containing all of the nodes in the akka cluster + * @param properties the properties passed ino the provider + * @param dbLib a reference to the {@code DbLibService} of the provider + * @see HealthResolver + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + */ + public SingleNodeHealthResolver(Map<String, ClusterActor> map, Properties properties, DbLibService dbLib) { + super(map, properties, dbLib); + resolveSites(); + } + + /** + * Implementation of {@code getClusterHealth()}. Uses the + * {@code ShardResolver} to gather health information about the controller. + * This method assumes the cluster is always healthy since it is a single + * node. + * + * @return an {@code ClusterHealth} object with health of the akka cluster + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see ClusterHealth + * @see ShardResolver + */ + @Override + public ClusterHealth getClusterHealth() { + log.info("getClusterHealth(): Getting cluster health..."); + shardResolver.getControllerHealth(memberMap); + return new ClusterHealth().withHealth(Health.HEALTHY); + } + + /** + * Implementation of {@code getSiteHealth()}. Uses the results from + * {@code getAdminHealth}, {@code getDatabaseHealth}, and + * {@code getClusterHealth} to determine the health of the site. If all + * components are healthy, the site is healthy. + * + * @return a List of {@code SiteHealth} objects with health of the site + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see SiteHealth + * @see ShardResolver + */ + @Override + public List<SiteHealth> getSiteHealth() { + log.info("getSiteHealth(): Getting site health..."); + AdminHealth adminHealth = getAdminHealth(); + DatabaseHealth databaseHealth = getDatabaseHealth(); + ClusterHealth clusterHealth = getClusterHealth(); + SiteHealth siteHealth = new SiteHealth() + .withAdminHealth(adminHealth) + .withDatabaseHealth(databaseHealth) + .withClusterHealth(clusterHealth) + .withRole("ACTIVE") + .withId(getSiteIdentifier()); + log.info("getSiteHealth(): Admin Health: {}", adminHealth.getHealth().toString()); + log.info("getSiteHealth(): Database Health: {}", databaseHealth.getHealth().toString()); + log.info("getSiteHealth(): Cluster Health: {}", clusterHealth.getHealth().toString()); + if(isHealthy(adminHealth.getHealth()) && isHealthy(databaseHealth.getHealth()) && isHealthy(clusterHealth.getHealth())) { + siteHealth.setHealth(Health.HEALTHY); + } + + return Collections.singletonList(siteHealth); + } + + /** + * Implementation of {@code tryFailover()}. No controller-level failover + * options are available in a single node architecture, so 400 Bad Request + * is returned, and no action is taken. + * + * @return an {@code SiteHealth} object with health of the site + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see FailoverStatus + * @see FailoverInput + */ + @Override + public FailoverStatus tryFailover(FailoverInput input) { + log.info("tryFailover(): Failover not supported in the current configuration."); + return new FailoverStatus().withStatusCode(400).withMessage("Failover not supported in current configuration."); + } + + /** + * Implementation of {@code resolveSites()}. Calls + * {@code resolveSiteForMember()} to resolve which site a member belongs to. + * + * @see HealthResolver + */ + @Override + public void resolveSites() { + log.info("Map contains {} entries", memberMap.size()); + memberMap.forEach((key, value) -> resolveSiteForMember(value)); + } + + /** + * Resolves which site a member belongs to. Since this is a Single node + * architecture, it is defaulted to <i>Site 1</i>. + * + * @see HealthResolver + */ + private void resolveSiteForMember(ClusterActor actor) { + actor.setSite("Site 1"); + log.info("resolveSiteForMember(): {} belongs to {}", actor.getNode(), actor.getSite()); + } +} diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SixNodeHealthResolver.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SixNodeHealthResolver.java new file mode 100644 index 000000000..e79262cf3 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/SixNodeHealthResolver.java @@ -0,0 +1,316 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.resolver; + +import org.json.JSONArray; +import org.json.JSONObject; + +import org.onap.ccsdk.sli.core.dblib.DbLibService; +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionManager; +import org.onap.ccsdk.sli.plugins.grtoolkit.connection.ConnectionResponse; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth; + +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverInput; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * Implementation of {@code HealthResolver} for a six node controller + * architecture, where three nodes are located in one data center, and the + * other three nodes are located in another. The sites are assumed to be in an + * Active/Standby configuration, with the Active site nodes voting and the + * Standby site notes non-voting. + * + * @author Anthony Haddox + * @see HealthResolver + */ +public class SixNodeHealthResolver extends HealthResolver { + private final Logger log = LoggerFactory.getLogger(SixNodeHealthResolver.class); + + /** + * Constructs the health resolver used by the {@code GrToolkitProvider} to + * determine the health of the application components. + * + * @param map a HashMap containing all of the nodes in the akka cluster + * @param properties the properties passed ino the provider + * @param dbLib a reference to the {@code DbLibService} of the provider + * @see HealthResolver + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + */ + public SixNodeHealthResolver(Map<String, ClusterActor> map, Properties properties, DbLibService dbLib) { + super(map, properties, dbLib); + resolveSites(); + } + + /** + * Implementation of {@code getClusterHealth()}. Uses the + * {@code ShardResolver} to gather health information about the controller. + * If 4 of 6 members are healthy, the cluster is deemed healthy. + * + * @return an {@code ClusterHealth} object with health of the akka cluster + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see ClusterHealth + * @see ShardResolver + */ + @Override + public ClusterHealth getClusterHealth() { + log.info("getClusterHealth(): Getting cluster health..."); + shardResolver.getControllerHealth(memberMap); + long healthyMembers = memberMap.values().stream().filter(member -> member.isUp() && ! member.isUnreachable()).count(); + return (healthyMembers > 4) ? new ClusterHealth().withHealth(Health.HEALTHY) : new ClusterHealth().withHealth(Health.FAULTY); + } + + /** + * Implementation of {@code getSiteHealth()}. Gathers health information on + * all of the contollers, then separates the nodes into voting and + * non-voting sites. Each site is then checked for its health and the + * result is returned as a List. + * + * @return a List of {@code SiteHealth} objects with health of the site + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see SiteHealth + * @see ShardResolver + */ + @Override + public List<SiteHealth> getSiteHealth() { + log.info("getSiteHealth(): Getting site health..."); + + // Get cluster health to populate memberMap with necessary values + getClusterHealth(); + List<ClusterActor> votingActors = memberMap.values().stream().filter(ClusterActor::isVoting).collect(Collectors.toList()); + List<ClusterActor> nonVotingActors = memberMap.values().stream().filter(member -> !member.isVoting()).collect(Collectors.toList()); + + SiteHealth votingSiteHealth = getSiteHealth(votingActors).withRole("ACTIVE"); + SiteHealth nonVotingSiteHealth = getSiteHealth(nonVotingActors).withRole("STANDBY"); + return Arrays.asList(votingSiteHealth, nonVotingSiteHealth); + } + + /** + * Gathers the site identifier, admin health, and database health of a + * site. + * + * @return a {@code SiteHealth} object with health of the site + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see ClusterActor + * @see SiteHealth + * @see ConnectionManager + */ + public SiteHealth getSiteHealth(List<ClusterActor> actorList) { + AdminHealth adminHealth = null; + DatabaseHealth databaseHealth = null; + String siteId = null; + int healthyMembers = 0; + + for(ClusterActor actor : actorList) { + if(actor.isUp() && !actor.isUnreachable()) { + healthyMembers++; + } + if(siteId == null) { + try { + String content = ConnectionManager.getConnectionResponse(httpProtocol + actor.getNode() + ":" + controllerPort + "/restconf/operations/gr-toolkit:site-identifier", ConnectionManager.HttpMethod.POST, null, credentials).content; + siteId = new JSONObject(content).getJSONObject(OUTPUT).getString("id"); + } catch(IOException e) { + log.error("getSiteHealth(): Error getting site identifier from {}", actor.getNode()); + log.error("getSiteHealth(): IOException", e); + } + } + if(adminHealth == null) { + try { + boolean isAdminHealthy = isRemoteComponentHealthy(httpProtocol + actor.getNode() + ":" + controllerPort + "/restconf/operations/gr-toolkit:admin-health"); + if(isAdminHealthy) { + adminHealth = new AdminHealth(Health.HEALTHY, 200); + } + } catch(IOException e) { + log.error("getSiteHealth(): Error getting admin health from {}", actor.getNode()); + log.error("getSiteHealth(): IOException", e); + } + } + if(databaseHealth == null) { + try { + boolean isDatabaseHealthy = isRemoteComponentHealthy(httpProtocol + actor.getNode() + ":" + controllerPort + "/restconf/operations/gr-toolkit:database-health"); + if(isDatabaseHealthy) { + databaseHealth = new DatabaseHealth(Health.HEALTHY); + } + } catch(IOException e) { + log.error("getSiteHealth(): Error getting database health from {}", actor.getNode()); + log.error("getSiteHealth(): IOException", e); + } + } + } + + if(siteId == null) { + siteId = "UNKNOWN SITE"; + } + if(adminHealth == null) { + adminHealth = new AdminHealth(Health.FAULTY, 500); + } + if(databaseHealth == null) { + databaseHealth = new DatabaseHealth(Health.FAULTY); + } + SiteHealth health = new SiteHealth() + .withAdminHealth(adminHealth) + .withDatabaseHealth(databaseHealth) + .withId(siteId); + if(isHealthy(adminHealth.getHealth()) && isHealthy(databaseHealth.getHealth()) && healthyMembers > 1) { + health.setHealth(Health.HEALTHY); + } + + return health; + } + + /** + * Implementation of {@code tryFailover()}. Performs a preliminary call to + * {@code getClusterHealth} to populate information about the cluster. If + * no voting members can be found, the method terminates immediately. The + * nodes are separated into voting and non-voting sites, and a driving + * operator is selected from the non-voting nodes to perform requests + * against. A payload to swap voting between sites is sent to the operator + * to perform a controller-level failover. + * + * @return an {@code SiteHealth} object with health of the site + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see FailoverStatus + * @see FailoverInput + */ + @Override + public FailoverStatus tryFailover(FailoverInput input) { + // Get Cluster Health to populate the memberMap with the necessary values + log.info("tryFailover(): Performing preliminary health check..."); + getClusterHealth(); + FailoverStatus status = new FailoverStatus(); + ConnectionResponse votingResponse = null; + List<ClusterActor> votingActors = memberMap.values().stream().filter(ClusterActor::isVoting).collect(Collectors.toList()); + List<ClusterActor> nonVotingActors = memberMap.values().stream().filter(member -> !member.isVoting()).collect(Collectors.toList()); + + if(nonVotingActors.size() == 0) { + status.setStatusCode(500); + status.setMessage("No nonvoting members found. Cannot perform voting switch."); + return status; + } + + ClusterActor operator; + try { + operator = nonVotingActors.stream().filter(this::isControllerHealthy).findFirst().get(); + } catch(NoSuchElementException e) { + log.error("tryFailover(): Could not find any healthy members.", e); + status.setStatusCode(500); + status.setMessage("Could not find any healthy members."); + return status; + } + + // Assuming two 3 node sites, 3 voting and 3 non voting + if(votingActors.size() < 3 || nonVotingActors.size() < 3) { + log.warn("tryFailover(): Sites do not contain an equal amount of voting and nonvoting members: Voting: {} | NonVoting: {}", votingActors.size(), nonVotingActors.size()); + } + log.info("tryFailover(): Swapping voting..."); + try { + JSONObject votingInput = new JSONObject(); + JSONObject inputBlock = new JSONObject(); + JSONArray votingStateArray = new JSONArray(); + JSONObject memberVotingState; + for(ClusterActor actor : votingActors) { + memberVotingState = new JSONObject(); + memberVotingState.put("member-name", actor.getMember()); + memberVotingState.put("voting", false); + votingStateArray.put(memberVotingState); + } + for(ClusterActor actor : nonVotingActors) { + memberVotingState = new JSONObject(); + memberVotingState.put("member-name", actor.getMember()); + memberVotingState.put("voting", true); + votingStateArray.put(memberVotingState); + } + inputBlock.put("member-voting-state", votingStateArray); + votingInput.put("input", inputBlock); + log.debug("tryFailover(): {}", votingInput); + // Change voting all shards + votingResponse = ConnectionManager.getConnectionResponse(httpProtocol + operator.getNode() + ":" + controllerPort + "/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards", ConnectionManager.HttpMethod.POST, votingInput.toString(), credentials); + } catch(IOException e) { + log.error("tryFailover(): Failure changing voting", e); + } + if(votingResponse != null) { + if(votingResponse.statusCode != 200) { + status.setStatusCode(votingResponse.statusCode); + status.setMessage("Failed to swap voting."); + } else { + status.setStatusCode(200); + status.setMessage("Failover complete."); + } + } else { + status.setStatusCode(500); + status.setMessage("Failed to swap voting."); + } + + return status; + } + + /** + * Implementation of {@code resolveSites()}. Calls + * {@code resolveSiteForMember()} to resolve which site a member belongs to. + * + * @see HealthResolver + */ + @Override + public void resolveSites() { + log.info("Map contains {} entries", memberMap.size()); + memberMap.forEach((key, value) -> resolveSiteForMember(value)); + } + + /** + * Resolves which site a member belongs to. Members 1-3 are assumed to be + * <i>Site 1</i> while members 4-6 are assumed to be <i>Site 2</i>. + * + * @see HealthResolver + */ + private void resolveSiteForMember(ClusterActor actor) { + try { + int memberNumber = Integer.parseInt(actor.getMember().split("-")[1]); + if(memberNumber < 4) { + actor.setSite("Site 1"); + } else { + actor.setSite("Site 2"); + } + log.info("resolveSiteForMember(): {} belongs to {}", actor.getNode(), actor.getSite()); + } catch (NumberFormatException e) { + log.error("resolveSiteForMember(): Could not parse member number for {}. Defaulting to Site 1.", actor.getNode()); + actor.setSite("resolveSiteForMember(): Site 1"); + } + } +}
\ No newline at end of file diff --git a/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ThreeNodeHealthResolver.java b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ThreeNodeHealthResolver.java new file mode 100644 index 000000000..2180b2bf8 --- /dev/null +++ b/plugins/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/resolver/ThreeNodeHealthResolver.java @@ -0,0 +1,162 @@ +/*- + * ============LICENSE_START======================================================= + * openECOMP : SDN-C + * ================================================================================ + * Copyright (C) 2019 AT&T Intellectual Property. All rights + * reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.ccsdk.sli.plugins.grtoolkit.resolver; + +import org.onap.ccsdk.sli.core.dblib.DbLibService; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.AdminHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.DatabaseHealth; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.FailoverStatus; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.Health; +import org.onap.ccsdk.sli.plugins.grtoolkit.data.SiteHealth; + +import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverInput; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Implementation of {@code HealthResolver} for a three node controller + * architecture, where all nodes are located within the same data center or + * geographic region. The nodes are assumed to be in an Active/Active/Active + * voting configuration. + * + * @author Anthony Haddox + * @see HealthResolver + */ +public class ThreeNodeHealthResolver extends HealthResolver { + private final Logger log = LoggerFactory.getLogger(ThreeNodeHealthResolver.class); + + /** + * Constructs the health resolver used by the {@code GrToolkitProvider} to + * determine the health of the application components. + * + * @param map a HashMap containing all of the nodes in the akka cluster + * @param properties the properties passed ino the provider + * @param dbLib a reference to the {@code DbLibService} of the provider + * @see HealthResolver + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + */ + public ThreeNodeHealthResolver(Map<String, ClusterActor> map, Properties properties, DbLibService dbLib) { + super(map, properties, dbLib); + resolveSites(); + } + + /** + * Implementation of {@code getClusterHealth()}. Uses the + * {@code ShardResolver} to gather health information about the controller. + * If 2 of 3 members are healthy, the cluster is deemed healthy. + * + * @return an {@code ClusterHealth} object with health of the akka cluster + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see ClusterHealth + * @see ShardResolver + */ + @Override + public ClusterHealth getClusterHealth() { + log.info("getClusterHealth(): Getting cluster health..."); + shardResolver.getControllerHealth(memberMap); + long healthyMembers = memberMap.values().stream().filter(member -> member.isUp() && ! member.isUnreachable()).count(); + return (healthyMembers > 1) ? new ClusterHealth().withHealth(Health.HEALTHY) : new ClusterHealth().withHealth(Health.FAULTY); + } + + /** + * Implementation of {@code getSiteHealth()}. Uses the results from + * {@code getAdminHealth}, {@code getDatabaseHealth}, and + * {@code getClusterHealth} to determine the health of the site. If all + * components are healthy, the site is healthy. + * + * @return a List of {@code SiteHealth} objects with health of the site + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see SiteHealth + * @see ShardResolver + */ + @Override + public List<SiteHealth> getSiteHealth() { + log.info("getSiteHealth(): Getting site health..."); + AdminHealth adminHealth = getAdminHealth(); + DatabaseHealth databaseHealth = getDatabaseHealth(); + ClusterHealth clusterHealth = getClusterHealth(); + SiteHealth siteHealth = new SiteHealth() + .withAdminHealth(adminHealth) + .withDatabaseHealth(databaseHealth) + .withClusterHealth(clusterHealth) + .withRole("ACTIVE") + .withId(getSiteIdentifier()); + log.info("getSiteHealth(): Admin Health: {}", adminHealth.getHealth().toString()); + log.info("getSiteHealth(): Database Health: {}", databaseHealth.getHealth().toString()); + log.info("getSiteHealth(): Cluster Health: {}", clusterHealth.getHealth().toString()); + if(isHealthy(adminHealth.getHealth()) && isHealthy(databaseHealth.getHealth()) && isHealthy(clusterHealth.getHealth())) { + siteHealth.setHealth(Health.HEALTHY); + } + + return Collections.singletonList(siteHealth); + } + + /** + * Implementation of {@code tryFailover()}. No controller-level failover + * options are available in a three node architecture, so 400 Bad Request + * is returned, and no action is taken. + * + * @return an {@code SiteHealth} object with health of the site + * @see org.onap.ccsdk.sli.plugins.grtoolkit.GrToolkitProvider + * @see HealthResolver + * @see FailoverStatus + * @see FailoverInput + */ + @Override + public FailoverStatus tryFailover(FailoverInput input) { + log.info("tryFailover(): Failover not supported in the current configuration."); + return new FailoverStatus().withStatusCode(400).withMessage("Failover not supported in current configuration."); + } + + /** + * Implementation of {@code resolveSites()}. Calls + * {@code resolveSiteForMember()} to resolve which site a member belongs to. + * + * @see HealthResolver + */ + @Override + public void resolveSites() { + log.info("Map contains {} entries", memberMap.size()); + memberMap.forEach((key, value) -> resolveSiteForMember(value)); + } + + /** + * Resolves which site a member belongs to. Since this is a three node + * co-located architecture, it is defaulted to <i>Site 1</i>. + * + * @see HealthResolver + */ + private void resolveSiteForMember(ClusterActor actor) { + actor.setSite("Site 1"); + log.info("resolveSiteForMember(): {} belongs to {}", actor.getNode(), actor.getSite()); + } +} |