Initial code commit

Change-Id: I9e82864d57a0633aeb6e0107084a80ab926bede8
Issue-ID: MUSIC-77
Signed-off-by: Tschaen, Brendan (bt054f) <bt054f@att.com>
Signed-off-by: Tschaen, Brendan (bptschaen) <bt054f@att.com>

12 files changed, 2676 insertions, 0 deletions

diff --git a/src/main/java/org/onap/music/prom/eelf/logging/EELFLoggerDelegate.java b/src/main/java/org/onap/music/prom/eelf/logging/EELFLoggerDelegate.java
new file mode 100644
index 0000000..c4eb801
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/eelf/logging/EELFLoggerDelegate.java
@@ -0,0 +1,329 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.eelf.logging;
+
+import static com.att.eelf.configuration.Configuration.MDC_SERVER_FQDN;
+import static com.att.eelf.configuration.Configuration.MDC_SERVER_IP_ADDRESS;
+import static com.att.eelf.configuration.Configuration.MDC_SERVICE_INSTANCE_ID;
+import static com.att.eelf.configuration.Configuration.MDC_SERVICE_NAME;
+
+import java.net.InetAddress;
+import java.text.MessageFormat;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.att.eelf.configuration.SLF4jWrapper;
+
+public class EELFLoggerDelegate extends SLF4jWrapper implements EELFLogger {
+
+	public static final EELFLogger errorLogger = EELFManager.getInstance().getErrorLogger();
+	public static final EELFLogger applicationLogger = EELFManager.getInstance().getApplicationLogger();
+	public static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
+	public static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
+	public static final EELFLogger debugLogger = EELFManager.getInstance().getDebugLogger();
+
+	private String className;
+	private static ConcurrentMap<String, EELFLoggerDelegate> classMap = new ConcurrentHashMap<>();
+
+	public EELFLoggerDelegate(final String className) {
+		super(className);
+		this.className = className;
+	}
+
+	/**
+	 * Convenience method that gets a logger for the specified class.
+	 * 
+	 * @see #getLogger(String)
+	 * 
+	 * @param clazz
+	 * @return Instance of EELFLoggerDelegate
+	 */
+	public static EELFLoggerDelegate getLogger(Class<?> clazz) {
+		return getLogger(clazz.getName());
+	}
+
+	/**
+	 * Gets a logger for the specified class name. If the logger does not already
+	 * exist in the map, this creates a new logger.
+	 * 
+	 * @param className
+	 *            If null or empty, uses EELFLoggerDelegate as the class name.
+	 * @return Instance of EELFLoggerDelegate
+	 */
+	public static EELFLoggerDelegate getLogger(final String className) {
+		String classNameNeverNull = className == null || "".equals(className) ? EELFLoggerDelegate.class.getName()
+				: className;
+		EELFLoggerDelegate delegate = classMap.get(classNameNeverNull);
+		if (delegate == null) {
+			delegate = new EELFLoggerDelegate(className);
+			classMap.put(className, delegate);
+		}
+		return delegate;
+	}
+
+	/**
+	 * Logs a message at the lowest level: trace.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void trace(EELFLogger logger, String msg) {
+		if (logger.isTraceEnabled()) {
+			logger.trace(msg);
+		}
+	}
+
+	/**
+	 * Logs a message with parameters at the lowest level: trace.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void trace(EELFLogger logger, String msg, Object... arguments) {
+		if (logger.isTraceEnabled()) {
+			logger.trace(msg, arguments);
+		}
+	}
+
+	/**
+	 * Logs a message and throwable at the lowest level: trace.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void trace(EELFLogger logger, String msg, Throwable th) {
+		if (logger.isTraceEnabled()) {
+			logger.trace(msg, th);
+		}
+	}
+
+	/**
+	 * Logs a message at the second-lowest level: debug.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void debug(EELFLogger logger, String msg) {
+		if (logger.isDebugEnabled()) {
+			logger.debug(msg);
+		}
+	}
+
+	/**
+	 * Logs a message with parameters at the second-lowest level: debug.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void debug(EELFLogger logger, String msg, Object... arguments) {
+		if (logger.isDebugEnabled()) {
+			logger.debug(msg, arguments);
+		}
+	}
+
+	/**
+	 * Logs a message and throwable at the second-lowest level: debug.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void debug(EELFLogger logger, String msg, Throwable th) {
+		if (logger.isDebugEnabled()) {
+			logger.debug(msg, th);
+		}
+	}
+
+	/**
+	 * Logs a message at info level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void info(EELFLogger logger, String msg) {
+		logger.info(className + " - "+msg);
+	}
+
+	/**
+	 * Logs a message with parameters at info level.
+	 *
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void info(EELFLogger logger, String msg, Object... arguments) {
+		logger.info(msg, arguments);
+	}
+
+	/**
+	 * Logs a message and throwable at info level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void info(EELFLogger logger, String msg, Throwable th) {
+		logger.info(msg, th);
+	}
+
+	/**
+	 * Logs a message at warn level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void warn(EELFLogger logger, String msg) {
+		logger.warn(msg);
+	}
+
+	/**
+	 * Logs a message with parameters at warn level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void warn(EELFLogger logger, String msg, Object... arguments) {
+		logger.warn(msg, arguments);
+	}
+
+	/**
+	 * Logs a message and throwable at warn level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void warn(EELFLogger logger, String msg, Throwable th) {
+		logger.warn(msg, th);
+	}
+
+	/**
+	 * Logs a message at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void error(EELFLogger logger, String msg) {
+		logger.error(className+ " - " + msg);
+	}
+
+	/**
+	 * Logs a message with parameters at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void error(EELFLogger logger, String msg, Object... arguments) {
+		logger.warn(msg, arguments);
+	}
+
+	/**
+	 * Logs a message and throwable at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void error(EELFLogger logger, String msg, Throwable th) {
+		logger.warn(msg, th);
+	}
+
+	/**
+	 * Logs a message with the associated alarm severity at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param severtiy
+	 */
+	public void error(EELFLogger logger, String msg, Object /*AlarmSeverityEnum*/ severtiy) {
+		logger.error(msg);
+	}
+
+	/**
+	 * Initializes the logger context.
+	 */
+	public void init() {
+		setGlobalLoggingContext();
+		final String msg = "############################ Logging is started. ############################";
+		// These loggers emit the current date-time without being told.
+		info(applicationLogger, msg);
+		error(errorLogger, msg);
+		debug(debugLogger, msg);
+        info(auditLogger, msg);
+		info(metricsLogger, msg);
+	}
+
+
+	/**
+	 * Builds a message using a template string and the arguments.
+	 * 
+	 * @param message
+	 * @param args
+	 * @return
+	 */
+	private String formatMessage(String message, Object... args) {
+		StringBuilder sbFormattedMessage = new StringBuilder();
+		if (args != null && args.length > 0 && message != null && message != "") {
+			MessageFormat mf = new MessageFormat(message);
+			sbFormattedMessage.append(mf.format(args));
+		} else {
+			sbFormattedMessage.append(message);
+		}
+
+		return sbFormattedMessage.toString();
+	}
+
+	/**
+	 * Loads all the default logging fields into the MDC context.
+	 */
+	private void setGlobalLoggingContext() {
+		MDC.put(MDC_SERVICE_INSTANCE_ID, "");
+		try {
+			MDC.put(MDC_SERVER_FQDN, InetAddress.getLocalHost().getHostName());
+			MDC.put(MDC_SERVER_IP_ADDRESS, InetAddress.getLocalHost().getHostAddress());
+		} catch (Exception e) {
+			errorLogger.error("setGlobalLoggingContext failed", e);
+		}
+	}
+
+	public static void mdcPut(String key, String value) {
+		MDC.put(key, value);
+	}
+
+	public static String mdcGet(String key) {
+		return MDC.get(key);
+	}
+
+	public static void mdcRemove(String key) {
+		MDC.remove(key);
+	}
+
+}
diff --git a/src/main/java/org/onap/music/prom/main/ConfigReader.java b/src/main/java/org/onap/music/prom/main/ConfigReader.java
new file mode 100644
index 0000000..224f4b2
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/main/ConfigReader.java
@@ -0,0 +1,86 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.main;
+
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+public class ConfigReader {
+	private static String configLocation = "config.json";
+	
+	public static void setConfigLocation(String pathToFile){
+		configLocation = pathToFile+"/config.json";
+	}
+	
+	private static JSONObject getJsonHandle(){
+		JSONParser parser = new JSONParser();
+		Object obj =null;
+		try {
+			obj = parser.parse(new FileReader(configLocation));
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (ParseException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		JSONObject jsonObject = (JSONObject) obj;	
+		return jsonObject;
+	}
+
+	public static ArrayList<String> getConfigListAttribute(String key){
+		ArrayList<String> value = (ArrayList<String>) getJsonHandle().get(key);
+		return value; 
+	}
+
+	public static String getConfigAttribute(String key){
+		Object value = getJsonHandle().get(key);
+		return (value!=null) ? String.valueOf(value) : null;
+	}
+	
+	public static String getConfigAttribute(String key, String defaultValue){
+		String toReturn = getConfigAttribute(key);
+		return (toReturn!=null) ? toReturn : defaultValue;
+	}
+
+	public static ArrayList<String> getExeCommandWithParams(String key){
+		String script = (String)getJsonHandle().getOrDefault(key, "");
+		String[] scriptParts = script.split(" ");
+		ArrayList<String> scriptWithPrams = new ArrayList<String>();
+		for(int i=0; i < scriptParts.length;i++)
+			scriptWithPrams.add(scriptParts[i]);
+		return scriptWithPrams;
+	}
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/onap/music/prom/main/PromDaemon.java b/src/main/java/org/onap/music/prom/main/PromDaemon.java
new file mode 100644
index 0000000..f39228b
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/main/PromDaemon.java
@@ -0,0 +1,608 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.main;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.onap.music.prom.eelf.logging.EELFLoggerDelegate;
+import org.onap.music.prom.musicinterface.MusicHandle;
+
+
+public class PromDaemon {
+	
+	private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(PromDaemon.class);
+	
+	String id;
+	String lockName,lockRef;
+	public  enum CoreState {PASSIVE, ACTIVE};
+	public  enum ScriptResult {ALREADY_RUNNING, SUCCESS_RESTART, FAIL_RESTART};
+	String keyspaceName;
+	String tableName;
+	
+	public PromDaemon(String id){
+		this.id = id; 
+		bootStrap();
+	}
+	
+	/** Do not use, only for testing **/
+	PromDaemon(){
+		
+	}
+	
+	private void bootStrap(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "Bootstrapping this site daemon");
+		keyspaceName = "prom_"+ConfigReader.getConfigAttribute("app-name");
+		MusicHandle.createKeyspaceEventual(keyspaceName);
+		
+		tableName = "Replicas";
+		Map<String,String> replicaFields = new HashMap<String,String>();
+		replicaFields.put("id", "text");
+		replicaFields.put("isactive", "boolean");
+		replicaFields.put("timeoflastupdate", "varint");
+		replicaFields.put("lockref", "text");
+		replicaFields.put("PRIMARY KEY", "(id)");
+
+		MusicHandle.createTableEventual(keyspaceName, tableName, replicaFields);	
+		MusicHandle.createIndexInTable(keyspaceName, tableName, "lockref");
+		
+		Map<String,Object> values = new HashMap<String,Object>();
+		values.put("id",this.id);
+		values.put("isactive","false");
+		values.put("timeoflastupdate", "0");
+		//values.put("lockref", "");
+		MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+		//MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+		
+		lockName = keyspaceName+".active.lock";
+	}
+	
+	/**
+	 * Get a lockRef if one doesn't exist. If a lockRef exists, return the same lockRef.
+	 * This is used if the daemon crashes and is able to recover or restart.
+	 * @return the lockRef for this site
+	 */
+	private String getLockRefOrOldLockRefIfExists(){	
+		//first check if a lock reference exists for this id..
+		Map<String,Object> replicaDetails = MusicHandle.readSpecificRow(keyspaceName, tableName, "id", this.id); 
+
+		if (replicaDetails == null || !replicaDetails.containsKey("row 0") 
+					|| !((Map<String,String>) replicaDetails.get("row 0")).containsKey("lockref")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No entry found in MUSIC Replicas table for this daemon.");
+
+			return MusicHandle.createLockRef(lockName);
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, replicaDetails.toString());
+
+		
+		String prevLockRef = ((Map<String, String>) replicaDetails.get("row 0")).get("lockref");
+		if (prevLockRef==null || prevLockRef.equals("")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "Previous running state detected,"
+									+ "but cannot get previous lock reference.");
+			return MusicHandle.createLockRef(lockName);
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, "Previous lock found for this prom replica:"+prevLockRef);
+		return prevLockRef;
+	}
+	
+	
+	/**
+	 * This function maintains the key invariant that it will return true for only one id
+	 * @return true if this replica is current lock holder
+	 */
+	private boolean isActiveLockHolder(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "isActiveLockHolder");
+		boolean isLockHolder = acquireLock();
+		if (isLockHolder) {//update active table
+			logger.info(EELFLoggerDelegate.applicationLogger, "Daemon is the current activeLockHolder");
+			Map<String,Object> values = new HashMap<String,Object>();
+			values.put("isactive","true");
+			values.put("id",this.id);
+			MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+		}
+		return isLockHolder;
+	}
+	
+	/**
+	 * tries to acquire lockRef
+	 * if lockRef no longer exists creates a new lock and updates locally
+	 * @return true if active lock holder, false otherwise
+	 */
+	private boolean acquireLock() {
+		logger.info(EELFLoggerDelegate.applicationLogger, "acquiringLock '" + lockRef +"'");
+		if (lockRef==null) return false;
+		
+		Map<String, Object> result = MusicHandle.acquireLock(lockRef);
+		Map<String, Object> lockMap = (Map<String, Object>) result.get("lock");
+		if (result.get("status").equals("FAILURE") &&
+				result.getOrDefault("message", "Lockid doesn't exist").equals("Lockid doesn't exist")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "Resulting json was: " +result);
+			logger.info(EELFLoggerDelegate.applicationLogger,
+					"Lockref " + lockRef + " doesn't exist, getting new lockref");
+			lockRef = MusicHandle.createLockRef(lockName);
+			logger.info(EELFLoggerDelegate.applicationLogger, "This site's new reference is " + lockRef);
+			result = MusicHandle.acquireLock(lockRef);
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, "result of acquiring lock " + result.get("status"));
+		logger.info(EELFLoggerDelegate.applicationLogger, "Current lock holder is " + MusicHandle.whoIsLockHolder(this.lockName));
+		return (result.get("status").equals("SUCCESS")?true:false);
+	}
+	
+	
+	/**
+	 * The main startup function for each daemon
+	 * @param startPassive dictates whether the node should start in an passive mode
+	 */
+	private void startHAFlow(boolean startPassive){
+		logger.info(EELFLoggerDelegate.applicationLogger, "startHAFlow"+startPassive);
+		if (startPassive) {
+			startAsPassiveReplica();
+		}
+		
+		lockRef = getLockRefOrOldLockRefIfExists();	
+
+		while (true) {
+			if (isActiveLockHolder()) {
+				activeFlow();
+			}
+			else {
+				passiveFlow();
+			}
+		}
+	}
+
+	/**
+	 * Waits until there is an active, running replica
+	 */
+	private void startAsPassiveReplica() {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"Starting in 'passive mode'. Checking to see if active has started");
+		String activeLockRef = MusicHandle.whoIsLockHolder(lockName);
+		Map<String,Object> active = getReplicaDetails(activeLockRef);
+		
+		while (active==null || !(Boolean)active.getOrDefault("isactive", false)
+				 || !isReplicaAlive((String)active.get("id"))) {
+			activeLockRef = MusicHandle.whoIsLockHolder(lockName);
+			active = getReplicaDetails(activeLockRef);
+			//back off if needed
+			try {
+				Long sleeptime = Long.parseLong(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "1000"));
+				if (sleeptime>0) {
+					logger.info(EELFLoggerDelegate.applicationLogger, "Sleeping for " + sleeptime + " ms");
+					Thread.sleep(sleeptime);
+				}
+			} catch (Exception e) {
+					logger.error(e.getMessage());
+			}
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, 
+				"Active site id=" + active.get("id") + " has started. Continuing in passive mode");
+	}
+	
+	/**
+	 * Make sure that the replica you are monitoring is running by running
+	 * the script provided.
+	 * 
+	 * Try to run the script noOfRetryAttempts times, as defined by the prom configuration.
+	 * This function will wait in between retry attempts, as determined by 'restart-backoff-time' 
+	 * defined in prom configuration file (immediate retry is default, if no value is provided)
+	 * 
+	 * @param script script to be run
+	 * @return ScriptResult based off scripts response
+	 */
+	private ScriptResult tryToEnsureCoreFunctioning(ArrayList<String> script){
+		logger.info(EELFLoggerDelegate.applicationLogger, "tryToEnsureCoreFunctioning");
+		int noOfAttempts = Integer.parseInt(ConfigReader.getConfigAttribute("no-of-retry-attempts"));
+		ScriptResult result = ScriptResult.FAIL_RESTART;
+		
+		while (noOfAttempts > 0) {			
+			result = PromUtil.executeBashScriptWithParams(script);
+			if (result == ScriptResult.ALREADY_RUNNING) {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"Executed core script, the core was already running");
+				return result;
+			} else if (result == ScriptResult.SUCCESS_RESTART) {
+				logger.info(EELFLoggerDelegate.applicationLogger, 
+						"Executed core script, the core had to be restarted");
+				return result;
+			} else if (result == ScriptResult.FAIL_RESTART) {
+				noOfAttempts--;
+				logger.info(EELFLoggerDelegate.applicationLogger, 
+						"Executed core script, the core could not be re-started, retry attempts left ="+noOfAttempts);
+			}
+			//backoff period in between restart attempts
+			try {
+				Thread.sleep(Long.parseLong(ConfigReader.getConfigAttribute("restart-backoff-time", "0")));
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"Tried enough times and still unable to start the core, giving up lock and starting passive flow..");
+		return result;
+	}
+	
+	/**
+	 * Update this replica's lockRef and update the heartbeat in replica table
+	 */
+	private void updateHealth(CoreState isactive) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "updateHealth " +isactive);
+		Map<String,Object> values = new HashMap<String,Object>();
+		values.put("id",this.id);
+		values.put("timeoflastupdate", System.currentTimeMillis());
+		values.put("lockref", this.lockRef);
+		values.put("isactive",isactive==CoreState.ACTIVE?true:false);
+		MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+	}
+	
+	/**
+	 * Checks to see if the replica is alive
+	 * @param id the id of the replica to check if is alive
+	 * @return
+	 */
+	private boolean isReplicaAlive(String id){
+		logger.info(EELFLoggerDelegate.applicationLogger, "isReplicaAlive " + id);
+		Map<String,Object> valueMap = MusicHandle.readSpecificRow(keyspaceName, tableName, "id", id);
+		if (valueMap == null || valueMap.isEmpty()) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No entry showing...");
+			return false; 
+		}
+		valueMap = (Map<String, Object>) valueMap.get("row 0");
+		
+		if (!valueMap.containsKey("timeoflastupdate") || valueMap.get("timeoflastupdate")==null) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No 'timeoflastupdate' entry showing...");
+			return false;
+		}
+
+		long lastUpdate = (Long)valueMap.get("timeoflastupdate");
+		logger.info(EELFLoggerDelegate.applicationLogger, id + "'s time of last update:"+lastUpdate);
+	    long timeOutPeriod = PromUtil.getPromTimeout();
+	    long currentTime = System.currentTimeMillis();
+	    logger.info(EELFLoggerDelegate.applicationLogger, "current time:"+currentTime);
+	    long timeSinceUpdate = currentTime-lastUpdate; 
+	    logger.info(EELFLoggerDelegate.applicationLogger, id + "'s time since update:"+timeSinceUpdate);
+	    if(timeSinceUpdate > timeOutPeriod) {
+	    	return false;
+	    } else {
+	    	return true;
+	    }
+	}
+	
+	private Map<String, Object> getReplicaDetails(String lockRef){
+		Map<String,Object> details = MusicHandle.readSpecificRow(keyspaceName, tableName, "lockref", lockRef);
+		if (details==null) { return null; }
+		return (Map<String, Object>) details.getOrDefault("row 0", null);
+	}
+
+	/**
+	 * Releases lock and ensures replica id's 'isactive' state to false
+	 * @param lockRef
+	 */
+	private void releaseLock(String lockRef){
+		logger.info(EELFLoggerDelegate.applicationLogger, "releaseLock " + lockRef);
+		if(lockRef == null){
+			logger.info(EELFLoggerDelegate.applicationLogger, "There is no lock entry..");
+			return;
+		}
+			
+		if(lockRef.equals("")){
+			logger.info(EELFLoggerDelegate.applicationLogger, "Already unlocked..");
+			return;
+		}
+		
+		Map<String, Object> replicaDetails = getReplicaDetails(lockRef);
+		String replicaId = "UNKNOWN";
+		if (replicaDetails!=null) {
+			replicaId = (String)replicaDetails.get("id");
+		}
+		
+		
+		logger.info(EELFLoggerDelegate.applicationLogger, "Unlocking prom "+replicaId + " with lockref"+ lockRef);
+		MusicHandle.unlock(lockRef);
+		logger.info(EELFLoggerDelegate.applicationLogger, "Unlocked prom "+replicaId);
+		if (replicaId.equals(this.id)) { //if unlocking myself, remove reference to lockref
+			this.lockRef=null;
+		}
+		
+		if (replicaId.equals("UNKNOWN")) {
+			return;
+		}
+		//create entry in replicas table
+		Map<String,Object> values = new HashMap<String,Object>();
+		values.put("isactive",false);		
+		values.put("lockref", "");
+		MusicHandle.updateTableEventual(keyspaceName, tableName, "id", replicaId, values);
+	}
+	
+	private void tryToEnsurePeerHealth(){
+		ArrayList<String> replicaList =  ConfigReader.getConfigListAttribute(("replica-id-list"));
+		for (Iterator<String> iterator = replicaList.iterator(); iterator.hasNext();) {
+			String replicaId = (String) iterator.next();
+			if(replicaId.equals(this.id) == false){
+				if(isReplicaAlive(replicaId) == false){
+					//restart if suspected dead
+					//releaseLock(replicaId);
+					//Don't hold up main thread for restart
+					Runnable restartThread = new RestartThread(replicaId);
+					new Thread(restartThread).start();
+					
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							lockRef + " status: "+MusicHandle.acquireLock(lockRef));
+				}
+			}
+		}	
+	}
+	
+	private boolean restartPromDaemon(String replicaId, int noOfAttempts){
+		logger.info(EELFLoggerDelegate.applicationLogger, "Prom Daemon--"+replicaId+"--needs to be restarted");
+
+		ArrayList<String> restartScript = ConfigReader.getExeCommandWithParams("restart-prom-"+replicaId);
+		if (restartScript!=null && restartScript.size()>0 && restartScript.get(0).length()>0) {
+			PromUtil.executeBashScriptWithParams(restartScript);
+		}
+		return true;//need to find a way to check if the script is running. Just check if process is running maybe? 
+/*
+		boolean result = false;
+		while(result == false){
+			ArrayList<String> restartScript = ConfigReader.getExeCommandWithParams("restart-prom-"+id);
+			PromUtil.executeBashScriptWithParams(restartScript);
+			result = Boolean.parseBoolean(resultString);
+			noOfAttempts--;
+			if(noOfAttempts <= 0)
+				break; 
+		}
+		return result; 
+*/	}
+	
+	/**
+	 * Give current active sufficient time (as defined by configured 'prom-timeout' value) to become passive.
+	 * If current active does not become passive in the configured amount of time, the current active site
+	 * is forcibly reset to a passive state.
+	 * 
+	 * This method should only be called after the lock of the previous active is released and this 
+	 * replica has become the new active
+	 * 
+	 * @param currentActiveId
+	 */
+	private void takeOverFromCurrentActive(String currentActiveLockRef){
+		if (currentActiveLockRef==null || currentActiveLockRef.equals(this.lockRef)) {
+			return;
+		}
+		
+		long startTime = System.currentTimeMillis();
+		long restartTimeout = PromUtil.getPromTimeout();
+		while(true){
+			Map<String,Object> replicaDetails = getReplicaDetails(currentActiveLockRef);
+			if (replicaDetails==null || !replicaDetails.containsKey("isactive") ||
+					!(Boolean)replicaDetails.get("isactive")) {
+				break;
+			}
+			
+			//waited long enough..just make the old active passive yourself
+			if ((System.currentTimeMillis() - startTime) > restartTimeout) {
+				logger.info(EELFLoggerDelegate.applicationLogger, 
+						"Old Active not responding..resetting Music state of old active to passive myself");
+				Map<String, Object> removeActive = new HashMap<String,Object>();
+				removeActive.put("isactive", false);
+				MusicHandle.updateTableEventual(keyspaceName, tableName, "lockref", currentActiveLockRef, removeActive);
+				break;
+			}
+			//make sure we don't time out while we wait
+			updateHealth(CoreState.PASSIVE);
+		}
+
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"Old Active has now become passive, so starting active flow ***");
+
+		//now you can take over as active! 
+	}
+	
+	
+	private void activeFlow(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "activeFlow");
+		while (true) {
+			if(acquireLock() == false){
+				logger.info(EELFLoggerDelegate.applicationLogger, "I no longer have the lock! Make myself passive");
+				return;
+			}
+			
+			ScriptResult result = tryToEnsureCoreFunctioning(ConfigReader.getExeCommandWithParams("ensure-active-"+id));
+			if (result == ScriptResult.ALREADY_RUNNING) {
+				//do nothing
+			} else if (result == ScriptResult.SUCCESS_RESTART) {
+				//do nothing
+			} else if (result == ScriptResult.FAIL_RESTART) {//unable to start core, just give up and become passive
+				releaseLock(lockRef);
+				return;
+			}
+			
+			updateHealth(CoreState.ACTIVE);
+			
+			logger.info(EELFLoggerDelegate.applicationLogger, 
+					"--(Active) Prom Daemon--"+id+"---CORE ACTIVE---Lock Ref:"+lockRef);
+
+			tryToEnsurePeerHealth();
+			logger.info(EELFLoggerDelegate.applicationLogger, 
+					"--(Active) Prom Daemon--"+id+"---PEERS CHECKED---");
+
+			//back off if needed
+			try {
+				Long sleeptime = Long.parseLong(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "0"));
+				if (sleeptime>0) {
+					logger.info(EELFLoggerDelegate.applicationLogger, "Sleeping for " + sleeptime + " ms");
+					Thread.sleep(sleeptime);
+				}
+			} catch (Exception e) {
+					logger.error(e.getMessage());
+			}
+		}
+	}
+	
+	private void passiveFlow(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "passiveFlow");
+		while(true){
+			ScriptResult result = tryToEnsureCoreFunctioning(ConfigReader.getExeCommandWithParams("ensure-passive-"+id));		
+			if (result == ScriptResult.ALREADY_RUNNING) {
+				if (lockRef==null) {
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							"Replica does not have a lock, but is running. Getting a lock now");
+					lockRef = MusicHandle.createLockRef(lockName);
+					logger.info(EELFLoggerDelegate.applicationLogger, "new lockRef " + lockRef);
+				}
+			} else if (result == ScriptResult.SUCCESS_RESTART) {
+				//we can now handle being after, put yourself back in queue
+				logger.info(EELFLoggerDelegate.applicationLogger, "Script successfully restarted. Getting a new lock");
+				lockRef = MusicHandle.createLockRef(lockName);
+				logger.info(EELFLoggerDelegate.applicationLogger, "new lockRef " + lockRef);
+			} else if (result == ScriptResult.FAIL_RESTART) {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"Site not working and could not restart, releasing lock"); 
+				releaseLock(lockRef);
+			}
+			
+			//update own health in music
+			updateHealth(CoreState.PASSIVE);
+			
+			logger.info(EELFLoggerDelegate.applicationLogger,
+					"-- {Passive} Prom Daemon--"+id+"---CORE PASSIVE---Lock Ref:"+lockRef);
+
+			//obtain active lock holder's id
+			String activeLockRef = MusicHandle.whoIsLockHolder(lockName);
+			releaseLockIfActiveIsDead(activeLockRef);
+			
+			if (isActiveLockHolder()) {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"***I am the active lockholder, so taking over from previous active***");
+				takeOverFromCurrentActive(activeLockRef);
+				return;
+			}
+
+			//back off if needed
+			try {
+				Long sleeptime = Long.parseLong(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "0"));
+				if (sleeptime>0) {
+					logger.info(EELFLoggerDelegate.applicationLogger, "Sleeping for " + sleeptime + " ms");
+					Thread.sleep(sleeptime);
+				}
+			} catch (Exception e) {
+				logger.error(e.getMessage());
+			}
+		}
+	}
+
+	/**
+	 * Releases the lock if the active lock holder is dead, not responsive, or cannot be found.
+	 * @param activeLockRef
+	 * @return the active id
+	 */
+	private void releaseLockIfActiveIsDead(String activeLockRef) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "releaseLockIfActiveIsDead " + activeLockRef);
+		Map<String, Object> activeDetails =  getReplicaDetails(activeLockRef);
+		Boolean activeIsAlive = false;
+		String activeId = null;
+		if (activeDetails!=null) {
+			activeId = (String)activeDetails.get("id");
+			logger.info(EELFLoggerDelegate.applicationLogger, "Active lockholder is site " + activeId);
+			activeIsAlive = isReplicaAlive(activeId);
+		}
+
+		if (activeIsAlive == false) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "Active lockholder is not alive");
+			if (activeId==null) {
+				if (activeLockRef!=null && !activeLockRef.equals("")) {
+					//no reference to the current lock, probably corrupt/stale data
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							"Unknown active lockholder. Releasing current lock");
+					MusicHandle.unlock(activeLockRef);
+				} else {
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							"*****No lock holders. Make sure there are healthy sites*****");
+				}
+			} else {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"Active " + activeId + " is suspected dead. Releasing it's lock.");
+				releaseLock(activeLockRef);
+			}
+		}
+	}
+	
+	private class RestartThread implements Runnable{
+		String replicaId;
+		public RestartThread(String replicaId) {
+			this.replicaId = replicaId;
+		}
+		public void run() {
+			restartPromDaemon(this.replicaId, 1);
+		}
+	}
+	
+	public static void main(String[] args){
+		Options opts = new Options();
+		
+		Option idOpt = new Option("i", "id", true, "prom identifier");
+		idOpt.setRequired(true);
+		opts.addOption(idOpt);
+		
+		Option passive = new Option("p", "passive", false, "start prom in passive mode (default false)");
+		opts.addOption(passive);
+		
+		Option config = new Option("c", "config", true, "location of config.json file (default same directory as prom jar)");
+		opts.addOption(config);
+		
+		CommandLineParser parser = new DefaultParser();
+		HelpFormatter formatter = new HelpFormatter();
+		CommandLine cmd;
+		try {
+			cmd = parser.parse(opts, args);
+		} catch (ParseException e) {
+			e.printStackTrace();
+			formatter.printHelp("prom", opts);
+			System.exit(1);
+			return;
+		}
+		
+		String id = cmd.getOptionValue("id");
+		boolean startPassive = false;
+		if (cmd.hasOption("passive")) {
+			startPassive = true;
+		}
+		if (cmd.hasOption("c")) {
+			ConfigReader.setConfigLocation(cmd.getOptionValue("c"));
+		}
+
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"--Prom Daemon version "+PromUtil.version+"--replica id "+id+"---START---"+(startPassive?"passive":"active"));
+		PromDaemon hd = new PromDaemon(id);
+		hd.startHAFlow(startPassive);
+	}
+
+}
diff --git a/src/main/java/org/onap/music/prom/main/PromUtil.java b/src/main/java/org/onap/music/prom/main/PromUtil.java
new file mode 100644
index 0000000..a0be5d7
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/main/PromUtil.java
@@ -0,0 +1,174 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.main;
+
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Properties;
+
+import org.onap.music.prom.eelf.logging.EELFLoggerDelegate;
+import org.onap.music.prom.main.PromDaemon.ScriptResult;
+import org.onap.music.prom.musicinterface.MusicHandle;
+
+
+
+public class PromUtil {
+	private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(PromDaemon.class);
+
+	public static String version;
+	static {
+		try {
+			final Properties properties = new Properties();
+			properties.load(PromUtil.class.getClassLoader().getResourceAsStream("project.properties"));
+			version = properties.getProperty("version");
+			logger.info(EELFLoggerDelegate.applicationLogger, "Prom version " + version);
+		} catch (IOException e) {
+			logger.error(e.getMessage());
+		}
+	}
+	
+	private static ArrayList<String> getMusicNodeIp(){
+		return ConfigReader.getConfigListAttribute("music-location");
+/*		String serverAddress;
+		serverAddress = agaveMusicNode;
+		while(isHostUp(serverAddress) != true)
+			serverAddress = toggle(serverAddress);
+		return serverAddress;
+*/	}
+	
+/*	public static String toggle(String serverAddress){
+		if(serverAddress.equals(agaveMusicNode)){
+			System.out.println("Agave is down...connect to Big Site");
+			serverAddress = bigSiteMusicNode;
+		}else if(serverAddress.equals(bigSiteMusicNode)){
+			System.out.println("Big Site is down...connect to Agave");
+			serverAddress = agaveMusicNode;
+		}
+		return serverAddress;
+	}*/
+	
+	public static ArrayList<String> getMusicNodeURL(){
+		ArrayList<String> ips = getMusicNodeIp();
+		ArrayList<String> urls = new ArrayList<String>();
+		for (String ip: ips) {
+			urls.add( "http://"+ip+":8080/MUSIC/rest/v" +PromUtil.getMusicVersion());
+		}
+		return urls;
+	}
+	
+	public static String getMusicVersion() {
+		String version = ConfigReader.getConfigAttribute("music-version", "2");
+		if (version==null) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"No MUSIC Version provided in your configuration file. Please "
+					+ "include 'musicVersion' in your config.json file.");
+			throw new Error("Required property 'music-version' is not provided");
+		}
+		return version;
+	}
+	
+	public static boolean isHostUp(String serverAddress) { 
+		Boolean isUp = false;
+	    try {
+			InetAddress inet = InetAddress.getByName(serverAddress);
+			isUp = inet.isReachable(1000);	
+		} catch (UnknownHostException e) {
+			logger.error(e.getMessage());
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			logger.error(e.getMessage());
+		}
+	    return isUp;
+	}
+	
+	
+	
+	/* MUSIC authentication functions */
+	public static String getAid() {
+		return ConfigReader.getConfigAttribute("aid", "");
+	}
+	
+	public static String getAppNamespace() {
+		return ConfigReader.getConfigAttribute("namespace", "");
+	}
+	
+	public static String getUserId() {
+		return ConfigReader.getConfigAttribute("userid", "");
+	}
+	
+	public static String getPassword() {
+		return ConfigReader.getConfigAttribute("password", "");
+	}
+	/* End of MUSIC authentication functions */
+	
+	public static int getPromTimeout() {
+		return Integer.parseInt(ConfigReader.getConfigAttribute("prom-timeout"));
+	}
+	
+	/**
+	 * Gets 'music-connection-timeout-ms' property from configuration file, returning a negative number if 
+	 * it doesn't exist
+	 * @return
+	 */
+	public static int getTimeoutToMusicMillis() {
+		return Integer.parseInt(ConfigReader.getConfigAttribute("music-connection-timeout-ms", "-1"));
+	}
+	
+	public static ScriptResult executeBashScriptWithParams(ArrayList<String> script){
+		logger.info(EELFLoggerDelegate.applicationLogger, "executeBashScript: " + script);
+		try {
+			ProcessBuilder pb = new ProcessBuilder(script);
+			final Process process = pb.start();
+			int exitCode = process.waitFor();
+
+			StringBuffer errorOutput = new StringBuffer();
+            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
+            String line = "";                       
+            while ((line = reader.readLine())!= null) {
+            		if(!line.equals(""))
+            			errorOutput.append(line + "\n");
+            }
+            System.out.print(errorOutput);
+			if (exitCode == 0)
+				return ScriptResult.ALREADY_RUNNING;
+			else if (exitCode == 1)
+				return ScriptResult.FAIL_RESTART;
+			else if (exitCode == 2)
+				return ScriptResult.SUCCESS_RESTART;
+
+		} catch (IOException e) {
+			logger.error("PromUtil executingBashScript: " + e.getMessage());
+		} catch (InterruptedException e) {
+			logger.error("PromUtil executingBashScript: " + e.getMessage());
+		}
+		return ScriptResult.FAIL_RESTART;
+	}
+	
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonDelete.java b/src/main/java/org/onap/music/prom/musicinterface/JsonDelete.java
new file mode 100644
index 0000000..11d3408
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonDelete.java
@@ -0,0 +1,60 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+public class JsonDelete {
+	
+    private ArrayList<String> columns = null;
+    private Map<String,String> consistencyInfo;
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+    public ArrayList<String> getColumns() {
+		return columns;
+	}
+	public void setColumns(ArrayList<String> columns) {
+		this.columns = columns;
+	}
+	String ttl, timestamp;
+
+	public String getTtl() {
+		return ttl;
+	}
+	public void setTtl(String ttl) {
+		this.ttl = ttl;
+	}
+	public String getTimestamp() {
+		return timestamp;
+	}
+	public void setTimestamp(String timestamp) {
+		this.timestamp = timestamp;
+	}
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonInsert.java b/src/main/java/org/onap/music/prom/musicinterface/JsonInsert.java
new file mode 100644
index 0000000..443da5a
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonInsert.java
@@ -0,0 +1,66 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+
+import java.util.Map;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+public class JsonInsert {
+	
+    private Map<String,Object> values;
+    String ttl, timestamp;
+	private Map<String,Object> row_specification;
+    private Map<String,String> consistencyInfo;
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+	public String getTtl() {
+		return ttl;
+	}
+	public void setTtl(String ttl) {
+		this.ttl = ttl;
+	}
+	public String getTimestamp() {
+		return timestamp;
+	}
+	public void setTimestamp(String timestamp) {
+		this.timestamp = timestamp;
+	}
+	public Map<String, Object> getValues() {
+		return values;
+	}
+	public void setValues(Map<String, Object> values) {
+		this.values = values;
+	}
+    public Map<String, Object> getRow_specification() {
+		return row_specification;
+	}
+	public void setRow_specification(Map<String, Object> row_specification) {
+		this.row_specification = row_specification;
+	}
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonKeySpace.java b/src/main/java/org/onap/music/prom/musicinterface/JsonKeySpace.java
new file mode 100644
index 0000000..9b91021
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonKeySpace.java
@@ -0,0 +1,57 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+
+import java.util.Map;
+
+
+public class JsonKeySpace {
+    private Map<String,Object> replicationInfo;
+	private String durabilityOfWrites;
+    private Map<String,String> consistencyInfo;
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+	public Map<String, Object> getReplicationInfo() {
+		return replicationInfo;
+	}
+	
+	public void setReplicationInfo(Map<String, Object> replicationInfo) {
+		this.replicationInfo = replicationInfo;
+	}
+
+	public String getDurabilityOfWrites() {
+		return durabilityOfWrites;
+	}
+	public void setDurabilityOfWrites(String durabilityOfWrites) {
+		this.durabilityOfWrites = durabilityOfWrites;
+	}
+		
+	
+
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonTable.java b/src/main/java/org/onap/music/prom/musicinterface/JsonTable.java
new file mode 100644
index 0000000..d4f8cc7
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonTable.java
@@ -0,0 +1,64 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+import java.util.Map;
+
+public class JsonTable {
+    private Map<String,String> fields;
+	private Map<String, Object> properties; 
+    private String clusteringOrder;
+    private Map<String,String> consistencyInfo;
+
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+    public Map<String, Object> getProperties() {
+		return properties;
+	}
+
+	public void setProperties(Map<String, Object> properties) {
+		this.properties = properties;
+	}
+    
+	public Map<String, String> getFields() {
+		return fields;
+	}
+
+	public void setFields(Map<String, String> fields) {
+		this.fields = fields;
+	}
+
+	public String getClusteringOrder() {
+		return clusteringOrder;
+	}
+
+	public void setClusteringOrder(String clusteringOrder) {
+		this.clusteringOrder = clusteringOrder;
+	}
+
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/MusicHandle.java b/src/main/java/org/onap/music/prom/musicinterface/MusicHandle.java
new file mode 100644
index 0000000..d62a1d2
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/MusicHandle.java
@@ -0,0 +1,709 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+import java.net.ConnectException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+
+import org.onap.music.prom.eelf.logging.EELFLoggerDelegate;
+import org.onap.music.prom.main.ConfigReader;
+import org.onap.music.prom.main.PromUtil;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+
+public class MusicHandle {
+	private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(MusicHandle.class);
+
+
+	/**
+	 * Adds MUSIC's authentication headers into the webresource
+	 * @param webResource
+	 */
+	private static Builder addMusicHeaders(WebResource webResource) {
+		String aid = PromUtil.getAid();
+		String namespace = PromUtil.getAppNamespace();
+		String userId = PromUtil.getUserId();
+		String password = PromUtil.getPassword();
+		Builder builder = webResource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON);
+		if (!aid.equals("")) {
+			builder.header("aid", aid);
+		}
+		if (!namespace.equals("")) {
+			builder.header("ns", namespace);
+		}
+		if (!userId.equals("")) {
+			builder.header("userId", userId);
+		}
+		if (!password.equals("")) {
+			builder.header("password", password);
+		}
+		
+		return builder;
+	}
+	
+	private static WebResource createMusicWebResource(String musicAPIurl) {
+		ClientConfig clientConfig = new DefaultClientConfig();
+
+		clientConfig.getFeatures().put(
+				JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE);
+		int timeout = getMaxConnectionTimeout();
+		clientConfig.getProperties().put(ClientConfig.PROPERTY_CONNECT_TIMEOUT, timeout);
+		clientConfig.getProperties().put(ClientConfig.PROPERTY_READ_TIMEOUT, timeout);
+
+		Client client = Client.create(clientConfig);
+		return client.resource(musicAPIurl);
+	}
+	
+	public static void createKeyspaceEventual(String keyspaceName){
+		logger.info(EELFLoggerDelegate.applicationLogger, "createKeyspaceEventual "+keyspaceName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				createKeyspaceEventual(musicUrl, keyspaceName);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create keyspace."
+							+ "Could not successfully reach any music instance to create keyspace.");
+		return;
+	}
+	
+	private static void createKeyspaceEventual(String musicUrl, String keyspaceName){
+		Map<String,Object> replicationInfo = new HashMap<String, Object>();
+		replicationInfo.put("class", "SimpleStrategy");
+		replicationInfo.put("replication_factor", 3);
+		String durabilityOfWrites="true";
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+		JsonKeySpace jsonKp = new JsonKeySpace();
+		jsonKp.setConsistencyInfo(consistencyInfo);
+		jsonKp.setDurabilityOfWrites(durabilityOfWrites);
+		jsonKp.setReplicationInfo(replicationInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName);
+
+		ClientResponse response = addMusicHeaders(webResource)
+									.post(ClientResponse.class, jsonKp);
+		
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			if (output.containsKey("error")) {
+				String errorMsg = String.valueOf(output.get("error"));
+				if (errorMsg.equals("err:Keyspace prom_sdnc already exists")) {
+					logger.warn(EELFLoggerDelegate.applicationLogger, 
+							"Not creating keyspace " + keyspaceName + " because it already exists. Continuing.");
+					//assume keyspace is already created and continue
+				}
+				else { //unhandled/unknown exception
+					logger.error(EELFLoggerDelegate.errorLogger,
+							"Failed to createKeySpaceEventual : Status Code "+ output.toString());
+					throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+				}
+			} else { //no exception message
+				logger.error(EELFLoggerDelegate.errorLogger,
+						"Failed to createKeySpaceEventual : Status Code "+ output.toString());
+				throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+			}
+		}
+	}
+
+	public static void createTableEventual(String keyspaceName, String tableName, Map<String,String> fields) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"createKeyspaceEventual "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				createTableEventual(musicUrl, keyspaceName, tableName, fields);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create table. "
+						+ "Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void createTableEventual(String musicUrl, String keyspaceName,
+										String tableName, Map<String,String> fields) {
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonTable jtab = new JsonTable();
+		jtab.setFields(fields);
+		jtab.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName);
+
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class, jtab);
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			if (output.containsKey("error")) {
+				String error = String.valueOf(output.get("error"));
+				if (error.equalsIgnoreCase("Table " + keyspaceName + "." + tableName + " already exists")) {
+					logger.warn(EELFLoggerDelegate.applicationLogger, 
+							"Not creating table " + tableName + " because it already exists. Continuing.");
+				} else { //unhandled exception message
+					logger.error(EELFLoggerDelegate.errorLogger,
+							"Failed to createTableEventual : Status Code "+ output.toString());
+					throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+				}
+			} else { //no exception message, MUSIC should give more info if failure
+				logger.error(EELFLoggerDelegate.errorLogger, 
+						"Failed to createTableEventual : Status Code "+ output.toString());
+				throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+			}
+		}
+	}
+
+	public static void createIndexInTable(String keyspaceName, String tableName, String colName) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"createIndexInTable "+keyspaceName+" tableName "+tableName + " colName" + colName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				createIndexInTable(musicUrl, keyspaceName, tableName, colName);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create index in table. "
+									+ "Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void createIndexInTable(String musicUrl, String keyspaceName, String tableName, String colName) {
+		WebResource webResource = createMusicWebResource(musicUrl +
+							"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/index/"+colName);
+
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class);
+
+		if (response.getStatus() != 200 && response.getStatus() != 204) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to createIndexInTable : Status Code " + response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+	}
+
+	public static void insertIntoTableEventual(String keyspaceName, String tableName, Map<String,Object> values) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"insertIntoTableEventual "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				insertIntoTableEventual(musicUrl, keyspaceName, tableName, values);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to insert into table."
+								+ " Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void insertIntoTableEventual(String musicUrl, String keyspaceName,
+											String tableName, Map<String,Object> values) {
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonInsert jIns = new JsonInsert();
+		jIns.setValues(values);
+		jIns.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/rows");
+
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class, jIns);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to insertIntoTableEventual : Status Code " + response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to createKeySpaceEventual : Status Code "+ output.toString());
+			throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+		}
+	}
+
+	public static void updateTableEventual(String keyspaceName, String tableName, String keyName,
+													String keyValue, Map<String,Object> values) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "updateTableEventual "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				updateTableEventual(musicUrl, keyspaceName, tableName, keyName, keyValue, values);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to update the table. "
+									+ "Could not successfully reach any music instance.");
+	}
+
+	
+	private static void updateTableEventual(String musicUrl, String keyspaceName, String tableName,
+									String keyName, String keyValue, Map<String,Object> values) {		
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonInsert jIns = new JsonInsert();
+		jIns.setValues(values);
+		jIns.setConsistencyInfo(consistencyInfo);
+		
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName
+								+"/tables/"+tableName+"/rows?"+keyName+"="+keyValue);
+
+		ClientResponse response = addMusicHeaders(webResource).put(ClientResponse.class, jIns);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to updateTableEventual : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to createKeySpaceEventual : Status Code "+ output.toString());
+			throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+		}
+	}
+
+	public static Map<String,Object> readSpecificRow(String keyspaceName, String tableName,
+			String keyName, String keyValue) {
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		Map<String, Object> result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = readSpecificRow(musicUrl, keyspaceName, tableName, keyName, keyValue);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to read row. "
+							+ "Could not successfully reach any music instance.");
+		result = null;
+		return result;
+	}
+	
+	private static Map<String,Object> readSpecificRow(String musicUrl, String keyspaceName, String tableName,
+												String keyName, String keyValue) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"readSpecificRow "+keyspaceName+" tableName "+tableName + " key" +keyName + " value" + keyValue);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName
+											+"/tables/"+tableName+"/rows?"+keyName+"="+keyValue);
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to insertIntoTableEventual : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+
+		Map<String, Object> rowMap = (Map<String, Object>) output.getOrDefault("result", null);
+
+		return rowMap;	
+	}
+
+	public static Map<String,Object> readAllRows(String keyspaceName, String tableName) {
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		Map<String, Object> result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = readAllRows(musicUrl, keyspaceName, tableName);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to read all rows. "
+								+ "Could not successfully reach any music instance.");
+		result = null;
+		return result;
+	}
+	
+	private static Map<String,Object> readAllRows(String musicUrl, String keyspaceName, String tableName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "readAllRows "+keyspaceName+" tableName "+tableName);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/rows");
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to readAllRows : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		return output;	
+	}
+
+	public static void dropTable(String keyspaceName, String tableName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "dropTable "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				dropTable(musicUrl, keyspaceName, tableName);
+				return;
+			}  catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to drop table."
+							+ " Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void dropTable(String musicUrl, String keyspaceName, String tableName) {
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonTable jsonTb = new JsonTable();
+		jsonTb.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName);
+	
+		ClientResponse response = addMusicHeaders(webResource).delete(ClientResponse.class, jsonTb);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to dropTable : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+	}
+
+	public static void dropKeySpace(String keyspaceName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "dropKeySpace "+keyspaceName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				dropKeySpace(musicUrl, keyspaceName);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to drop keyspace."
+							+ " Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void dropKeySpace(String musicUrl, String keyspaceName) {		
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonKeySpace jsonKp = new JsonKeySpace();
+		jsonKp.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName);
+
+		ClientResponse response = addMusicHeaders(webResource).delete(ClientResponse.class, jsonKp);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to dropTable : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+	}
+
+	public static String createLockRef(String lockName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "createLockRef "+lockName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		String result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = createLockRef(musicUrl, lockName);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create lock reference. "
+											+ "Could not successfully reach any music instance.");
+		result = "";
+		return result;
+	}
+	
+	private static String createLockRef(String musicUrl, String lockName) {
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/create/"+lockName);
+		
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class);
+
+		if (response.getStatus() != 200 ) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to createLockRef : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+		Map<String,Object> responseMap = response.getEntity(Map.class);
+		if (!responseMap.containsKey("status") || !responseMap.get("status").equals("SUCCESS") ||
+				!responseMap.containsKey("lock")) {
+			logger.error(EELFLoggerDelegate.errorLogger, 
+					"Failed to createLockRef : Status Code "+ responseMap.toString());
+			return "";
+		}
+		String lockRef = ((Map<String, String>) responseMap.get("lock")).get("lock");
+		logger.info(EELFLoggerDelegate.applicationLogger, "This site's lockReference is "+lockRef);
+		return lockRef;
+	}
+
+	/**
+	 * Try to acquire the lock lockid.
+	 * If cannot reach any music instance, return status:FAILURE
+	 * @param lockId
+	 * @return
+	 */
+	public static Map<String,Object> acquireLock(String lockId) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "acquireLock "+lockId);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		Map<String, Object> result;
+
+		for (String musicUrl: musicUrls) {
+			try {
+				result = acquireLock(musicUrl, lockId);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to acquireLock. Could not successfully reach any music instance.");
+		result = new HashMap<String, Object>();
+		result.put("status", "FAILURE");
+		return result;
+	}
+	
+	private static Map<String,Object> acquireLock(String musicUrl, String lockId){
+		//should be fixed in MUSIC, but putting patch here too
+		if (lockId==null) {
+			Map<String,Object> fail = new HashMap<String, Object>();
+			fail.put("status", "FAILURE");
+			return fail;
+		}
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/acquire/"+lockId);
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() != 200) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to acquireLock : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+
+		return output;
+	}
+
+	public static String whoIsLockHolder(String lockName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "whoIsLockHolder "+lockName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		String result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = whoIsLockHolder(musicUrl, lockName);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to check who the lock holder is. "
+											+ "Could not successfully reach any music instance.");
+		result = null;
+		return result;
+	}
+	
+	private static String whoIsLockHolder(String musicUrl, String lockName) {	
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/enquire/"+lockName);
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() != 200) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to determine whoIsLockHolder : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+		Map<String,String> lockoutput = (Map<String, String>) response.getEntity(Map.class).get("lock");
+		if (lockoutput.get("lock-holder").equals("No lock holder!")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No lock holder");
+			return null;
+		}
+		return (String) lockoutput.get("lock-holder");
+	}
+
+	public static void unlock(String lockId) {
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				unlock(musicUrl, lockId);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to unlock the lock. "
+								+ "Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void unlock(String musicUrl, String lockId) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "unlock "+lockId);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/release/"+lockId);
+
+		ClientResponse response = addMusicHeaders(webResource).delete(ClientResponse.class);
+
+		Map<String,Object> responseMap = response.getEntity(Map.class);
+		
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to unlock : Status Code "+response.getStatus());
+			if (responseMap.containsKey("error")) {
+				logger.error(EELFLoggerDelegate.errorLogger, "Failed to unlock : "+responseMap.get("error"));
+			}
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+	}
+
+
+	
+	/**
+	 * Gets a connection timeout to music. This function will return the 
+	 * configured parameter given in the prom json config, if available.
+	 * Otherwise, it will calculate a timeout such that it the connection will be able
+	 * to cycle through the different music locations prior to other nodes assuming this
+	 * replica is dead.
+	 * @return
+	 */
+	private static int getMaxConnectionTimeout() {
+		int timeout = PromUtil.getTimeoutToMusicMillis();
+		if (timeout<=0) { // user hasn't defined a valid timeout
+			ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+			int promTimeout = PromUtil.getPromTimeout();
+			timeout = promTimeout/musicUrls.size();
+		}
+		return timeout;
+	}
+	
+	
+	public static void main(String[] args){
+		Map<String,Object> results = MusicHandle.readAllRows("votingappbharath", "replicas");
+		for (Map.Entry<String, Object> entry : results.entrySet()){
+			Map<String, Object> valueMap = (Map<String, Object>)entry.getValue();
+			for (Map.Entry<String, Object> rowentry : valueMap.entrySet()){
+				if(rowentry.getKey().equals("timeoflastupdate")){
+					System.out.println(rowentry.getValue());
+				}
+				break;
+			}
+			break;
+		}
+	}
+
+}
+
+
+
diff --git a/src/main/resources/project.properties b/src/main/resources/project.properties
new file mode 100644
index 0000000..a2273e2
--- /dev/null
+++ b/src/main/resources/project.properties
@@ -0,0 +1,2 @@
+version=${project.version}
+artifactId=${project.artifactId}
\ No newline at end of file
diff --git a/src/test/java/org/onap/music/prom/main/PromDaemonTest.java b/src/test/java/org/onap/music/prom/main/PromDaemonTest.java
new file mode 100644
index 0000000..be2d798
--- /dev/null
+++ b/src/test/java/org/onap/music/prom/main/PromDaemonTest.java
@@ -0,0 +1,361 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+
+package org.onap.music.prom.main;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.hamcrest.core.IsAnything;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.VerificationModeFactory;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.onap.music.prom.main.ConfigReader;
+import org.onap.music.prom.main.PromDaemon;
+import org.onap.music.prom.main.PromUtil;
+import org.onap.music.prom.main.PromDaemon.CoreState;
+import org.onap.music.prom.musicinterface.MusicHandle;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.reflect.Whitebox;
+
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({MusicHandle.class, ConfigReader.class, PromUtil.class})
+public class PromDaemonTest {
+	static PromDaemon promDaemon;
+
+	@BeforeClass
+	public static void beforeClass() {
+		promDaemon = Mockito.spy(PromDaemon.class);
+	}
+	
+	@Before
+	public void before() {
+		promDaemon.lockName = "lockName";
+		promDaemon.keyspaceName = "keyspaceName";
+		promDaemon.id = "anIdToTestFor";
+		
+		PowerMockito.mockStatic(ConfigReader.class);
+		PowerMockito.when(ConfigReader.getConfigAttribute("prom-timeout")).thenReturn("1000");
+		PowerMockito.when(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "1000")).thenReturn("1");
+	}
+	
+	
+	@Test
+	public void bootstrapTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		PowerMockito.mockStatic(ConfigReader.class);
+		
+		PowerMockito.when(ConfigReader.getConfigAttribute("app-name")).thenReturn("testing");
+		
+		Whitebox.invokeMethod(promDaemon,  "bootStrap");
+		
+		assertEquals("prom_testing", promDaemon.keyspaceName);
+		assertEquals("prom_testing.active.lock", promDaemon.lockName);
+	}
+	
+	@Test
+	public void acquireLockTrue() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		falseMap.put("status", "SUCCESS");
+		falseMap.put("message", "You already have the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean acquireLock = Whitebox.invokeMethod(promDaemon,  "acquireLock");
+		assertTrue(acquireLock);
+	}
+	
+	@Test
+	public void acquireLockFalse() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		falseMap.put("status", "FAILURE");
+		falseMap.put("message", "you don't own the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean acquireLock = Whitebox.invokeMethod(promDaemon,  "acquireLock");
+		assertFalse(acquireLock);
+	}
+	
+	@Test
+	public void acquireNullLock() throws Exception {
+		promDaemon.lockRef = null;
+		
+		Boolean acquireLock = Whitebox.invokeMethod(promDaemon,  "acquireLock");
+		assertFalse(acquireLock);
+	}
+
+	@Test
+	public void activeLockHolderTestTrue() throws Exception{
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		falseMap.put("status", "SUCCESS");
+		falseMap.put("message", "You already have the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean isActiveLockHolder = Whitebox.invokeMethod(promDaemon,  "isActiveLockHolder");
+		assertTrue(isActiveLockHolder);
+	}
+	
+	@Test
+	public void activeLockHolderTestFalse() throws Exception{
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		falseMap.put("status", "FAILURE");
+		falseMap.put("message", "You do not own the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean isActiveLockHolder = Whitebox.invokeMethod(promDaemon,  "isActiveLockHolder");
+		assertFalse(isActiveLockHolder);
+	}
+	
+	@Test
+	public void activeLockHolderTestStaleLock() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> staleLockMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		staleLockMap.put("status", "FAILURE");
+		staleLockMap.put("message", "Lockid doesn't exist");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(staleLockMap);
+		
+		PowerMockito.when(MusicHandle.createLockRef("lockName")).thenReturn("testLock2");
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		falseMap.put("status", "FAILURE");
+		falseMap.put("message", "You do not own the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock2")).thenReturn(falseMap);
+		
+		Boolean isActiveLockHolder = Whitebox.invokeMethod(promDaemon,  "isActiveLockHolder");
+		assertFalse(isActiveLockHolder);
+		assertEquals("testLock2", promDaemon.lockRef);
+	}
+	
+	
+	@Test
+	public void releaseLockTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		Whitebox.invokeMethod(promDaemon, "releaseLock", null);
+		Whitebox.invokeMethod(promDaemon, "releaseLock", "");
+		
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
+			.thenReturn(null);
+		Whitebox.invokeMethod(promDaemon, "releaseLock", "lock1");
+		
+		//should actually release now
+		ArgumentCaptor<Map> mapCaptor = ArgumentCaptor.forClass(Map.class);
+		HashMap<String,Object> map = new HashMap<String,Object>();
+		HashMap<String,Object> repDetails = new HashMap<String,Object>();
+		repDetails.put("id", promDaemon.id);
+		map.put("row 0", repDetails);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
+		.thenReturn(map);
+		Whitebox.invokeMethod(promDaemon, "releaseLock", "lock1");
+		
+		PowerMockito.verifyStatic();
+		MusicHandle.updateTableEventual(Mockito.anyString(), Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), mapCaptor.capture());
+		assertEquals(false, mapCaptor.getValue().get("isactive"));
+	}
+
+	@Test
+	public void activeHealthTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+
+		ArgumentCaptor<String> keyspaceCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<String> tableCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<Map> mapCaptor = ArgumentCaptor.forClass(Map.class);
+		Whitebox.invokeMethod(promDaemon, "updateHealth", CoreState.ACTIVE);
+
+		PowerMockito.verifyStatic();
+		MusicHandle.insertIntoTableEventual(keyspaceCaptor.capture(), tableCaptor.capture(), mapCaptor.capture());
+		Map<String, Object> returnedMap = mapCaptor.getValue();
+
+		assertTrue((Boolean) returnedMap.get("isactive"));
+		assertTrue(System.currentTimeMillis()-500< (long) returnedMap.get("timeoflastupdate"));
+	}
+	
+	@Test
+	public void passiveHealthTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+
+		ArgumentCaptor<String> keyspaceCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<String> tableCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<Map> mapCaptor = ArgumentCaptor.forClass(Map.class);
+		Whitebox.invokeMethod(promDaemon, "updateHealth", CoreState.PASSIVE);
+
+		PowerMockito.verifyStatic();
+		MusicHandle.insertIntoTableEventual(keyspaceCaptor.capture(), tableCaptor.capture(), mapCaptor.capture());
+		Map<String, Object> returnedMap = mapCaptor.getValue();
+
+		assertFalse((Boolean) returnedMap.get("isactive"));
+		//make sure call was somewhat recent, as synched with current system clock
+		//may need to make this more strict or less strict depending
+		assertTrue(System.currentTimeMillis()-500< (long) returnedMap.get("timeoflastupdate"));
+	}
+	
+
+	@Test
+	public void replicaIsAliveTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		
+		//no test replica
+		Boolean isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+		
+		//return null pointer
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(null);
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+
+		//is active is dead
+		Map<String,Object> deadReplica = new HashMap<String,Object>();
+		Map<String,Object> replicaInfo = new HashMap<String,Object>();
+		replicaInfo.put("id", "testReplica");
+		replicaInfo.put("isactive", "false");
+		deadReplica.put("row 0", replicaInfo);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(deadReplica);
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+		
+		//timed out
+		replicaInfo.put("timeoflastupdate", System.currentTimeMillis()-1000);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(deadReplica);
+
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+		
+		//alive
+		replicaInfo.put("timeoflastupdate", System.currentTimeMillis());
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(deadReplica);
+		PowerMockito.when(ConfigReader.getConfigAttribute("prom-timeout")).thenReturn("1000");
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertTrue(isReplicaAlive);
+	}
+	
+	
+	/**
+	 * try to start as passive. First iteration will fail because the replica is stale.
+	 * Second iteration should exit the method. In failure cases, this might throw an 
+	 * exception to prevent an infinite loop.
+	 * 
+	 * @throws Exception
+	 */
+	@Test
+	public void startAsPassiveReplicaTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		String activeLock = "actLock";
+		Map<String, Object> staleActiveMap = new HashMap<String, Object>();
+		Map<String, Object> staleInfo = new HashMap<String, Object>();
+		staleInfo.put("id", "activeReplica");
+		staleInfo.put("isactive", true);
+		staleInfo.put("timeoflastupdate", System.currentTimeMillis()-1001);
+		staleActiveMap.put("row 0", staleInfo);
+		Map<String, Object> activeActiveMap = new HashMap<String, Object>();
+		Map<String, Object> activeInfo = new HashMap<String, Object>();
+		activeInfo.put("id", "activeReplica");
+		activeInfo.put("isactive", true);
+		activeInfo.put("timeoflastupdate", System.currentTimeMillis());
+		activeActiveMap.put("row 0", activeInfo);
+		
+		PowerMockito.when(MusicHandle.whoIsLockHolder(promDaemon.lockName)).thenReturn(activeLock);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
+			.thenReturn(staleActiveMap).thenReturn(staleActiveMap)
+			.thenReturn(activeActiveMap).thenReturn(activeActiveMap)
+			.thenThrow(new RuntimeException("Should exit before reaching here"));
+		
+		Whitebox.invokeMethod(promDaemon,  "startAsPassiveReplica");
+		
+		//make sure we went through 2 iterations. Each iteration makes 2 calls to readSpecific row so 2x2 is 4
+		PowerMockito.verifyStatic(VerificationModeFactory.times(4));
+		MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+	}
+	
+	
+	@Test
+	public void getLockRefOrOldLockRefIfExistsTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		//no entry in music
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef1");
+		String lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef1", lockref);
+		
+		//entry in music doesn't have lockref column
+		Map<String, Object> entriesInMusic = new HashMap<String, Object>();
+		Map<String, Object> entry = new HashMap<String, Object>();
+		entry.put("id", promDaemon.id);
+		entriesInMusic.put("row 0", entry);
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+				.thenReturn(entriesInMusic);
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef2");
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef2", lockref);
+
+		//entry in music didn't previously have a lockref
+		entry.put("lockref", null);
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+		.thenReturn(entriesInMusic);
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef3");
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef3", lockref);
+		
+		//entry in music didn't previously have a lockref
+		entry.put("lockref", "");
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+				.thenReturn(entriesInMusic);
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef4");
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef4", lockref);
+		
+		//entry had a previous lock entry
+		entry.put("lockref", "previousLockRef");
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+				.thenReturn(entriesInMusic);
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("previousLockRef", lockref);
+	}
+}
diff --git a/src/test/java/org/onap/music/prom/musicinterface/MusicHandleTest.java b/src/test/java/org/onap/music/prom/musicinterface/MusicHandleTest.java
new file mode 100644
index 0000000..48f75c3
--- /dev/null
+++ b/src/test/java/org/onap/music/prom/musicinterface/MusicHandleTest.java
@@ -0,0 +1,160 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+
+package org.onap.music.prom.musicinterface;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.onap.music.prom.main.ConfigReader;
+import org.onap.music.prom.main.PromUtil;
+import org.onap.music.prom.musicinterface.MusicHandle;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.api.client.config.ClientConfig;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ConfigReader.class, Client.class, ClientResponse.class,
+		WebResource.class, WebResource.Builder.class, PromUtil.class})
+public class MusicHandleTest {
+	
+	ClientResponse response;
+	Client client;
+	WebResource webresource;
+	WebResource.Builder webresourceBuilder;
+	
+	
+	@Before
+	public void before() throws Exception {
+		PowerMockito.mockStatic(ConfigReader.class);
+		ArrayList<String> urls = new ArrayList<String>();
+		Collections.addAll(urls, "1.2.3.4", "5.6.7.8");
+		PowerMockito.when(ConfigReader.getConfigAttribute(Mockito.anyString(), Mockito.anyString()))
+			.thenCallRealMethod();
+		
+		PowerMockito.mockStatic(PromUtil.class);
+		//PowerMockito.spy(PromUtil.class);
+		PowerMockito.when(PromUtil.getPromTimeout()).thenReturn(Integer.parseInt("1000"));
+		PowerMockito.when(PromUtil.getMusicNodeURL()).thenReturn(urls);
+		PowerMockito.when(PromUtil.getAid()).thenReturn("");
+		PowerMockito.when(PromUtil.getAppNamespace()).thenReturn("");
+		PowerMockito.when(PromUtil.getUserId()).thenReturn("");
+		PowerMockito.when(PromUtil.getPassword()).thenReturn("");
+		
+		
+		response = PowerMockito.mock(ClientResponse.class);
+		PowerMockito.mockStatic(Client.class);
+		client = PowerMockito.mock(Client.class);
+		webresource = PowerMockito.mock(WebResource.class);
+		webresourceBuilder = PowerMockito.mock(WebResource.Builder.class);
+		
+		//PowerMockito.when(Client.create()).thenReturn(client);
+		PowerMockito.when(Client.create((ClientConfig) Mockito.anyObject())).thenReturn(client);
+		PowerMockito.when(client.resource(Mockito.anyString())).thenReturn(webresource);
+		PowerMockito.when(webresource.accept(MediaType.APPLICATION_JSON)).thenReturn(webresourceBuilder);
+		PowerMockito.when(webresourceBuilder.type(MediaType.APPLICATION_JSON)).thenReturn(webresourceBuilder);
+		PowerMockito.when(webresourceBuilder.get(ClientResponse.class)).thenReturn(response);
+		PowerMockito.when(webresourceBuilder.post(ClientResponse.class)).thenReturn(response);
+
+	}
+
+	@Test
+	public void acquireLockTestFailure() {
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "FAILURE");
+		PowerMockito.when(response.getStatus()).thenReturn(200);
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		Map<String, Object> result = MusicHandle.acquireLock("testLock");
+		assertEquals("FAILURE", result.get("status"));
+	}
+	
+	@Test
+	public void acquireLockTestFailureCannotReachFirstMusic() {
+		PowerMockito.when(response.getStatus()).thenReturn(404, 404, 404, 200);
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "SUCCESS");
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		Map<String, Object> result = MusicHandle.acquireLock("testLock");
+		assertEquals("SUCCESS", result.get("status"));
+	}
+	
+	@Test
+	public void acuireLockTestCannotReachAnyMusic() {
+		PowerMockito.when(response.getStatus()).thenReturn(404);
+		Map<String, Object> result = MusicHandle.acquireLock("testLock");
+		assertEquals("FAILURE", result.get("status"));
+	}
+
+
+	@Test
+	public void createLockRefSuccess() {
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "SUCCESS");
+		Map<String, Object> lockMap = new HashMap<String, Object>();
+		lockMap.put("lock", "abc_lockref");
+		acquireLockResponse.put("lock", lockMap);
+		PowerMockito.when(response.getStatus()).thenReturn(200);
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		
+		String result = MusicHandle.createLockRef("testLock");
+		assertEquals("abc_lockref", result);
+	}
+	
+	@Test
+	public void createLockRefFailure() {
+		//Fail all music instances
+		PowerMockito.when(response.getStatus()).thenReturn(404);
+		String result = MusicHandle.createLockRef("testLock");
+		assertEquals("", result);
+	}
+	
+	@Test
+	public void createLockRefFailFirstMusic() {
+		PowerMockito.when(response.getStatus()).thenReturn(404, 404, 200);
+		
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "SUCCESS");
+		Map<String, Object> lockMap = new HashMap<String, Object>();
+		lockMap.put("lock", "abc_lockref");
+		acquireLockResponse.put("lock", lockMap);
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		
+		String result = MusicHandle.createLockRef("testLock");
+		assertEquals("abc_lockref", result);
+	}
+}