Initial code commit

Change-Id: I9e82864d57a0633aeb6e0107084a80ab926bede8
Issue-ID: MUSIC-77
Signed-off-by: Tschaen, Brendan (bt054f) <bt054f@att.com>
Signed-off-by: Tschaen, Brendan (bptschaen) <bt054f@att.com>

28 files changed, 3337 insertions, 0 deletions

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..63621cc
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,26 @@
+FROM library/ubuntu:latest
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y  software-properties-common && \
+    add-apt-repository ppa:webupd8team/java -y && \
+    apt-get update && \
+    echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \
+    apt-get install -y oracle-java8-installer && \
+    apt-get install -y python && \
+    apt-get install -y python-pip && \
+    apt-get clean
+
+#for promoverride script
+RUN pip install requests
+
+#REMOVE THIS BEFORE SHIPPING
+#RUN apt-get update && \
+#    apt-get install -y vim
+
+
+COPY target/prom.jar sampleApp/startPromDaemon.sh sampleApp/promoverride.py /app/
+WORKDIR /app
+
+CMD ./startPromDaemon.sh -i $ID -c /opt/app/prom/ $PASSIVE -z
+
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..7135650
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,23 @@
+The following license applies to all files in this and sub-directories. Licenses
+are included in individual source files where appropriate, and if it differs
+from this text, it supersedes this.  Any file that does not have license text
+defaults to being covered by this text; not all files support the addition of
+licenses. 
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2018 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..7c27b45
--- /dev/null
+++ b/README.md
@@ -0,0 +1,69 @@
+
+# Policy Driven Ownership and Management (PROM) Protocol for Active-Passive Systems 
+
+Often we wish to deploy service replicas in an active-passive mode where there is only one active service and if that fails, one of the passives takes over as the new active. The trouble is, to implement this, the writer of the service has to worry about challenging distributed systems concepts like group membership, failure detection, leader election, split-brain problems and so on. prom addresses this issue, by providing a library that services can simply configure and deploy as a companion daemon to their service replicas, that will handle all distributed systems issues highlighted above. 
+
+Note: prom relies on system clocks for this timeout to detect failure. In order to make sure your system is closely synchronized, consider using NTP or similar.
+
+
+<a name="local-install"> 
+
+## Setup and Usage
+
+</a>
+
+- The starting point for prom is that you wish to replicate a service on multiple servers/hosts/VMs (refereed to as a node) such that one of them is active and the others are passive at all times. 
+- Ensure that MUSIC <a href="">https://github.com/att/music</a> is running across all these nodes as a cluster. 
+-  Build prom and copy the resultant prom.jar into all the nodes along with config.json file and the startPromDaemon.sh script (sample files provided in this repository under the sampleApp folder). 
+-  Modify the config.json (same at all nodes). We explain the config.json through an example: 
+		
+		
+		{
+		    "app-name":"votingAppBharath",
+                    "aid":"",
+                    "namespace":"",
+                    "userid":"",
+                    "password":"",
+		    "ensure-active-0":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 0 active",
+		    "ensure-active-1":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 1 active",
+		    "ensure-active-2":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 2 active",
+		    "ensure-active-3":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 3 active",
+		    "ensure-passive-0":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 0 passive",
+		    "ensure-passive-1":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 1 passive",
+		    "ensure-passive-2":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 2 passive",
+		    "ensure-passive-3":"/home/ubuntu/votingapp/ensureVotingAppRunning.sh 3 passive",
+		    "restart-prom-0":"ssh -i /home/ubuntu/votingapp/bharath_cirrus101.pem ubuntu@135.197.240.156 /home/ubuntu/votingapp/restartPromIfDead.sh 0",
+		    "restart-prom-1":"ssh -i /home/ubuntu/votingapp/bharath_cirrus101.pem ubuntu@135.197.240.158 /home/ubuntu/votingapp/restartPromIfDead.sh 1",
+		    "restart-prom-2":"ssh -i /home/ubuntu/votingapp/bharath_bigsite.pem ubuntu@135.197.226.68 /home/ubuntu/votingapp/restartPromIfDead.sh 2",
+		    "restart-prom-3":"ssh -i /home/ubuntu/votingapp/bharath_bigsite.pem ubuntu@135.197.226.49 /home/ubuntu/votingapp/restartPromIfDead.sh 3",
+		    "prom-timeout":"50000",
+		    "restart-backoff-time":"1000",
+		    "core-monitor-sleep-time":"1000",
+		    "no-of-retry-attempts":"3",
+		    "replica-id-list":["0","1","2","3"]
+		    "music-location":["localhost"]
+                    "music-version":2
+		} 
+	The *app-name*	 is simply the name chosen for the service. 
+
+	The *aid* is the identification provided by MUSIC during onboarding, if AAF authentication is not used.
+	
+	The *start-active-i* signifies that this is the site that should start in the active mode. Valid entries for this are "true" or "false". NOTE: if multiple entries have *start-active-i* set to true, any of them are viable options for the single initial active site.
+	
+	The *ensure-active-i* and *ensure-passive-i* scripts need to be provided for 	all the service replicas, wherein the i corresponds to each of their ids. 	The ids must start from 0 with single increments. As seen in the example, 	the command within the string will be invoked by prom to run the servce in 	either active or passive mode. These scripts should return the linux exit 	code of 0 if they run successfully. 
+	
+	The *restart-prom-i* scripts are used by the hal daemons running along with 	each replica to restart each other. Since the hal daemons reside on 	different nodes, they will need ssh (and associated keys) to communicate 	with each other. 
+	
+	The *prom-timeout* field decides the time in ms after which one of the passive proms 	will take-over as leader after the current leader stops updating MUSIC with 	its health. The *noOfRetryAttempts* is used by prom to decide how many times 	it wants to try and start the local service replica in either active or 	passive mode (by calling the ensure- scripts). The *replicaIdList* is a 	comma separated list of the replica 	ids. Finally, the *musicLocation* should 	contain the public IP of the MUSIC 	node this prom daemon wants to talk to. 	Typically this is localhost if MUSIC is co-located on the same node as the 	prom deamon and service replica. 
+
+        There is an optional *music-connection-timeout-ms* parameter that allows the user to configure the timeout to MUSIC. After this much time has elapsed for connection to MUSIC, the daemon will try the next MUSIC instance (if one exists).
+	
+	The *restart-backoff-time* backs off for the set amount of time in ms if the restart script fails. If configured, this will allow the site time to recover before trying to restart again. This is an optional parameter (default to immediate retry).
+	
+	The *core-monitor-sleep-time* is describes the time in ms between checking if the prom site is active. Default is 0.
+		
+- Once the config.json has been placed on all nodes in the same location as the prom.jar and the startPromDaemon.sh, on one of the nodes (typically the one that you want as active), run the command:
+
+		./startPromDaemon.sh -i <node id>
+		
+	The prom protocol will now take over and start the service replicas in active-passive mode. The prom log can be found on each node i in the logs/EELF/application.log. 
diff --git a/distribution/prom_container.sh b/distribution/prom_container.sh
new file mode 100755
index 0000000..c991d07
--- /dev/null
+++ b/distribution/prom_container.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# ensure that the config.json is in the same location
+# along with ensure active/passive scripts
+
+
+PROM_IMG=prom
+WORKING_DIR=`dirname "$(realpath $0)"`
+
+#
+# change this location if necessary
+# this is the location of the config.json and the ensure active/passive scripts
+#
+CONFIG_SCRIPTS_DIR=$WORKING_DIR
+
+usage () {
+  echo "Usage: $0 <start/stop> <prom id> [-p]"
+}
+
+
+if [ "$#" -lt 2 ]; then
+  usage
+  exit 1
+fi
+
+PROM_ID=$2
+
+
+PROM_PASSIVE="\"\""
+if [[ "$#" -ge 3 && ${3//[-]} == p* ]]; then
+    PROM_PASSIVE="-p"
+fi
+
+
+
+
+
+echo "Starting prom, id:'$PROM_ID'"
+
+if [ "$1" = "start" ];
+then
+  docker run -d --hostname $PROM_ID \
+    -e ID=$PROM_ID \
+    -e PASSIVE=$PASSIVE \
+    --net="host" \
+    -v $CONFIG_SCRIPTS_DIR:/opt/app/prom \
+    $PROM_IMG
+#    --name prom \ 
+elif [ "$1" = "stop" ];
+then
+  docker stop prom;
+  sleep 5;
+else
+  usage
+fi
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..ed5996f
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+/*
+ * 
+This licence applies to all files in this repository unless otherwise specifically
+stated inside of the file. 
+
+   Copyright (c) 2016 AT&T Intellectual Property
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at:
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+ */
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+   <modelVersion>4.0.0</modelVersion>
+   <groupId>com.att.prom</groupId>
+   <artifactId>prom</artifactId>
+   <version>1.0.2</version>
+   <repositories>
+      <repository>
+         <id>maven2-repository.java.net</id>
+         <name>Java.net Repository for Maven</name>
+         <url>http://download.java.net/maven/2/</url>
+         <layout>default</layout>
+      </repository>
+   </repositories>
+   <dependencies>
+      <dependency>
+         <groupId>com.sun.jersey</groupId>
+         <artifactId>jersey-server</artifactId>
+         <version>1.9</version>
+      </dependency>
+      <dependency>
+         <groupId>com.sun.jersey</groupId>
+         <artifactId>jersey-json</artifactId>
+         <version>1.18.1</version>
+      </dependency>
+      <dependency>
+         <groupId>com.owlike</groupId>
+         <artifactId>genson</artifactId>
+         <version>0.99</version>
+      </dependency>
+      <dependency>
+         <groupId>com.sun.jersey</groupId>
+         <artifactId>jersey-client</artifactId>
+         <version>1.9</version>
+      </dependency>
+      <dependency>
+         <groupId>com.sun.jersey</groupId>
+         <artifactId>jersey-json</artifactId>
+         <version>1.9</version>
+      </dependency>
+      <dependency>
+         <groupId>com.googlecode.json-simple</groupId>
+         <artifactId>json-simple</artifactId>
+         <version>1.1</version>
+      </dependency>
+      <dependency>
+         <groupId>commons-cli</groupId>
+         <artifactId>commons-cli</artifactId>
+         <version>1.3.1</version>
+      </dependency>
+      <dependency>
+         <groupId>com.att.eelf</groupId>
+         <artifactId>eelf-core</artifactId>
+         <version>1.0.1-oss</version>
+      </dependency>
+   </dependencies>
+   <properties>
+      <maven.compiler.source>1.7</maven.compiler.source>
+      <maven.compiler.target>1.7</maven.compiler.target>
+   </properties>
+   <build>
+        <resources>
+           <resource>
+              <directory>src/main/resources</directory>
+              <filtering>true</filtering>
+           </resource>
+        </resources>   
+     <finalName>prom</finalName>
+     <plugins>
+       <plugin>
+         <groupId>org.apache.maven.plugins</groupId>
+         <artifactId>maven-shade-plugin</artifactId>
+         <version>3.1.0</version>
+         <configuration>
+           <transformers>
+             <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+             <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+               <mainClass>protocol.PromDaemon</mainClass>
+             </transformer>
+           </transformers>
+         </configuration>
+         <executions>
+           <execution>
+             <phase>package</phase>
+             <goals>
+               <goal>shade</goal>
+             </goals>
+           </execution>
+         </executions>
+       </plugin>
+     </plugins>
+   </build>
+   <!-- Note: For artifact deployment to Maven Central <distributionManagement> below is needed in pom.xml file -->
+   <!-- ================================================================================== -->
+   <!-- Maven Central Repository Information -->
+   <!-- ================================================================================== -->
+   <distributionManagement>
+      <repository>
+         <id>com.att.prom</id>
+         <name>att-repository-releases</name>
+         <url>http://mavencentral.it.att.com:8084/nexus/content/repositories/att-repository-releases</url>
+      </repository>
+      <snapshotRepository>
+         <id>com.att.prom</id>
+         <name>att-repository-snapshots</name>
+         <url>http://mavencentral.it.att.com:8084/nexus/content/repositories/att-repository-snapshots</url>
+      </snapshotRepository>
+   </distributionManagement>
+</project>
+
diff --git a/sampleApp/config.json b/sampleApp/config.json
new file mode 100755
index 0000000..dd3ac8f
--- /dev/null
+++ b/sampleApp/config.json
@@ -0,0 +1,19 @@
+ {

+     "appName":"",

+     "aid":"",

+     "namespace":"",

+     "userid":"",

+     "password":"",

+     "ensure-active-0": "./ensureSdncActive.sh",

+     "ensure-active-1": "./ensureSdncActive.sh",

+     "ensure-passive-0":"./ensureSdncStandby.sh",

+     "ensure-passive-1":"./ensureSdncStandby.sh",

+     "restart-prom-0":"ssh ...",

+     "restart-prom-1":"",

+     "core-monitor-sleep-time":"1000",

+     "prom-timeout":"5000",

+     "noOfRetryAttempts":"3",

+     "replicaIdList":["0", "1"],

+     "musicLocation":"127.0.0.1",

+     "musicVersion":2

+ } 

diff --git a/sampleApp/ensureSdncActive.sh b/sampleApp/ensureSdncActive.sh
new file mode 100755
index 0000000..70202c8
--- /dev/null
+++ b/sampleApp/ensureSdncActive.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+dir=`dirname $0`
+# query SDN-C cluster status
+clusterStatus=$( $dir/sdnc.cluster )
+
+if [ "ACTIVE" = "$clusterStatus" ];then
+  # peform health-check
+  health=$( $dir/sdnc.monitor )
+  
+  if [ "HEALTHY" = "$health" ]; then
+    echo "Cluster is ACTIVE and HEALTHY"
+    exit 0
+  fi
+  echo "Cluster is ACTIVE and UNHEALTHY"
+  exit 1
+
+elif [ "STANDBY" = "$clusterStatus" ]; then
+  # perform takeover
+  echo "Cluster is STANDBY - taking over"
+  takeoverResult=$( $dir/sdnc.failover )
+  if [ "SUCCESS" = "$takeoverResult" ]; then
+    exit 0
+  fi
+  echo "Cluster takeover failed"
+  exit 1
+fi
+
+echo "Unknown cluster status '$clusterStatus'"
+exit 1
diff --git a/sampleApp/ensureSdncStandby.sh b/sampleApp/ensureSdncStandby.sh
new file mode 100755
index 0000000..b9e9864
--- /dev/null
+++ b/sampleApp/ensureSdncStandby.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+dir=`dirname $0` 
+# query SDN-C cluster status
+clusterStatus=$( $dir/sdnc.cluster )
+
+if [ "ACTIVE" = "$clusterStatus" ];then
+  # check that standby cluster is healthy
+  health=$( $dir/sdnc.monitor )
+  if [ "FAILURE" = "$health" ];then
+    echo "Backup site is unhealthy - can't accept traffic!"
+    exit 1
+  fi
+
+  # assume transient error as other side transitions to ACTIVE
+  echo "Cluster is ACTIVE but PROM wants STANDBY! Panic!"
+  exit 0
+
+elif [ "STANDBY" = "$clusterStatus" ]; then
+  echo "Cluster is standing by"
+  exit 0
+fi
+
+echo "Unknown cluster status '$clusterStatus'"
+exit 1
diff --git a/sampleApp/promoverride.py b/sampleApp/promoverride.py
new file mode 100755
index 0000000..ec15590
--- /dev/null
+++ b/sampleApp/promoverride.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python2
+
+# -*- encoding: utf-8 -*-
+# -------------------------------------------------------------------------
+#   Copyright (c) 2018 AT&T Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+
+import sys
+import getopt
+import json
+import requests
+
+musicLocation =""
+base_url = ""
+keyspaceName = ""
+tableName = "replicas"
+aid = ""
+namespace = ""
+
+
+def parseConfig(config):
+    global musicLocation, base_url, keyspaceName, aid, namespace
+    config = json.load(open(config))
+    musicLocations = config["music-location"]
+    base_url = "http://" + musicLocations[0] + ":8080/MUSIC/rest/v2"
+    keyspaceName = "prom_" + config["app-name"]
+    aid = config["aid"]
+    namespace = config["namespace"]
+
+def getHeaders():
+    headers = {'aid': aid, 'ns': namespace}
+    return headers
+
+def getReplica(id):
+    response = requests.get(base_url+"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/rows?id="+id,
+                       headers=getHeaders())
+    return response.json()["result"]["row 0"]
+
+def getAllReplicas():
+    response = requests.get(base_url+"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/rows",
+                       headers=getHeaders())
+    print json.dumps(response.json()["result"], indent=2, sort_keys=True)
+
+def acquireLock(lockref):
+    response = requests.get(base_url+"/locks/acquire/"+lockref,
+                       headers=getHeaders())
+    return response.json()
+
+def releaseLock(lockref):
+    print "releasing lock: " + lockref
+    response = requests.delete(base_url+"/locks/release/"+lockref,
+                       headers=getHeaders())
+    #return response.json()
+    return
+
+def getCurrentLockHolder(lockname):
+    response = requests.get(base_url+"/locks/enquire/"+lockname,
+                        headers=getHeaders())
+    return response.json()
+
+def releaseLocksUntil(lockname, lockref):
+    """release locks until the lockref passed in is the current lock holder
+     this essentially forces the lockref to become the active prom site"""
+    acquire = acquireLock(lockref)
+    while acquire["status"]=="FAILURE":
+        if acquire["message"]=="Lockid doesn't exist":
+            print "[ERROR] Lock" , lockref, "cannot be found."
+            return False
+        currentLockHolder = getCurrentLockHolder(lockname)
+        if currentLockHolder["lock"]["lock-holder"] is not lockref:
+            releaseLock(currentLockHolder["lock"]["lock-holder"])
+        acquire = acquireLock(lockref)
+    return True
+
+def deleteLock(lockname):
+    response = requests.delete(base_url + "/locks/delete/"+lockname,
+                          headers=getHeaders())
+    return response.json()
+
+
+def usage():
+    print "usage: promoverride -c <prom config file> -i <prom_id>"
+    print " -c, --config <prom config file> OPTIONAL location of the 'config.json' file for prom." \
+          " Default location is current directory"
+    print " -i <prom_id> is the replica site to force to become active"
+    print " -l, --list to list current prom instances"
+    print "\n Config file is needed to read information about music location and keyspace information"
+
+if __name__=="__main__":
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "c:i:l", ["config=", "id=", "list"])
+    except getopt.GetoptError as err:
+        print(err)
+        usage()
+        exit(1)
+    # defaults here
+    configFile = "config.json"
+    id = None
+    listInstances = False
+
+    for opt, args in opts:
+        if opt in ("-c", "--config"):
+            configFile = args
+        elif opt in ("-i", "--id"):
+            id = args
+        elif opt in ("-l", "--list"):
+            listInstances = True
+        else:
+            assert False, "unhandled option " + str(opt)
+
+    parseConfig(configFile)
+
+    if listInstances:
+        # todo list current instances
+        getAllReplicas()
+        exit(0)
+
+    if id == None:
+        print "Mandatory prom id not provided."
+        usage()
+        exit(1)
+
+    replicaInfo = getReplica(id)
+    print "Forcing prom site ", id, " to become active"
+    if releaseLocksUntil(keyspaceName+".active.lock", replicaInfo["lockref"]) is True:
+        print "prom site", id, " should now be active"
diff --git a/sampleApp/sdnc.cluster b/sampleApp/sdnc.cluster
new file mode 100755
index 0000000..1734f9a
--- /dev/null
+++ b/sampleApp/sdnc.cluster
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# SDNC Resiliency project
+# SHELL script to determine whether cluster is the active SDNC cluster or the GeoR Stanby SDNC cluster.
+#  The status of the cluster is determined by examining the response data obtained from jolokia calls to the ODL nodes.
+#  return values:
+#         ACTIVE - cluster is the active cluster
+#         STANDBY - cluster is the standby cluster
+#
+echo "ACTIVE"
diff --git a/sampleApp/sdnc.cluster.standby b/sampleApp/sdnc.cluster.standby
new file mode 100755
index 0000000..8ed0566
--- /dev/null
+++ b/sampleApp/sdnc.cluster.standby
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# SDNC Resiliency project
+# SHELL script to determine whether cluster is the active SDNC cluster or the GeoR Stanby SDNC cluster.
+#  The status of the cluster is determined by examining the response data obtained from jolokia calls to the ODL nodes.
+#  return values:
+#         ACTIVE - cluster is the active cluster
+#         STANDBY - cluster is the standby cluster
+#
+echo "STANDBY"
diff --git a/sampleApp/sdnc.failover b/sampleApp/sdnc.failover
new file mode 100755
index 0000000..5a7884f
--- /dev/null
+++ b/sampleApp/sdnc.failover
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+#
+# SDNC Resiliency project
+# SHELL script to execute the SDNC cluster failover tasks
+#
+#  return values:
+#         SUCCESS - failover tasks were executed successfully
+#         FAILURE - failover tasks failed
+# 
+
+echo "SUCCESS"
diff --git a/sampleApp/sdnc.failover.failure b/sampleApp/sdnc.failover.failure
new file mode 100755
index 0000000..56a4f91
--- /dev/null
+++ b/sampleApp/sdnc.failover.failure
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+#
+# SDNC Resiliency project
+# SHELL script to execute the SDNC cluster failover tasks
+#
+#  return values:
+#         SUCCESS - failover tasks were executed successfully
+#         FAILURE - failover tasks failed
+# 
+
+echo "FAILURE"
diff --git a/sampleApp/sdnc.monitor b/sampleApp/sdnc.monitor
new file mode 100755
index 0000000..bc3b73b
--- /dev/null
+++ b/sampleApp/sdnc.monitor
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# SDNC Resiliency project
+# SHELL script to query status of the SDNC cluster
+#
+#  return values:
+#         HEALTHY - the cluster health is determined to be able to successfully process requests
+#         FAILURE - the cluster is in unhealthy state, the execution of failover is required
+#
+echo "HEALTHY"
diff --git a/sampleApp/sdnc.monitor.failure b/sampleApp/sdnc.monitor.failure
new file mode 100755
index 0000000..d6e30ac
--- /dev/null
+++ b/sampleApp/sdnc.monitor.failure
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# SDNC Resiliency project
+# SHELL script to query status of the SDNC cluster
+#
+#  return values:
+#         HEALTHY - the cluster health is determined to be able to successfully process requests
+#         FAILURE - the cluster is in unhealthy state, the execution of failover is required
+#
+echo "FAILURE"
diff --git a/sampleApp/startPromDaemon.sh b/sampleApp/startPromDaemon.sh
new file mode 100755
index 0000000..2eb868f
--- /dev/null
+++ b/sampleApp/startPromDaemon.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+usage () {
+  echo "Usage: $0 -i <prom id> [-p] [-c <config.json directory>] [-z]"
+  echo "where"
+  echo -e "\t -i <prom_id> the identifier of the prom daemon"
+  echo -e "\t -p specifies whether the daemon must start as passive"
+  echo -e "\t -c is the directory where the prom config.json resides"
+  echo -e "\t -z keep std out open after daemon is started (for docker containers only)"
+  exit 1
+}
+
+
+passive="" #default=can be active or passive
+config=$PWD   # default config directory is working directory
+id_flag=0   #make sure user passes in id
+docker_deployment=false
+
+while getopts ":i:pc:z" o; do
+  case "${o}" in
+    i)
+      id=${OPTARG}
+      id_flag=1
+      ;;
+    p)
+      passive="-p"
+      ;;
+    c)
+      config=${OPTARG}
+      ;;
+    z)
+      docker_deployment=true
+      echo "docker deployment"
+      ;;
+    *)
+      usage
+      ;;
+  esac
+done      
+
+if [ $id_flag -eq 0 ]; then
+  echo "ERROR: Required parameter <prom id> not provided."
+  usage
+fi
+
+echo "config location is $config"
+echo "prom id is $id"
+echo "passive is $passive"
+
+if $docker_deployment ; then
+  echo "Container version detected, keeping syso open"
+  #keep container running
+fi
+ 
+dir=$PWD
+ps aux > $dir/PromLog$id.out
+promId=`grep "prom.jar $id" $dir/PromLog$id.out | awk '{ print $2 }'`
+if [ -z "${promId}" ]; then
+#		echo prom dead
+    echo "Starting prom $id"
+    java -jar $dir/prom.jar --id $id $passive --config $config > $dir/prom$id.out & 
+fi
+sleep 3
+ps aux > $dir/PromLog$id.out
+promId=`grep "prom.jar" $dir/PromLog$id.out | awk '{ print $2 }'`
+if [ -z "${promId}" ]; then
+    echo "NotRunning"
+else 
+		echo $promId
+fi
+rm $dir/PromLog$id.out
+
+if $docker_deployment ; then
+  echo "Container version detected, keeping syso open"
+  #keep container running
+  tail -f /dev/null
+fi
diff --git a/src/main/java/org/onap/music/prom/eelf/logging/EELFLoggerDelegate.java b/src/main/java/org/onap/music/prom/eelf/logging/EELFLoggerDelegate.java
new file mode 100644
index 0000000..c4eb801
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/eelf/logging/EELFLoggerDelegate.java
@@ -0,0 +1,329 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.eelf.logging;
+
+import static com.att.eelf.configuration.Configuration.MDC_SERVER_FQDN;
+import static com.att.eelf.configuration.Configuration.MDC_SERVER_IP_ADDRESS;
+import static com.att.eelf.configuration.Configuration.MDC_SERVICE_INSTANCE_ID;
+import static com.att.eelf.configuration.Configuration.MDC_SERVICE_NAME;
+
+import java.net.InetAddress;
+import java.text.MessageFormat;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.att.eelf.configuration.SLF4jWrapper;
+
+public class EELFLoggerDelegate extends SLF4jWrapper implements EELFLogger {
+
+	public static final EELFLogger errorLogger = EELFManager.getInstance().getErrorLogger();
+	public static final EELFLogger applicationLogger = EELFManager.getInstance().getApplicationLogger();
+	public static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
+	public static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
+	public static final EELFLogger debugLogger = EELFManager.getInstance().getDebugLogger();
+
+	private String className;
+	private static ConcurrentMap<String, EELFLoggerDelegate> classMap = new ConcurrentHashMap<>();
+
+	public EELFLoggerDelegate(final String className) {
+		super(className);
+		this.className = className;
+	}
+
+	/**
+	 * Convenience method that gets a logger for the specified class.
+	 * 
+	 * @see #getLogger(String)
+	 * 
+	 * @param clazz
+	 * @return Instance of EELFLoggerDelegate
+	 */
+	public static EELFLoggerDelegate getLogger(Class<?> clazz) {
+		return getLogger(clazz.getName());
+	}
+
+	/**
+	 * Gets a logger for the specified class name. If the logger does not already
+	 * exist in the map, this creates a new logger.
+	 * 
+	 * @param className
+	 *            If null or empty, uses EELFLoggerDelegate as the class name.
+	 * @return Instance of EELFLoggerDelegate
+	 */
+	public static EELFLoggerDelegate getLogger(final String className) {
+		String classNameNeverNull = className == null || "".equals(className) ? EELFLoggerDelegate.class.getName()
+				: className;
+		EELFLoggerDelegate delegate = classMap.get(classNameNeverNull);
+		if (delegate == null) {
+			delegate = new EELFLoggerDelegate(className);
+			classMap.put(className, delegate);
+		}
+		return delegate;
+	}
+
+	/**
+	 * Logs a message at the lowest level: trace.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void trace(EELFLogger logger, String msg) {
+		if (logger.isTraceEnabled()) {
+			logger.trace(msg);
+		}
+	}
+
+	/**
+	 * Logs a message with parameters at the lowest level: trace.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void trace(EELFLogger logger, String msg, Object... arguments) {
+		if (logger.isTraceEnabled()) {
+			logger.trace(msg, arguments);
+		}
+	}
+
+	/**
+	 * Logs a message and throwable at the lowest level: trace.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void trace(EELFLogger logger, String msg, Throwable th) {
+		if (logger.isTraceEnabled()) {
+			logger.trace(msg, th);
+		}
+	}
+
+	/**
+	 * Logs a message at the second-lowest level: debug.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void debug(EELFLogger logger, String msg) {
+		if (logger.isDebugEnabled()) {
+			logger.debug(msg);
+		}
+	}
+
+	/**
+	 * Logs a message with parameters at the second-lowest level: debug.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void debug(EELFLogger logger, String msg, Object... arguments) {
+		if (logger.isDebugEnabled()) {
+			logger.debug(msg, arguments);
+		}
+	}
+
+	/**
+	 * Logs a message and throwable at the second-lowest level: debug.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void debug(EELFLogger logger, String msg, Throwable th) {
+		if (logger.isDebugEnabled()) {
+			logger.debug(msg, th);
+		}
+	}
+
+	/**
+	 * Logs a message at info level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void info(EELFLogger logger, String msg) {
+		logger.info(className + " - "+msg);
+	}
+
+	/**
+	 * Logs a message with parameters at info level.
+	 *
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void info(EELFLogger logger, String msg, Object... arguments) {
+		logger.info(msg, arguments);
+	}
+
+	/**
+	 * Logs a message and throwable at info level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void info(EELFLogger logger, String msg, Throwable th) {
+		logger.info(msg, th);
+	}
+
+	/**
+	 * Logs a message at warn level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void warn(EELFLogger logger, String msg) {
+		logger.warn(msg);
+	}
+
+	/**
+	 * Logs a message with parameters at warn level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void warn(EELFLogger logger, String msg, Object... arguments) {
+		logger.warn(msg, arguments);
+	}
+
+	/**
+	 * Logs a message and throwable at warn level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void warn(EELFLogger logger, String msg, Throwable th) {
+		logger.warn(msg, th);
+	}
+
+	/**
+	 * Logs a message at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 */
+	public void error(EELFLogger logger, String msg) {
+		logger.error(className+ " - " + msg);
+	}
+
+	/**
+	 * Logs a message with parameters at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param arguments
+	 */
+	public void error(EELFLogger logger, String msg, Object... arguments) {
+		logger.warn(msg, arguments);
+	}
+
+	/**
+	 * Logs a message and throwable at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param th
+	 */
+	public void error(EELFLogger logger, String msg, Throwable th) {
+		logger.warn(msg, th);
+	}
+
+	/**
+	 * Logs a message with the associated alarm severity at error level.
+	 * 
+	 * @param logger
+	 * @param msg
+	 * @param severtiy
+	 */
+	public void error(EELFLogger logger, String msg, Object /*AlarmSeverityEnum*/ severtiy) {
+		logger.error(msg);
+	}
+
+	/**
+	 * Initializes the logger context.
+	 */
+	public void init() {
+		setGlobalLoggingContext();
+		final String msg = "############################ Logging is started. ############################";
+		// These loggers emit the current date-time without being told.
+		info(applicationLogger, msg);
+		error(errorLogger, msg);
+		debug(debugLogger, msg);
+        info(auditLogger, msg);
+		info(metricsLogger, msg);
+	}
+
+
+	/**
+	 * Builds a message using a template string and the arguments.
+	 * 
+	 * @param message
+	 * @param args
+	 * @return
+	 */
+	private String formatMessage(String message, Object... args) {
+		StringBuilder sbFormattedMessage = new StringBuilder();
+		if (args != null && args.length > 0 && message != null && message != "") {
+			MessageFormat mf = new MessageFormat(message);
+			sbFormattedMessage.append(mf.format(args));
+		} else {
+			sbFormattedMessage.append(message);
+		}
+
+		return sbFormattedMessage.toString();
+	}
+
+	/**
+	 * Loads all the default logging fields into the MDC context.
+	 */
+	private void setGlobalLoggingContext() {
+		MDC.put(MDC_SERVICE_INSTANCE_ID, "");
+		try {
+			MDC.put(MDC_SERVER_FQDN, InetAddress.getLocalHost().getHostName());
+			MDC.put(MDC_SERVER_IP_ADDRESS, InetAddress.getLocalHost().getHostAddress());
+		} catch (Exception e) {
+			errorLogger.error("setGlobalLoggingContext failed", e);
+		}
+	}
+
+	public static void mdcPut(String key, String value) {
+		MDC.put(key, value);
+	}
+
+	public static String mdcGet(String key) {
+		return MDC.get(key);
+	}
+
+	public static void mdcRemove(String key) {
+		MDC.remove(key);
+	}
+
+}
diff --git a/src/main/java/org/onap/music/prom/main/ConfigReader.java b/src/main/java/org/onap/music/prom/main/ConfigReader.java
new file mode 100644
index 0000000..224f4b2
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/main/ConfigReader.java
@@ -0,0 +1,86 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.main;
+
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+public class ConfigReader {
+	private static String configLocation = "config.json";
+	
+	public static void setConfigLocation(String pathToFile){
+		configLocation = pathToFile+"/config.json";
+	}
+	
+	private static JSONObject getJsonHandle(){
+		JSONParser parser = new JSONParser();
+		Object obj =null;
+		try {
+			obj = parser.parse(new FileReader(configLocation));
+		} catch (FileNotFoundException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (ParseException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		JSONObject jsonObject = (JSONObject) obj;	
+		return jsonObject;
+	}
+
+	public static ArrayList<String> getConfigListAttribute(String key){
+		ArrayList<String> value = (ArrayList<String>) getJsonHandle().get(key);
+		return value; 
+	}
+
+	public static String getConfigAttribute(String key){
+		Object value = getJsonHandle().get(key);
+		return (value!=null) ? String.valueOf(value) : null;
+	}
+	
+	public static String getConfigAttribute(String key, String defaultValue){
+		String toReturn = getConfigAttribute(key);
+		return (toReturn!=null) ? toReturn : defaultValue;
+	}
+
+	public static ArrayList<String> getExeCommandWithParams(String key){
+		String script = (String)getJsonHandle().getOrDefault(key, "");
+		String[] scriptParts = script.split(" ");
+		ArrayList<String> scriptWithPrams = new ArrayList<String>();
+		for(int i=0; i < scriptParts.length;i++)
+			scriptWithPrams.add(scriptParts[i]);
+		return scriptWithPrams;
+	}
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/onap/music/prom/main/PromDaemon.java b/src/main/java/org/onap/music/prom/main/PromDaemon.java
new file mode 100644
index 0000000..f39228b
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/main/PromDaemon.java
@@ -0,0 +1,608 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.main;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.onap.music.prom.eelf.logging.EELFLoggerDelegate;
+import org.onap.music.prom.musicinterface.MusicHandle;
+
+
+public class PromDaemon {
+	
+	private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(PromDaemon.class);
+	
+	String id;
+	String lockName,lockRef;
+	public  enum CoreState {PASSIVE, ACTIVE};
+	public  enum ScriptResult {ALREADY_RUNNING, SUCCESS_RESTART, FAIL_RESTART};
+	String keyspaceName;
+	String tableName;
+	
+	public PromDaemon(String id){
+		this.id = id; 
+		bootStrap();
+	}
+	
+	/** Do not use, only for testing **/
+	PromDaemon(){
+		
+	}
+	
+	private void bootStrap(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "Bootstrapping this site daemon");
+		keyspaceName = "prom_"+ConfigReader.getConfigAttribute("app-name");
+		MusicHandle.createKeyspaceEventual(keyspaceName);
+		
+		tableName = "Replicas";
+		Map<String,String> replicaFields = new HashMap<String,String>();
+		replicaFields.put("id", "text");
+		replicaFields.put("isactive", "boolean");
+		replicaFields.put("timeoflastupdate", "varint");
+		replicaFields.put("lockref", "text");
+		replicaFields.put("PRIMARY KEY", "(id)");
+
+		MusicHandle.createTableEventual(keyspaceName, tableName, replicaFields);	
+		MusicHandle.createIndexInTable(keyspaceName, tableName, "lockref");
+		
+		Map<String,Object> values = new HashMap<String,Object>();
+		values.put("id",this.id);
+		values.put("isactive","false");
+		values.put("timeoflastupdate", "0");
+		//values.put("lockref", "");
+		MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+		//MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+		
+		lockName = keyspaceName+".active.lock";
+	}
+	
+	/**
+	 * Get a lockRef if one doesn't exist. If a lockRef exists, return the same lockRef.
+	 * This is used if the daemon crashes and is able to recover or restart.
+	 * @return the lockRef for this site
+	 */
+	private String getLockRefOrOldLockRefIfExists(){	
+		//first check if a lock reference exists for this id..
+		Map<String,Object> replicaDetails = MusicHandle.readSpecificRow(keyspaceName, tableName, "id", this.id); 
+
+		if (replicaDetails == null || !replicaDetails.containsKey("row 0") 
+					|| !((Map<String,String>) replicaDetails.get("row 0")).containsKey("lockref")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No entry found in MUSIC Replicas table for this daemon.");
+
+			return MusicHandle.createLockRef(lockName);
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, replicaDetails.toString());
+
+		
+		String prevLockRef = ((Map<String, String>) replicaDetails.get("row 0")).get("lockref");
+		if (prevLockRef==null || prevLockRef.equals("")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "Previous running state detected,"
+									+ "but cannot get previous lock reference.");
+			return MusicHandle.createLockRef(lockName);
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, "Previous lock found for this prom replica:"+prevLockRef);
+		return prevLockRef;
+	}
+	
+	
+	/**
+	 * This function maintains the key invariant that it will return true for only one id
+	 * @return true if this replica is current lock holder
+	 */
+	private boolean isActiveLockHolder(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "isActiveLockHolder");
+		boolean isLockHolder = acquireLock();
+		if (isLockHolder) {//update active table
+			logger.info(EELFLoggerDelegate.applicationLogger, "Daemon is the current activeLockHolder");
+			Map<String,Object> values = new HashMap<String,Object>();
+			values.put("isactive","true");
+			values.put("id",this.id);
+			MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+		}
+		return isLockHolder;
+	}
+	
+	/**
+	 * tries to acquire lockRef
+	 * if lockRef no longer exists creates a new lock and updates locally
+	 * @return true if active lock holder, false otherwise
+	 */
+	private boolean acquireLock() {
+		logger.info(EELFLoggerDelegate.applicationLogger, "acquiringLock '" + lockRef +"'");
+		if (lockRef==null) return false;
+		
+		Map<String, Object> result = MusicHandle.acquireLock(lockRef);
+		Map<String, Object> lockMap = (Map<String, Object>) result.get("lock");
+		if (result.get("status").equals("FAILURE") &&
+				result.getOrDefault("message", "Lockid doesn't exist").equals("Lockid doesn't exist")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "Resulting json was: " +result);
+			logger.info(EELFLoggerDelegate.applicationLogger,
+					"Lockref " + lockRef + " doesn't exist, getting new lockref");
+			lockRef = MusicHandle.createLockRef(lockName);
+			logger.info(EELFLoggerDelegate.applicationLogger, "This site's new reference is " + lockRef);
+			result = MusicHandle.acquireLock(lockRef);
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, "result of acquiring lock " + result.get("status"));
+		logger.info(EELFLoggerDelegate.applicationLogger, "Current lock holder is " + MusicHandle.whoIsLockHolder(this.lockName));
+		return (result.get("status").equals("SUCCESS")?true:false);
+	}
+	
+	
+	/**
+	 * The main startup function for each daemon
+	 * @param startPassive dictates whether the node should start in an passive mode
+	 */
+	private void startHAFlow(boolean startPassive){
+		logger.info(EELFLoggerDelegate.applicationLogger, "startHAFlow"+startPassive);
+		if (startPassive) {
+			startAsPassiveReplica();
+		}
+		
+		lockRef = getLockRefOrOldLockRefIfExists();	
+
+		while (true) {
+			if (isActiveLockHolder()) {
+				activeFlow();
+			}
+			else {
+				passiveFlow();
+			}
+		}
+	}
+
+	/**
+	 * Waits until there is an active, running replica
+	 */
+	private void startAsPassiveReplica() {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"Starting in 'passive mode'. Checking to see if active has started");
+		String activeLockRef = MusicHandle.whoIsLockHolder(lockName);
+		Map<String,Object> active = getReplicaDetails(activeLockRef);
+		
+		while (active==null || !(Boolean)active.getOrDefault("isactive", false)
+				 || !isReplicaAlive((String)active.get("id"))) {
+			activeLockRef = MusicHandle.whoIsLockHolder(lockName);
+			active = getReplicaDetails(activeLockRef);
+			//back off if needed
+			try {
+				Long sleeptime = Long.parseLong(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "1000"));
+				if (sleeptime>0) {
+					logger.info(EELFLoggerDelegate.applicationLogger, "Sleeping for " + sleeptime + " ms");
+					Thread.sleep(sleeptime);
+				}
+			} catch (Exception e) {
+					logger.error(e.getMessage());
+			}
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger, 
+				"Active site id=" + active.get("id") + " has started. Continuing in passive mode");
+	}
+	
+	/**
+	 * Make sure that the replica you are monitoring is running by running
+	 * the script provided.
+	 * 
+	 * Try to run the script noOfRetryAttempts times, as defined by the prom configuration.
+	 * This function will wait in between retry attempts, as determined by 'restart-backoff-time' 
+	 * defined in prom configuration file (immediate retry is default, if no value is provided)
+	 * 
+	 * @param script script to be run
+	 * @return ScriptResult based off scripts response
+	 */
+	private ScriptResult tryToEnsureCoreFunctioning(ArrayList<String> script){
+		logger.info(EELFLoggerDelegate.applicationLogger, "tryToEnsureCoreFunctioning");
+		int noOfAttempts = Integer.parseInt(ConfigReader.getConfigAttribute("no-of-retry-attempts"));
+		ScriptResult result = ScriptResult.FAIL_RESTART;
+		
+		while (noOfAttempts > 0) {			
+			result = PromUtil.executeBashScriptWithParams(script);
+			if (result == ScriptResult.ALREADY_RUNNING) {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"Executed core script, the core was already running");
+				return result;
+			} else if (result == ScriptResult.SUCCESS_RESTART) {
+				logger.info(EELFLoggerDelegate.applicationLogger, 
+						"Executed core script, the core had to be restarted");
+				return result;
+			} else if (result == ScriptResult.FAIL_RESTART) {
+				noOfAttempts--;
+				logger.info(EELFLoggerDelegate.applicationLogger, 
+						"Executed core script, the core could not be re-started, retry attempts left ="+noOfAttempts);
+			}
+			//backoff period in between restart attempts
+			try {
+				Thread.sleep(Long.parseLong(ConfigReader.getConfigAttribute("restart-backoff-time", "0")));
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
+		}
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"Tried enough times and still unable to start the core, giving up lock and starting passive flow..");
+		return result;
+	}
+	
+	/**
+	 * Update this replica's lockRef and update the heartbeat in replica table
+	 */
+	private void updateHealth(CoreState isactive) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "updateHealth " +isactive);
+		Map<String,Object> values = new HashMap<String,Object>();
+		values.put("id",this.id);
+		values.put("timeoflastupdate", System.currentTimeMillis());
+		values.put("lockref", this.lockRef);
+		values.put("isactive",isactive==CoreState.ACTIVE?true:false);
+		MusicHandle.insertIntoTableEventual(keyspaceName, tableName, values);
+	}
+	
+	/**
+	 * Checks to see if the replica is alive
+	 * @param id the id of the replica to check if is alive
+	 * @return
+	 */
+	private boolean isReplicaAlive(String id){
+		logger.info(EELFLoggerDelegate.applicationLogger, "isReplicaAlive " + id);
+		Map<String,Object> valueMap = MusicHandle.readSpecificRow(keyspaceName, tableName, "id", id);
+		if (valueMap == null || valueMap.isEmpty()) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No entry showing...");
+			return false; 
+		}
+		valueMap = (Map<String, Object>) valueMap.get("row 0");
+		
+		if (!valueMap.containsKey("timeoflastupdate") || valueMap.get("timeoflastupdate")==null) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No 'timeoflastupdate' entry showing...");
+			return false;
+		}
+
+		long lastUpdate = (Long)valueMap.get("timeoflastupdate");
+		logger.info(EELFLoggerDelegate.applicationLogger, id + "'s time of last update:"+lastUpdate);
+	    long timeOutPeriod = PromUtil.getPromTimeout();
+	    long currentTime = System.currentTimeMillis();
+	    logger.info(EELFLoggerDelegate.applicationLogger, "current time:"+currentTime);
+	    long timeSinceUpdate = currentTime-lastUpdate; 
+	    logger.info(EELFLoggerDelegate.applicationLogger, id + "'s time since update:"+timeSinceUpdate);
+	    if(timeSinceUpdate > timeOutPeriod) {
+	    	return false;
+	    } else {
+	    	return true;
+	    }
+	}
+	
+	private Map<String, Object> getReplicaDetails(String lockRef){
+		Map<String,Object> details = MusicHandle.readSpecificRow(keyspaceName, tableName, "lockref", lockRef);
+		if (details==null) { return null; }
+		return (Map<String, Object>) details.getOrDefault("row 0", null);
+	}
+
+	/**
+	 * Releases lock and ensures replica id's 'isactive' state to false
+	 * @param lockRef
+	 */
+	private void releaseLock(String lockRef){
+		logger.info(EELFLoggerDelegate.applicationLogger, "releaseLock " + lockRef);
+		if(lockRef == null){
+			logger.info(EELFLoggerDelegate.applicationLogger, "There is no lock entry..");
+			return;
+		}
+			
+		if(lockRef.equals("")){
+			logger.info(EELFLoggerDelegate.applicationLogger, "Already unlocked..");
+			return;
+		}
+		
+		Map<String, Object> replicaDetails = getReplicaDetails(lockRef);
+		String replicaId = "UNKNOWN";
+		if (replicaDetails!=null) {
+			replicaId = (String)replicaDetails.get("id");
+		}
+		
+		
+		logger.info(EELFLoggerDelegate.applicationLogger, "Unlocking prom "+replicaId + " with lockref"+ lockRef);
+		MusicHandle.unlock(lockRef);
+		logger.info(EELFLoggerDelegate.applicationLogger, "Unlocked prom "+replicaId);
+		if (replicaId.equals(this.id)) { //if unlocking myself, remove reference to lockref
+			this.lockRef=null;
+		}
+		
+		if (replicaId.equals("UNKNOWN")) {
+			return;
+		}
+		//create entry in replicas table
+		Map<String,Object> values = new HashMap<String,Object>();
+		values.put("isactive",false);		
+		values.put("lockref", "");
+		MusicHandle.updateTableEventual(keyspaceName, tableName, "id", replicaId, values);
+	}
+	
+	private void tryToEnsurePeerHealth(){
+		ArrayList<String> replicaList =  ConfigReader.getConfigListAttribute(("replica-id-list"));
+		for (Iterator<String> iterator = replicaList.iterator(); iterator.hasNext();) {
+			String replicaId = (String) iterator.next();
+			if(replicaId.equals(this.id) == false){
+				if(isReplicaAlive(replicaId) == false){
+					//restart if suspected dead
+					//releaseLock(replicaId);
+					//Don't hold up main thread for restart
+					Runnable restartThread = new RestartThread(replicaId);
+					new Thread(restartThread).start();
+					
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							lockRef + " status: "+MusicHandle.acquireLock(lockRef));
+				}
+			}
+		}	
+	}
+	
+	private boolean restartPromDaemon(String replicaId, int noOfAttempts){
+		logger.info(EELFLoggerDelegate.applicationLogger, "Prom Daemon--"+replicaId+"--needs to be restarted");
+
+		ArrayList<String> restartScript = ConfigReader.getExeCommandWithParams("restart-prom-"+replicaId);
+		if (restartScript!=null && restartScript.size()>0 && restartScript.get(0).length()>0) {
+			PromUtil.executeBashScriptWithParams(restartScript);
+		}
+		return true;//need to find a way to check if the script is running. Just check if process is running maybe? 
+/*
+		boolean result = false;
+		while(result == false){
+			ArrayList<String> restartScript = ConfigReader.getExeCommandWithParams("restart-prom-"+id);
+			PromUtil.executeBashScriptWithParams(restartScript);
+			result = Boolean.parseBoolean(resultString);
+			noOfAttempts--;
+			if(noOfAttempts <= 0)
+				break; 
+		}
+		return result; 
+*/	}
+	
+	/**
+	 * Give current active sufficient time (as defined by configured 'prom-timeout' value) to become passive.
+	 * If current active does not become passive in the configured amount of time, the current active site
+	 * is forcibly reset to a passive state.
+	 * 
+	 * This method should only be called after the lock of the previous active is released and this 
+	 * replica has become the new active
+	 * 
+	 * @param currentActiveId
+	 */
+	private void takeOverFromCurrentActive(String currentActiveLockRef){
+		if (currentActiveLockRef==null || currentActiveLockRef.equals(this.lockRef)) {
+			return;
+		}
+		
+		long startTime = System.currentTimeMillis();
+		long restartTimeout = PromUtil.getPromTimeout();
+		while(true){
+			Map<String,Object> replicaDetails = getReplicaDetails(currentActiveLockRef);
+			if (replicaDetails==null || !replicaDetails.containsKey("isactive") ||
+					!(Boolean)replicaDetails.get("isactive")) {
+				break;
+			}
+			
+			//waited long enough..just make the old active passive yourself
+			if ((System.currentTimeMillis() - startTime) > restartTimeout) {
+				logger.info(EELFLoggerDelegate.applicationLogger, 
+						"Old Active not responding..resetting Music state of old active to passive myself");
+				Map<String, Object> removeActive = new HashMap<String,Object>();
+				removeActive.put("isactive", false);
+				MusicHandle.updateTableEventual(keyspaceName, tableName, "lockref", currentActiveLockRef, removeActive);
+				break;
+			}
+			//make sure we don't time out while we wait
+			updateHealth(CoreState.PASSIVE);
+		}
+
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"Old Active has now become passive, so starting active flow ***");
+
+		//now you can take over as active! 
+	}
+	
+	
+	private void activeFlow(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "activeFlow");
+		while (true) {
+			if(acquireLock() == false){
+				logger.info(EELFLoggerDelegate.applicationLogger, "I no longer have the lock! Make myself passive");
+				return;
+			}
+			
+			ScriptResult result = tryToEnsureCoreFunctioning(ConfigReader.getExeCommandWithParams("ensure-active-"+id));
+			if (result == ScriptResult.ALREADY_RUNNING) {
+				//do nothing
+			} else if (result == ScriptResult.SUCCESS_RESTART) {
+				//do nothing
+			} else if (result == ScriptResult.FAIL_RESTART) {//unable to start core, just give up and become passive
+				releaseLock(lockRef);
+				return;
+			}
+			
+			updateHealth(CoreState.ACTIVE);
+			
+			logger.info(EELFLoggerDelegate.applicationLogger, 
+					"--(Active) Prom Daemon--"+id+"---CORE ACTIVE---Lock Ref:"+lockRef);
+
+			tryToEnsurePeerHealth();
+			logger.info(EELFLoggerDelegate.applicationLogger, 
+					"--(Active) Prom Daemon--"+id+"---PEERS CHECKED---");
+
+			//back off if needed
+			try {
+				Long sleeptime = Long.parseLong(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "0"));
+				if (sleeptime>0) {
+					logger.info(EELFLoggerDelegate.applicationLogger, "Sleeping for " + sleeptime + " ms");
+					Thread.sleep(sleeptime);
+				}
+			} catch (Exception e) {
+					logger.error(e.getMessage());
+			}
+		}
+	}
+	
+	private void passiveFlow(){
+		logger.info(EELFLoggerDelegate.applicationLogger, "passiveFlow");
+		while(true){
+			ScriptResult result = tryToEnsureCoreFunctioning(ConfigReader.getExeCommandWithParams("ensure-passive-"+id));		
+			if (result == ScriptResult.ALREADY_RUNNING) {
+				if (lockRef==null) {
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							"Replica does not have a lock, but is running. Getting a lock now");
+					lockRef = MusicHandle.createLockRef(lockName);
+					logger.info(EELFLoggerDelegate.applicationLogger, "new lockRef " + lockRef);
+				}
+			} else if (result == ScriptResult.SUCCESS_RESTART) {
+				//we can now handle being after, put yourself back in queue
+				logger.info(EELFLoggerDelegate.applicationLogger, "Script successfully restarted. Getting a new lock");
+				lockRef = MusicHandle.createLockRef(lockName);
+				logger.info(EELFLoggerDelegate.applicationLogger, "new lockRef " + lockRef);
+			} else if (result == ScriptResult.FAIL_RESTART) {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"Site not working and could not restart, releasing lock"); 
+				releaseLock(lockRef);
+			}
+			
+			//update own health in music
+			updateHealth(CoreState.PASSIVE);
+			
+			logger.info(EELFLoggerDelegate.applicationLogger,
+					"-- {Passive} Prom Daemon--"+id+"---CORE PASSIVE---Lock Ref:"+lockRef);
+
+			//obtain active lock holder's id
+			String activeLockRef = MusicHandle.whoIsLockHolder(lockName);
+			releaseLockIfActiveIsDead(activeLockRef);
+			
+			if (isActiveLockHolder()) {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"***I am the active lockholder, so taking over from previous active***");
+				takeOverFromCurrentActive(activeLockRef);
+				return;
+			}
+
+			//back off if needed
+			try {
+				Long sleeptime = Long.parseLong(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "0"));
+				if (sleeptime>0) {
+					logger.info(EELFLoggerDelegate.applicationLogger, "Sleeping for " + sleeptime + " ms");
+					Thread.sleep(sleeptime);
+				}
+			} catch (Exception e) {
+				logger.error(e.getMessage());
+			}
+		}
+	}
+
+	/**
+	 * Releases the lock if the active lock holder is dead, not responsive, or cannot be found.
+	 * @param activeLockRef
+	 * @return the active id
+	 */
+	private void releaseLockIfActiveIsDead(String activeLockRef) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "releaseLockIfActiveIsDead " + activeLockRef);
+		Map<String, Object> activeDetails =  getReplicaDetails(activeLockRef);
+		Boolean activeIsAlive = false;
+		String activeId = null;
+		if (activeDetails!=null) {
+			activeId = (String)activeDetails.get("id");
+			logger.info(EELFLoggerDelegate.applicationLogger, "Active lockholder is site " + activeId);
+			activeIsAlive = isReplicaAlive(activeId);
+		}
+
+		if (activeIsAlive == false) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "Active lockholder is not alive");
+			if (activeId==null) {
+				if (activeLockRef!=null && !activeLockRef.equals("")) {
+					//no reference to the current lock, probably corrupt/stale data
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							"Unknown active lockholder. Releasing current lock");
+					MusicHandle.unlock(activeLockRef);
+				} else {
+					logger.info(EELFLoggerDelegate.applicationLogger,
+							"*****No lock holders. Make sure there are healthy sites*****");
+				}
+			} else {
+				logger.info(EELFLoggerDelegate.applicationLogger,
+						"Active " + activeId + " is suspected dead. Releasing it's lock.");
+				releaseLock(activeLockRef);
+			}
+		}
+	}
+	
+	private class RestartThread implements Runnable{
+		String replicaId;
+		public RestartThread(String replicaId) {
+			this.replicaId = replicaId;
+		}
+		public void run() {
+			restartPromDaemon(this.replicaId, 1);
+		}
+	}
+	
+	public static void main(String[] args){
+		Options opts = new Options();
+		
+		Option idOpt = new Option("i", "id", true, "prom identifier");
+		idOpt.setRequired(true);
+		opts.addOption(idOpt);
+		
+		Option passive = new Option("p", "passive", false, "start prom in passive mode (default false)");
+		opts.addOption(passive);
+		
+		Option config = new Option("c", "config", true, "location of config.json file (default same directory as prom jar)");
+		opts.addOption(config);
+		
+		CommandLineParser parser = new DefaultParser();
+		HelpFormatter formatter = new HelpFormatter();
+		CommandLine cmd;
+		try {
+			cmd = parser.parse(opts, args);
+		} catch (ParseException e) {
+			e.printStackTrace();
+			formatter.printHelp("prom", opts);
+			System.exit(1);
+			return;
+		}
+		
+		String id = cmd.getOptionValue("id");
+		boolean startPassive = false;
+		if (cmd.hasOption("passive")) {
+			startPassive = true;
+		}
+		if (cmd.hasOption("c")) {
+			ConfigReader.setConfigLocation(cmd.getOptionValue("c"));
+		}
+
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"--Prom Daemon version "+PromUtil.version+"--replica id "+id+"---START---"+(startPassive?"passive":"active"));
+		PromDaemon hd = new PromDaemon(id);
+		hd.startHAFlow(startPassive);
+	}
+
+}
diff --git a/src/main/java/org/onap/music/prom/main/PromUtil.java b/src/main/java/org/onap/music/prom/main/PromUtil.java
new file mode 100644
index 0000000..a0be5d7
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/main/PromUtil.java
@@ -0,0 +1,174 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.main;
+
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Properties;
+
+import org.onap.music.prom.eelf.logging.EELFLoggerDelegate;
+import org.onap.music.prom.main.PromDaemon.ScriptResult;
+import org.onap.music.prom.musicinterface.MusicHandle;
+
+
+
+public class PromUtil {
+	private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(PromDaemon.class);
+
+	public static String version;
+	static {
+		try {
+			final Properties properties = new Properties();
+			properties.load(PromUtil.class.getClassLoader().getResourceAsStream("project.properties"));
+			version = properties.getProperty("version");
+			logger.info(EELFLoggerDelegate.applicationLogger, "Prom version " + version);
+		} catch (IOException e) {
+			logger.error(e.getMessage());
+		}
+	}
+	
+	private static ArrayList<String> getMusicNodeIp(){
+		return ConfigReader.getConfigListAttribute("music-location");
+/*		String serverAddress;
+		serverAddress = agaveMusicNode;
+		while(isHostUp(serverAddress) != true)
+			serverAddress = toggle(serverAddress);
+		return serverAddress;
+*/	}
+	
+/*	public static String toggle(String serverAddress){
+		if(serverAddress.equals(agaveMusicNode)){
+			System.out.println("Agave is down...connect to Big Site");
+			serverAddress = bigSiteMusicNode;
+		}else if(serverAddress.equals(bigSiteMusicNode)){
+			System.out.println("Big Site is down...connect to Agave");
+			serverAddress = agaveMusicNode;
+		}
+		return serverAddress;
+	}*/
+	
+	public static ArrayList<String> getMusicNodeURL(){
+		ArrayList<String> ips = getMusicNodeIp();
+		ArrayList<String> urls = new ArrayList<String>();
+		for (String ip: ips) {
+			urls.add( "http://"+ip+":8080/MUSIC/rest/v" +PromUtil.getMusicVersion());
+		}
+		return urls;
+	}
+	
+	public static String getMusicVersion() {
+		String version = ConfigReader.getConfigAttribute("music-version", "2");
+		if (version==null) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"No MUSIC Version provided in your configuration file. Please "
+					+ "include 'musicVersion' in your config.json file.");
+			throw new Error("Required property 'music-version' is not provided");
+		}
+		return version;
+	}
+	
+	public static boolean isHostUp(String serverAddress) { 
+		Boolean isUp = false;
+	    try {
+			InetAddress inet = InetAddress.getByName(serverAddress);
+			isUp = inet.isReachable(1000);	
+		} catch (UnknownHostException e) {
+			logger.error(e.getMessage());
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			logger.error(e.getMessage());
+		}
+	    return isUp;
+	}
+	
+	
+	
+	/* MUSIC authentication functions */
+	public static String getAid() {
+		return ConfigReader.getConfigAttribute("aid", "");
+	}
+	
+	public static String getAppNamespace() {
+		return ConfigReader.getConfigAttribute("namespace", "");
+	}
+	
+	public static String getUserId() {
+		return ConfigReader.getConfigAttribute("userid", "");
+	}
+	
+	public static String getPassword() {
+		return ConfigReader.getConfigAttribute("password", "");
+	}
+	/* End of MUSIC authentication functions */
+	
+	public static int getPromTimeout() {
+		return Integer.parseInt(ConfigReader.getConfigAttribute("prom-timeout"));
+	}
+	
+	/**
+	 * Gets 'music-connection-timeout-ms' property from configuration file, returning a negative number if 
+	 * it doesn't exist
+	 * @return
+	 */
+	public static int getTimeoutToMusicMillis() {
+		return Integer.parseInt(ConfigReader.getConfigAttribute("music-connection-timeout-ms", "-1"));
+	}
+	
+	public static ScriptResult executeBashScriptWithParams(ArrayList<String> script){
+		logger.info(EELFLoggerDelegate.applicationLogger, "executeBashScript: " + script);
+		try {
+			ProcessBuilder pb = new ProcessBuilder(script);
+			final Process process = pb.start();
+			int exitCode = process.waitFor();
+
+			StringBuffer errorOutput = new StringBuffer();
+            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
+            String line = "";                       
+            while ((line = reader.readLine())!= null) {
+            		if(!line.equals(""))
+            			errorOutput.append(line + "\n");
+            }
+            System.out.print(errorOutput);
+			if (exitCode == 0)
+				return ScriptResult.ALREADY_RUNNING;
+			else if (exitCode == 1)
+				return ScriptResult.FAIL_RESTART;
+			else if (exitCode == 2)
+				return ScriptResult.SUCCESS_RESTART;
+
+		} catch (IOException e) {
+			logger.error("PromUtil executingBashScript: " + e.getMessage());
+		} catch (InterruptedException e) {
+			logger.error("PromUtil executingBashScript: " + e.getMessage());
+		}
+		return ScriptResult.FAIL_RESTART;
+	}
+	
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonDelete.java b/src/main/java/org/onap/music/prom/musicinterface/JsonDelete.java
new file mode 100644
index 0000000..11d3408
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonDelete.java
@@ -0,0 +1,60 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+public class JsonDelete {
+	
+    private ArrayList<String> columns = null;
+    private Map<String,String> consistencyInfo;
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+    public ArrayList<String> getColumns() {
+		return columns;
+	}
+	public void setColumns(ArrayList<String> columns) {
+		this.columns = columns;
+	}
+	String ttl, timestamp;
+
+	public String getTtl() {
+		return ttl;
+	}
+	public void setTtl(String ttl) {
+		this.ttl = ttl;
+	}
+	public String getTimestamp() {
+		return timestamp;
+	}
+	public void setTimestamp(String timestamp) {
+		this.timestamp = timestamp;
+	}
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonInsert.java b/src/main/java/org/onap/music/prom/musicinterface/JsonInsert.java
new file mode 100644
index 0000000..443da5a
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonInsert.java
@@ -0,0 +1,66 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+
+import java.util.Map;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+public class JsonInsert {
+	
+    private Map<String,Object> values;
+    String ttl, timestamp;
+	private Map<String,Object> row_specification;
+    private Map<String,String> consistencyInfo;
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+	public String getTtl() {
+		return ttl;
+	}
+	public void setTtl(String ttl) {
+		this.ttl = ttl;
+	}
+	public String getTimestamp() {
+		return timestamp;
+	}
+	public void setTimestamp(String timestamp) {
+		this.timestamp = timestamp;
+	}
+	public Map<String, Object> getValues() {
+		return values;
+	}
+	public void setValues(Map<String, Object> values) {
+		this.values = values;
+	}
+    public Map<String, Object> getRow_specification() {
+		return row_specification;
+	}
+	public void setRow_specification(Map<String, Object> row_specification) {
+		this.row_specification = row_specification;
+	}
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonKeySpace.java b/src/main/java/org/onap/music/prom/musicinterface/JsonKeySpace.java
new file mode 100644
index 0000000..9b91021
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonKeySpace.java
@@ -0,0 +1,57 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+
+import java.util.Map;
+
+
+public class JsonKeySpace {
+    private Map<String,Object> replicationInfo;
+	private String durabilityOfWrites;
+    private Map<String,String> consistencyInfo;
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+	public Map<String, Object> getReplicationInfo() {
+		return replicationInfo;
+	}
+	
+	public void setReplicationInfo(Map<String, Object> replicationInfo) {
+		this.replicationInfo = replicationInfo;
+	}
+
+	public String getDurabilityOfWrites() {
+		return durabilityOfWrites;
+	}
+	public void setDurabilityOfWrites(String durabilityOfWrites) {
+		this.durabilityOfWrites = durabilityOfWrites;
+	}
+		
+	
+
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/JsonTable.java b/src/main/java/org/onap/music/prom/musicinterface/JsonTable.java
new file mode 100644
index 0000000..d4f8cc7
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/JsonTable.java
@@ -0,0 +1,64 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+import java.util.Map;
+
+public class JsonTable {
+    private Map<String,String> fields;
+	private Map<String, Object> properties; 
+    private String clusteringOrder;
+    private Map<String,String> consistencyInfo;
+
+
+	public Map<String, String> getConsistencyInfo() {
+		return consistencyInfo;
+	}
+
+	public void setConsistencyInfo(Map<String, String> consistencyInfo) {
+		this.consistencyInfo = consistencyInfo;
+	}
+
+    public Map<String, Object> getProperties() {
+		return properties;
+	}
+
+	public void setProperties(Map<String, Object> properties) {
+		this.properties = properties;
+	}
+    
+	public Map<String, String> getFields() {
+		return fields;
+	}
+
+	public void setFields(Map<String, String> fields) {
+		this.fields = fields;
+	}
+
+	public String getClusteringOrder() {
+		return clusteringOrder;
+	}
+
+	public void setClusteringOrder(String clusteringOrder) {
+		this.clusteringOrder = clusteringOrder;
+	}
+
+}
diff --git a/src/main/java/org/onap/music/prom/musicinterface/MusicHandle.java b/src/main/java/org/onap/music/prom/musicinterface/MusicHandle.java
new file mode 100644
index 0000000..d62a1d2
--- /dev/null
+++ b/src/main/java/org/onap/music/prom/musicinterface/MusicHandle.java
@@ -0,0 +1,709 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+package org.onap.music.prom.musicinterface;
+import java.net.ConnectException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+
+import org.onap.music.prom.eelf.logging.EELFLoggerDelegate;
+import org.onap.music.prom.main.ConfigReader;
+import org.onap.music.prom.main.PromUtil;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+
+public class MusicHandle {
+	private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(MusicHandle.class);
+
+
+	/**
+	 * Adds MUSIC's authentication headers into the webresource
+	 * @param webResource
+	 */
+	private static Builder addMusicHeaders(WebResource webResource) {
+		String aid = PromUtil.getAid();
+		String namespace = PromUtil.getAppNamespace();
+		String userId = PromUtil.getUserId();
+		String password = PromUtil.getPassword();
+		Builder builder = webResource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON);
+		if (!aid.equals("")) {
+			builder.header("aid", aid);
+		}
+		if (!namespace.equals("")) {
+			builder.header("ns", namespace);
+		}
+		if (!userId.equals("")) {
+			builder.header("userId", userId);
+		}
+		if (!password.equals("")) {
+			builder.header("password", password);
+		}
+		
+		return builder;
+	}
+	
+	private static WebResource createMusicWebResource(String musicAPIurl) {
+		ClientConfig clientConfig = new DefaultClientConfig();
+
+		clientConfig.getFeatures().put(
+				JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE);
+		int timeout = getMaxConnectionTimeout();
+		clientConfig.getProperties().put(ClientConfig.PROPERTY_CONNECT_TIMEOUT, timeout);
+		clientConfig.getProperties().put(ClientConfig.PROPERTY_READ_TIMEOUT, timeout);
+
+		Client client = Client.create(clientConfig);
+		return client.resource(musicAPIurl);
+	}
+	
+	public static void createKeyspaceEventual(String keyspaceName){
+		logger.info(EELFLoggerDelegate.applicationLogger, "createKeyspaceEventual "+keyspaceName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				createKeyspaceEventual(musicUrl, keyspaceName);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create keyspace."
+							+ "Could not successfully reach any music instance to create keyspace.");
+		return;
+	}
+	
+	private static void createKeyspaceEventual(String musicUrl, String keyspaceName){
+		Map<String,Object> replicationInfo = new HashMap<String, Object>();
+		replicationInfo.put("class", "SimpleStrategy");
+		replicationInfo.put("replication_factor", 3);
+		String durabilityOfWrites="true";
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+		JsonKeySpace jsonKp = new JsonKeySpace();
+		jsonKp.setConsistencyInfo(consistencyInfo);
+		jsonKp.setDurabilityOfWrites(durabilityOfWrites);
+		jsonKp.setReplicationInfo(replicationInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName);
+
+		ClientResponse response = addMusicHeaders(webResource)
+									.post(ClientResponse.class, jsonKp);
+		
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			if (output.containsKey("error")) {
+				String errorMsg = String.valueOf(output.get("error"));
+				if (errorMsg.equals("err:Keyspace prom_sdnc already exists")) {
+					logger.warn(EELFLoggerDelegate.applicationLogger, 
+							"Not creating keyspace " + keyspaceName + " because it already exists. Continuing.");
+					//assume keyspace is already created and continue
+				}
+				else { //unhandled/unknown exception
+					logger.error(EELFLoggerDelegate.errorLogger,
+							"Failed to createKeySpaceEventual : Status Code "+ output.toString());
+					throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+				}
+			} else { //no exception message
+				logger.error(EELFLoggerDelegate.errorLogger,
+						"Failed to createKeySpaceEventual : Status Code "+ output.toString());
+				throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+			}
+		}
+	}
+
+	public static void createTableEventual(String keyspaceName, String tableName, Map<String,String> fields) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"createKeyspaceEventual "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				createTableEventual(musicUrl, keyspaceName, tableName, fields);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create table. "
+						+ "Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void createTableEventual(String musicUrl, String keyspaceName,
+										String tableName, Map<String,String> fields) {
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonTable jtab = new JsonTable();
+		jtab.setFields(fields);
+		jtab.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName);
+
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class, jtab);
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			if (output.containsKey("error")) {
+				String error = String.valueOf(output.get("error"));
+				if (error.equalsIgnoreCase("Table " + keyspaceName + "." + tableName + " already exists")) {
+					logger.warn(EELFLoggerDelegate.applicationLogger, 
+							"Not creating table " + tableName + " because it already exists. Continuing.");
+				} else { //unhandled exception message
+					logger.error(EELFLoggerDelegate.errorLogger,
+							"Failed to createTableEventual : Status Code "+ output.toString());
+					throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+				}
+			} else { //no exception message, MUSIC should give more info if failure
+				logger.error(EELFLoggerDelegate.errorLogger, 
+						"Failed to createTableEventual : Status Code "+ output.toString());
+				throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+			}
+		}
+	}
+
+	public static void createIndexInTable(String keyspaceName, String tableName, String colName) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"createIndexInTable "+keyspaceName+" tableName "+tableName + " colName" + colName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				createIndexInTable(musicUrl, keyspaceName, tableName, colName);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create index in table. "
+									+ "Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void createIndexInTable(String musicUrl, String keyspaceName, String tableName, String colName) {
+		WebResource webResource = createMusicWebResource(musicUrl +
+							"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/index/"+colName);
+
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class);
+
+		if (response.getStatus() != 200 && response.getStatus() != 204) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to createIndexInTable : Status Code " + response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+	}
+
+	public static void insertIntoTableEventual(String keyspaceName, String tableName, Map<String,Object> values) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"insertIntoTableEventual "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				insertIntoTableEventual(musicUrl, keyspaceName, tableName, values);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to insert into table."
+								+ " Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void insertIntoTableEventual(String musicUrl, String keyspaceName,
+											String tableName, Map<String,Object> values) {
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonInsert jIns = new JsonInsert();
+		jIns.setValues(values);
+		jIns.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/rows");
+
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class, jIns);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to insertIntoTableEventual : Status Code " + response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to createKeySpaceEventual : Status Code "+ output.toString());
+			throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+		}
+	}
+
+	public static void updateTableEventual(String keyspaceName, String tableName, String keyName,
+													String keyValue, Map<String,Object> values) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "updateTableEventual "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				updateTableEventual(musicUrl, keyspaceName, tableName, keyName, keyValue, values);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to update the table. "
+									+ "Could not successfully reach any music instance.");
+	}
+
+	
+	private static void updateTableEventual(String musicUrl, String keyspaceName, String tableName,
+									String keyName, String keyValue, Map<String,Object> values) {		
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonInsert jIns = new JsonInsert();
+		jIns.setValues(values);
+		jIns.setConsistencyInfo(consistencyInfo);
+		
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName
+								+"/tables/"+tableName+"/rows?"+keyName+"="+keyValue);
+
+		ClientResponse response = addMusicHeaders(webResource).put(ClientResponse.class, jIns);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to updateTableEventual : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		if (!output.containsKey("status") || !output.get("status").equals("SUCCESS")) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to createKeySpaceEventual : Status Code "+ output.toString());
+			throw new RuntimeException("Failed: MUSIC Response " + output.toString());
+		}
+	}
+
+	public static Map<String,Object> readSpecificRow(String keyspaceName, String tableName,
+			String keyName, String keyValue) {
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		Map<String, Object> result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = readSpecificRow(musicUrl, keyspaceName, tableName, keyName, keyValue);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to read row. "
+							+ "Could not successfully reach any music instance.");
+		result = null;
+		return result;
+	}
+	
+	private static Map<String,Object> readSpecificRow(String musicUrl, String keyspaceName, String tableName,
+												String keyName, String keyValue) {
+		logger.info(EELFLoggerDelegate.applicationLogger,
+				"readSpecificRow "+keyspaceName+" tableName "+tableName + " key" +keyName + " value" + keyValue);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName
+											+"/tables/"+tableName+"/rows?"+keyName+"="+keyValue);
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to insertIntoTableEventual : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+
+		Map<String, Object> rowMap = (Map<String, Object>) output.getOrDefault("result", null);
+
+		return rowMap;	
+	}
+
+	public static Map<String,Object> readAllRows(String keyspaceName, String tableName) {
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		Map<String, Object> result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = readAllRows(musicUrl, keyspaceName, tableName);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to read all rows. "
+								+ "Could not successfully reach any music instance.");
+		result = null;
+		return result;
+	}
+	
+	private static Map<String,Object> readAllRows(String musicUrl, String keyspaceName, String tableName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "readAllRows "+keyspaceName+" tableName "+tableName);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName+"/rows");
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to readAllRows : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+		return output;	
+	}
+
+	public static void dropTable(String keyspaceName, String tableName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "dropTable "+keyspaceName+" tableName "+tableName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				dropTable(musicUrl, keyspaceName, tableName);
+				return;
+			}  catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to drop table."
+							+ " Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void dropTable(String musicUrl, String keyspaceName, String tableName) {
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonTable jsonTb = new JsonTable();
+		jsonTb.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName+"/tables/"+tableName);
+	
+		ClientResponse response = addMusicHeaders(webResource).delete(ClientResponse.class, jsonTb);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to dropTable : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+	}
+
+	public static void dropKeySpace(String keyspaceName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "dropKeySpace "+keyspaceName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				dropKeySpace(musicUrl, keyspaceName);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to drop keyspace."
+							+ " Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void dropKeySpace(String musicUrl, String keyspaceName) {		
+		Map<String,String> consistencyInfo= new HashMap<String, String>();
+		consistencyInfo.put("type", "eventual");
+
+		JsonKeySpace jsonKp = new JsonKeySpace();
+		jsonKp.setConsistencyInfo(consistencyInfo);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/keyspaces/"+keyspaceName);
+
+		ClientResponse response = addMusicHeaders(webResource).delete(ClientResponse.class, jsonKp);
+
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to dropTable : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : "+ response.getStatus());
+		}
+	}
+
+	public static String createLockRef(String lockName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "createLockRef "+lockName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		String result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = createLockRef(musicUrl, lockName);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to create lock reference. "
+											+ "Could not successfully reach any music instance.");
+		result = "";
+		return result;
+	}
+	
+	private static String createLockRef(String musicUrl, String lockName) {
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/create/"+lockName);
+		
+		ClientResponse response = addMusicHeaders(webResource).post(ClientResponse.class);
+
+		if (response.getStatus() != 200 ) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to createLockRef : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+		Map<String,Object> responseMap = response.getEntity(Map.class);
+		if (!responseMap.containsKey("status") || !responseMap.get("status").equals("SUCCESS") ||
+				!responseMap.containsKey("lock")) {
+			logger.error(EELFLoggerDelegate.errorLogger, 
+					"Failed to createLockRef : Status Code "+ responseMap.toString());
+			return "";
+		}
+		String lockRef = ((Map<String, String>) responseMap.get("lock")).get("lock");
+		logger.info(EELFLoggerDelegate.applicationLogger, "This site's lockReference is "+lockRef);
+		return lockRef;
+	}
+
+	/**
+	 * Try to acquire the lock lockid.
+	 * If cannot reach any music instance, return status:FAILURE
+	 * @param lockId
+	 * @return
+	 */
+	public static Map<String,Object> acquireLock(String lockId) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "acquireLock "+lockId);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		Map<String, Object> result;
+
+		for (String musicUrl: musicUrls) {
+			try {
+				result = acquireLock(musicUrl, lockId);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to acquireLock. Could not successfully reach any music instance.");
+		result = new HashMap<String, Object>();
+		result.put("status", "FAILURE");
+		return result;
+	}
+	
+	private static Map<String,Object> acquireLock(String musicUrl, String lockId){
+		//should be fixed in MUSIC, but putting patch here too
+		if (lockId==null) {
+			Map<String,Object> fail = new HashMap<String, Object>();
+			fail.put("status", "FAILURE");
+			return fail;
+		}
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/acquire/"+lockId);
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() != 200) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to acquireLock : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+		Map<String,Object> output = response.getEntity(Map.class);
+
+		return output;
+	}
+
+	public static String whoIsLockHolder(String lockName) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "whoIsLockHolder "+lockName);
+		
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		String result;
+		for (String musicUrl: musicUrls) {
+			try {
+				result = whoIsLockHolder(musicUrl, lockName);
+				return result;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to check who the lock holder is. "
+											+ "Could not successfully reach any music instance.");
+		result = null;
+		return result;
+	}
+	
+	private static String whoIsLockHolder(String musicUrl, String lockName) {	
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/enquire/"+lockName);
+
+		ClientResponse response = addMusicHeaders(webResource).get(ClientResponse.class);
+
+		if (response.getStatus() != 200) {
+			logger.error(EELFLoggerDelegate.errorLogger,
+					"Failed to determine whoIsLockHolder : Status Code "+response.getStatus());
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+
+		Map<String,String> lockoutput = (Map<String, String>) response.getEntity(Map.class).get("lock");
+		if (lockoutput.get("lock-holder").equals("No lock holder!")) {
+			logger.info(EELFLoggerDelegate.applicationLogger, "No lock holder");
+			return null;
+		}
+		return (String) lockoutput.get("lock-holder");
+	}
+
+	public static void unlock(String lockId) {
+		ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+		for (String musicUrl: musicUrls) {
+			try {
+				unlock(musicUrl, lockId);
+				return;
+			} catch (ClientHandlerException che) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, "Timed out connection to MUSIC. Consider setting"
+						+ " 'music-connection-timeout-ms' in the configuration");
+			} catch (RuntimeException e) {
+				logger.warn(EELFLoggerDelegate.applicationLogger, e.getMessage());
+			}
+			logger.warn(EELFLoggerDelegate.applicationLogger, "Could not reach music at '" + musicUrl +"'");
+		}
+		logger.error(EELFLoggerDelegate.errorLogger, "Unable to unlock the lock. "
+								+ "Could not successfully reach any music instance.");
+		return;
+	}
+	
+	private static void unlock(String musicUrl, String lockId) {
+		logger.info(EELFLoggerDelegate.applicationLogger, "unlock "+lockId);
+
+		WebResource webResource = createMusicWebResource(musicUrl+"/locks/release/"+lockId);
+
+		ClientResponse response = addMusicHeaders(webResource).delete(ClientResponse.class);
+
+		Map<String,Object> responseMap = response.getEntity(Map.class);
+		
+		if (response.getStatus() < 200 || response.getStatus() > 299) {
+			logger.error(EELFLoggerDelegate.errorLogger, "Failed to unlock : Status Code "+response.getStatus());
+			if (responseMap.containsKey("error")) {
+				logger.error(EELFLoggerDelegate.errorLogger, "Failed to unlock : "+responseMap.get("error"));
+			}
+			throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
+		}
+	}
+
+
+	
+	/**
+	 * Gets a connection timeout to music. This function will return the 
+	 * configured parameter given in the prom json config, if available.
+	 * Otherwise, it will calculate a timeout such that it the connection will be able
+	 * to cycle through the different music locations prior to other nodes assuming this
+	 * replica is dead.
+	 * @return
+	 */
+	private static int getMaxConnectionTimeout() {
+		int timeout = PromUtil.getTimeoutToMusicMillis();
+		if (timeout<=0) { // user hasn't defined a valid timeout
+			ArrayList<String> musicUrls = PromUtil.getMusicNodeURL();
+			int promTimeout = PromUtil.getPromTimeout();
+			timeout = promTimeout/musicUrls.size();
+		}
+		return timeout;
+	}
+	
+	
+	public static void main(String[] args){
+		Map<String,Object> results = MusicHandle.readAllRows("votingappbharath", "replicas");
+		for (Map.Entry<String, Object> entry : results.entrySet()){
+			Map<String, Object> valueMap = (Map<String, Object>)entry.getValue();
+			for (Map.Entry<String, Object> rowentry : valueMap.entrySet()){
+				if(rowentry.getKey().equals("timeoflastupdate")){
+					System.out.println(rowentry.getValue());
+				}
+				break;
+			}
+			break;
+		}
+	}
+
+}
+
+
+
diff --git a/src/main/resources/project.properties b/src/main/resources/project.properties
new file mode 100644
index 0000000..a2273e2
--- /dev/null
+++ b/src/main/resources/project.properties
@@ -0,0 +1,2 @@
+version=${project.version}
+artifactId=${project.artifactId}
\ No newline at end of file
diff --git a/src/test/java/org/onap/music/prom/main/PromDaemonTest.java b/src/test/java/org/onap/music/prom/main/PromDaemonTest.java
new file mode 100644
index 0000000..be2d798
--- /dev/null
+++ b/src/test/java/org/onap/music/prom/main/PromDaemonTest.java
@@ -0,0 +1,361 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+
+package org.onap.music.prom.main;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.hamcrest.core.IsAnything;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.VerificationModeFactory;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.onap.music.prom.main.ConfigReader;
+import org.onap.music.prom.main.PromDaemon;
+import org.onap.music.prom.main.PromUtil;
+import org.onap.music.prom.main.PromDaemon.CoreState;
+import org.onap.music.prom.musicinterface.MusicHandle;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.reflect.Whitebox;
+
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({MusicHandle.class, ConfigReader.class, PromUtil.class})
+public class PromDaemonTest {
+	static PromDaemon promDaemon;
+
+	@BeforeClass
+	public static void beforeClass() {
+		promDaemon = Mockito.spy(PromDaemon.class);
+	}
+	
+	@Before
+	public void before() {
+		promDaemon.lockName = "lockName";
+		promDaemon.keyspaceName = "keyspaceName";
+		promDaemon.id = "anIdToTestFor";
+		
+		PowerMockito.mockStatic(ConfigReader.class);
+		PowerMockito.when(ConfigReader.getConfigAttribute("prom-timeout")).thenReturn("1000");
+		PowerMockito.when(ConfigReader.getConfigAttribute("core-monitor-sleep-time", "1000")).thenReturn("1");
+	}
+	
+	
+	@Test
+	public void bootstrapTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		PowerMockito.mockStatic(ConfigReader.class);
+		
+		PowerMockito.when(ConfigReader.getConfigAttribute("app-name")).thenReturn("testing");
+		
+		Whitebox.invokeMethod(promDaemon,  "bootStrap");
+		
+		assertEquals("prom_testing", promDaemon.keyspaceName);
+		assertEquals("prom_testing.active.lock", promDaemon.lockName);
+	}
+	
+	@Test
+	public void acquireLockTrue() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		falseMap.put("status", "SUCCESS");
+		falseMap.put("message", "You already have the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean acquireLock = Whitebox.invokeMethod(promDaemon,  "acquireLock");
+		assertTrue(acquireLock);
+	}
+	
+	@Test
+	public void acquireLockFalse() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		falseMap.put("status", "FAILURE");
+		falseMap.put("message", "you don't own the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean acquireLock = Whitebox.invokeMethod(promDaemon,  "acquireLock");
+		assertFalse(acquireLock);
+	}
+	
+	@Test
+	public void acquireNullLock() throws Exception {
+		promDaemon.lockRef = null;
+		
+		Boolean acquireLock = Whitebox.invokeMethod(promDaemon,  "acquireLock");
+		assertFalse(acquireLock);
+	}
+
+	@Test
+	public void activeLockHolderTestTrue() throws Exception{
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		falseMap.put("status", "SUCCESS");
+		falseMap.put("message", "You already have the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean isActiveLockHolder = Whitebox.invokeMethod(promDaemon,  "isActiveLockHolder");
+		assertTrue(isActiveLockHolder);
+	}
+	
+	@Test
+	public void activeLockHolderTestFalse() throws Exception{
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		falseMap.put("status", "FAILURE");
+		falseMap.put("message", "You do not own the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(falseMap);	
+		
+		Boolean isActiveLockHolder = Whitebox.invokeMethod(promDaemon,  "isActiveLockHolder");
+		assertFalse(isActiveLockHolder);
+	}
+	
+	@Test
+	public void activeLockHolderTestStaleLock() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		HashMap<String, Object> staleLockMap = new HashMap<String, Object>();
+		promDaemon.lockRef = "testLock";
+		staleLockMap.put("status", "FAILURE");
+		staleLockMap.put("message", "Lockid doesn't exist");
+		PowerMockito.when(MusicHandle.acquireLock("testLock")).thenReturn(staleLockMap);
+		
+		PowerMockito.when(MusicHandle.createLockRef("lockName")).thenReturn("testLock2");
+		
+		HashMap<String, Object> falseMap = new HashMap<String, Object>();
+		falseMap.put("status", "FAILURE");
+		falseMap.put("message", "You do not own the lock");
+		PowerMockito.when(MusicHandle.acquireLock("testLock2")).thenReturn(falseMap);
+		
+		Boolean isActiveLockHolder = Whitebox.invokeMethod(promDaemon,  "isActiveLockHolder");
+		assertFalse(isActiveLockHolder);
+		assertEquals("testLock2", promDaemon.lockRef);
+	}
+	
+	
+	@Test
+	public void releaseLockTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		Whitebox.invokeMethod(promDaemon, "releaseLock", null);
+		Whitebox.invokeMethod(promDaemon, "releaseLock", "");
+		
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
+			.thenReturn(null);
+		Whitebox.invokeMethod(promDaemon, "releaseLock", "lock1");
+		
+		//should actually release now
+		ArgumentCaptor<Map> mapCaptor = ArgumentCaptor.forClass(Map.class);
+		HashMap<String,Object> map = new HashMap<String,Object>();
+		HashMap<String,Object> repDetails = new HashMap<String,Object>();
+		repDetails.put("id", promDaemon.id);
+		map.put("row 0", repDetails);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
+		.thenReturn(map);
+		Whitebox.invokeMethod(promDaemon, "releaseLock", "lock1");
+		
+		PowerMockito.verifyStatic();
+		MusicHandle.updateTableEventual(Mockito.anyString(), Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), mapCaptor.capture());
+		assertEquals(false, mapCaptor.getValue().get("isactive"));
+	}
+
+	@Test
+	public void activeHealthTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+
+		ArgumentCaptor<String> keyspaceCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<String> tableCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<Map> mapCaptor = ArgumentCaptor.forClass(Map.class);
+		Whitebox.invokeMethod(promDaemon, "updateHealth", CoreState.ACTIVE);
+
+		PowerMockito.verifyStatic();
+		MusicHandle.insertIntoTableEventual(keyspaceCaptor.capture(), tableCaptor.capture(), mapCaptor.capture());
+		Map<String, Object> returnedMap = mapCaptor.getValue();
+
+		assertTrue((Boolean) returnedMap.get("isactive"));
+		assertTrue(System.currentTimeMillis()-500< (long) returnedMap.get("timeoflastupdate"));
+	}
+	
+	@Test
+	public void passiveHealthTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+
+		ArgumentCaptor<String> keyspaceCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<String> tableCaptor = ArgumentCaptor.forClass(String.class);
+		ArgumentCaptor<Map> mapCaptor = ArgumentCaptor.forClass(Map.class);
+		Whitebox.invokeMethod(promDaemon, "updateHealth", CoreState.PASSIVE);
+
+		PowerMockito.verifyStatic();
+		MusicHandle.insertIntoTableEventual(keyspaceCaptor.capture(), tableCaptor.capture(), mapCaptor.capture());
+		Map<String, Object> returnedMap = mapCaptor.getValue();
+
+		assertFalse((Boolean) returnedMap.get("isactive"));
+		//make sure call was somewhat recent, as synched with current system clock
+		//may need to make this more strict or less strict depending
+		assertTrue(System.currentTimeMillis()-500< (long) returnedMap.get("timeoflastupdate"));
+	}
+	
+
+	@Test
+	public void replicaIsAliveTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		
+		//no test replica
+		Boolean isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+		
+		//return null pointer
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(null);
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+
+		//is active is dead
+		Map<String,Object> deadReplica = new HashMap<String,Object>();
+		Map<String,Object> replicaInfo = new HashMap<String,Object>();
+		replicaInfo.put("id", "testReplica");
+		replicaInfo.put("isactive", "false");
+		deadReplica.put("row 0", replicaInfo);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(deadReplica);
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+		
+		//timed out
+		replicaInfo.put("timeoflastupdate", System.currentTimeMillis()-1000);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(deadReplica);
+
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertFalse(isReplicaAlive);
+		
+		//alive
+		replicaInfo.put("timeoflastupdate", System.currentTimeMillis());
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(),
+				Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(deadReplica);
+		PowerMockito.when(ConfigReader.getConfigAttribute("prom-timeout")).thenReturn("1000");
+		isReplicaAlive = Whitebox.invokeMethod(promDaemon,  "isReplicaAlive", "testReplica");
+		assertTrue(isReplicaAlive);
+	}
+	
+	
+	/**
+	 * try to start as passive. First iteration will fail because the replica is stale.
+	 * Second iteration should exit the method. In failure cases, this might throw an 
+	 * exception to prevent an infinite loop.
+	 * 
+	 * @throws Exception
+	 */
+	@Test
+	public void startAsPassiveReplicaTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		String activeLock = "actLock";
+		Map<String, Object> staleActiveMap = new HashMap<String, Object>();
+		Map<String, Object> staleInfo = new HashMap<String, Object>();
+		staleInfo.put("id", "activeReplica");
+		staleInfo.put("isactive", true);
+		staleInfo.put("timeoflastupdate", System.currentTimeMillis()-1001);
+		staleActiveMap.put("row 0", staleInfo);
+		Map<String, Object> activeActiveMap = new HashMap<String, Object>();
+		Map<String, Object> activeInfo = new HashMap<String, Object>();
+		activeInfo.put("id", "activeReplica");
+		activeInfo.put("isactive", true);
+		activeInfo.put("timeoflastupdate", System.currentTimeMillis());
+		activeActiveMap.put("row 0", activeInfo);
+		
+		PowerMockito.when(MusicHandle.whoIsLockHolder(promDaemon.lockName)).thenReturn(activeLock);
+		PowerMockito.when(MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
+			.thenReturn(staleActiveMap).thenReturn(staleActiveMap)
+			.thenReturn(activeActiveMap).thenReturn(activeActiveMap)
+			.thenThrow(new RuntimeException("Should exit before reaching here"));
+		
+		Whitebox.invokeMethod(promDaemon,  "startAsPassiveReplica");
+		
+		//make sure we went through 2 iterations. Each iteration makes 2 calls to readSpecific row so 2x2 is 4
+		PowerMockito.verifyStatic(VerificationModeFactory.times(4));
+		MusicHandle.readSpecificRow(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+	}
+	
+	
+	@Test
+	public void getLockRefOrOldLockRefIfExistsTest() throws Exception {
+		PowerMockito.mockStatic(MusicHandle.class);
+		
+		//no entry in music
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef1");
+		String lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef1", lockref);
+		
+		//entry in music doesn't have lockref column
+		Map<String, Object> entriesInMusic = new HashMap<String, Object>();
+		Map<String, Object> entry = new HashMap<String, Object>();
+		entry.put("id", promDaemon.id);
+		entriesInMusic.put("row 0", entry);
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+				.thenReturn(entriesInMusic);
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef2");
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef2", lockref);
+
+		//entry in music didn't previously have a lockref
+		entry.put("lockref", null);
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+		.thenReturn(entriesInMusic);
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef3");
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef3", lockref);
+		
+		//entry in music didn't previously have a lockref
+		entry.put("lockref", "");
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+				.thenReturn(entriesInMusic);
+		PowerMockito.when(MusicHandle.createLockRef(promDaemon.lockName)).thenReturn("aNewLockRef4");
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("aNewLockRef4", lockref);
+		
+		//entry had a previous lock entry
+		entry.put("lockref", "previousLockRef");
+		PowerMockito.when(MusicHandle.readSpecificRow(promDaemon.keyspaceName, promDaemon.tableName, "id", promDaemon.id))
+				.thenReturn(entriesInMusic);
+		lockref = Whitebox.invokeMethod(promDaemon, "getLockRefOrOldLockRefIfExists");
+		assertEquals("previousLockRef", lockref);
+	}
+}
diff --git a/src/test/java/org/onap/music/prom/musicinterface/MusicHandleTest.java b/src/test/java/org/onap/music/prom/musicinterface/MusicHandleTest.java
new file mode 100644
index 0000000..48f75c3
--- /dev/null
+++ b/src/test/java/org/onap/music/prom/musicinterface/MusicHandleTest.java
@@ -0,0 +1,160 @@
+/*
+ * ============LICENSE_START==========================================
+ * org.onap.music.prom
+ * ===================================================================
+ *  Copyright (c) 2018 AT&T Intellectual Property
+ * ===================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ * ============LICENSE_END=============================================
+ * ====================================================================
+ */
+
+package org.onap.music.prom.musicinterface;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.onap.music.prom.main.ConfigReader;
+import org.onap.music.prom.main.PromUtil;
+import org.onap.music.prom.musicinterface.MusicHandle;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.api.client.config.ClientConfig;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ConfigReader.class, Client.class, ClientResponse.class,
+		WebResource.class, WebResource.Builder.class, PromUtil.class})
+public class MusicHandleTest {
+	
+	ClientResponse response;
+	Client client;
+	WebResource webresource;
+	WebResource.Builder webresourceBuilder;
+	
+	
+	@Before
+	public void before() throws Exception {
+		PowerMockito.mockStatic(ConfigReader.class);
+		ArrayList<String> urls = new ArrayList<String>();
+		Collections.addAll(urls, "1.2.3.4", "5.6.7.8");
+		PowerMockito.when(ConfigReader.getConfigAttribute(Mockito.anyString(), Mockito.anyString()))
+			.thenCallRealMethod();
+		
+		PowerMockito.mockStatic(PromUtil.class);
+		//PowerMockito.spy(PromUtil.class);
+		PowerMockito.when(PromUtil.getPromTimeout()).thenReturn(Integer.parseInt("1000"));
+		PowerMockito.when(PromUtil.getMusicNodeURL()).thenReturn(urls);
+		PowerMockito.when(PromUtil.getAid()).thenReturn("");
+		PowerMockito.when(PromUtil.getAppNamespace()).thenReturn("");
+		PowerMockito.when(PromUtil.getUserId()).thenReturn("");
+		PowerMockito.when(PromUtil.getPassword()).thenReturn("");
+		
+		
+		response = PowerMockito.mock(ClientResponse.class);
+		PowerMockito.mockStatic(Client.class);
+		client = PowerMockito.mock(Client.class);
+		webresource = PowerMockito.mock(WebResource.class);
+		webresourceBuilder = PowerMockito.mock(WebResource.Builder.class);
+		
+		//PowerMockito.when(Client.create()).thenReturn(client);
+		PowerMockito.when(Client.create((ClientConfig) Mockito.anyObject())).thenReturn(client);
+		PowerMockito.when(client.resource(Mockito.anyString())).thenReturn(webresource);
+		PowerMockito.when(webresource.accept(MediaType.APPLICATION_JSON)).thenReturn(webresourceBuilder);
+		PowerMockito.when(webresourceBuilder.type(MediaType.APPLICATION_JSON)).thenReturn(webresourceBuilder);
+		PowerMockito.when(webresourceBuilder.get(ClientResponse.class)).thenReturn(response);
+		PowerMockito.when(webresourceBuilder.post(ClientResponse.class)).thenReturn(response);
+
+	}
+
+	@Test
+	public void acquireLockTestFailure() {
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "FAILURE");
+		PowerMockito.when(response.getStatus()).thenReturn(200);
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		Map<String, Object> result = MusicHandle.acquireLock("testLock");
+		assertEquals("FAILURE", result.get("status"));
+	}
+	
+	@Test
+	public void acquireLockTestFailureCannotReachFirstMusic() {
+		PowerMockito.when(response.getStatus()).thenReturn(404, 404, 404, 200);
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "SUCCESS");
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		Map<String, Object> result = MusicHandle.acquireLock("testLock");
+		assertEquals("SUCCESS", result.get("status"));
+	}
+	
+	@Test
+	public void acuireLockTestCannotReachAnyMusic() {
+		PowerMockito.when(response.getStatus()).thenReturn(404);
+		Map<String, Object> result = MusicHandle.acquireLock("testLock");
+		assertEquals("FAILURE", result.get("status"));
+	}
+
+
+	@Test
+	public void createLockRefSuccess() {
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "SUCCESS");
+		Map<String, Object> lockMap = new HashMap<String, Object>();
+		lockMap.put("lock", "abc_lockref");
+		acquireLockResponse.put("lock", lockMap);
+		PowerMockito.when(response.getStatus()).thenReturn(200);
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		
+		String result = MusicHandle.createLockRef("testLock");
+		assertEquals("abc_lockref", result);
+	}
+	
+	@Test
+	public void createLockRefFailure() {
+		//Fail all music instances
+		PowerMockito.when(response.getStatus()).thenReturn(404);
+		String result = MusicHandle.createLockRef("testLock");
+		assertEquals("", result);
+	}
+	
+	@Test
+	public void createLockRefFailFirstMusic() {
+		PowerMockito.when(response.getStatus()).thenReturn(404, 404, 200);
+		
+		Map<String, Object> acquireLockResponse = new HashMap<String, Object>();
+		acquireLockResponse.put("status", "SUCCESS");
+		Map<String, Object> lockMap = new HashMap<String, Object>();
+		lockMap.put("lock", "abc_lockref");
+		acquireLockResponse.put("lock", lockMap);
+		PowerMockito.when(response.getEntity(Map.class)).thenReturn(acquireLockResponse);
+		
+		String result = MusicHandle.createLockRef("testLock");
+		assertEquals("abc_lockref", result);
+	}
+}