[Work in Progress] Add Heal NS API endpoint

This patch set adds the Heal NS API for VNF Healing.

Issue-Id: VFC-39
Change-Id: I0889b361e3222019791445e4ec81883c4f1f904d
Signed-off-by: Shashank Kumar Shankar <shashank.kumar.shankar@intel.com>

11 files changed, 450 insertions, 4 deletions

diff --git a/lcm/ns/const.py b/lcm/ns/const.py
index a0e9973d..b4165e5e 100644
--- a/lcm/ns/const.py
+++ b/lcm/ns/const.py
@@ -16,4 +16,5 @@ from lcm.pub.utils.enumutil import enum
 OWNER_TYPE = enum(VNF=0, VNFM=1, NS=2)
 
 NS_INST_STATUS = enum(EMPTY='empty', INSTANTIATING='instantiating', TERMINATING='terminating',
-                      ACTIVE='active', FAILED='failed', INACTIVE='inactive', UPDATING='updating', SCALING='scaling')
+                      ACTIVE='active', FAILED='failed', INACTIVE='inactive', UPDATING='updating', SCALING='scaling',
+                      HEALING='healing')
diff --git a/lcm/ns/ns_heal.py b/lcm/ns/ns_heal.py
new file mode 100644
index 00000000..6973e466
--- /dev/null
+++ b/lcm/ns/ns_heal.py
@@ -0,0 +1,117 @@
+# Copyright 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import threading
+import traceback
+import datetime
+import time
+
+from lcm.ns.const import NS_INST_STATUS
+from lcm.pub.database.models import JobModel, NSInstModel
+from lcm.ns.vnfs.heal_vnfs import NFHealService
+from lcm.pub.exceptions import NSLCMException
+from lcm.pub.utils.jobutil import JobUtil, JOB_MODEL_STATUS
+from lcm.pub.utils.values import ignore_case_get
+
+JOB_ERROR = 255
+logger = logging.getLogger(__name__)
+
+
+class NSHealService(threading.Thread):
+    def __init__(self, ns_instance_id, request_data, job_id):
+        super(NSHealService, self).__init__()
+        self.ns_instance_id = ns_instance_id
+        self.request_data = request_data
+        self.job_id = job_id
+
+        self.heal_vnf_data = ''
+
+    def run(self):
+        try:
+            self.do_biz()
+        except NSLCMException as e:
+            JobUtil.add_job_status(self.job_id, JOB_ERROR, e.message)
+        except:
+            logger.error(traceback.format_exc())
+            JobUtil.add_job_status(self.job_id, JOB_ERROR, 'ns heal fail')
+
+    def do_biz(self):
+        self.update_job(1, desc='ns heal start')
+        self.update_ns_status(NS_INST_STATUS.HEALING)
+        self.get_and_check_params()
+        self.do_vnfs_heal()
+        self.update_ns_status(NS_INST_STATUS.ACTIVE)
+        self.update_job(100, desc='ns heal success')
+
+    def get_and_check_params(self):
+        self.heal_vnf_data = ignore_case_get(self.request_data, 'healVnfData')
+        if not self.heal_vnf_data:
+            logger.error('healVnfData parameter does not exist or value is incorrect.')
+            raise NSLCMException('healVnfData parameter does not exist or value incorrect.')
+
+    def do_vnfs_heal(self):
+        vnf_heal_params = self.prepare_vnf_heal_params(self.heal_vnf_data)
+        count = len(self.heal_vnf_data)
+        # TODO(sshank): Check progress_range
+        progress_range = [11 + 80 / count, 10 + 80 / count]
+        status = self.do_vnf_heal(vnf_heal_params, progress_range)
+        if status is JOB_MODEL_STATUS.FINISHED:
+            logger.info('nf[%s] heal handle end' % vnf_heal_params.get('vnfInstanceId'))
+            self.update_job(progress_range[1],
+                            desc='nf[%s] heal handle end' % vnf_heal_params.get('vnfInstanceId'))
+        else:
+            logger.error('nf heal failed')
+            raise NSLCMException('nf heal failed')
+
+    def do_vnf_heal(self, vnf_heal_params, progress_range):
+        vnf_instance_id = vnf_heal_params.get('vnfInstanceId')
+        nf_service = NFHealService(vnf_instance_id, vnf_heal_params)
+        nf_service.start()
+        self.update_job(progress_range[0], desc='nf[%s] heal handle start' % vnf_instance_id)
+        status = self.wait_job_finish(nf_service.job_id)
+        return status
+
+    def prepare_vnf_heal_params(self, vnf_data):
+        vnf_instance_id = ignore_case_get(vnf_data, 'vnfInstanceId')
+        cause = ignore_case_get(vnf_data, "cause")
+        additional_params = ignore_case_get(vnf_data, "additionalParams")
+        result = {
+            "vnfInstanceId": vnf_instance_id,
+            "cause": cause,
+            "additionalParams": additional_params
+        }
+        return result
+
+    @staticmethod
+    def wait_job_finish(sub_job_id, timeout=3600):
+        query_interval = 2
+        start_time = end_time = datetime.datetime.now()
+        while (end_time - start_time).seconds < timeout:
+            job_result = JobModel.objects.get(jobid=sub_job_id)
+            time.sleep(query_interval)
+            end_time = datetime.datetime.now()
+            if job_result.progress == 100:
+                return JOB_MODEL_STATUS.FINISHED
+            elif job_result.progress > 100:
+                return JOB_MODEL_STATUS.ERROR
+            else:
+                continue
+        return JOB_MODEL_STATUS.TIMEOUT
+
+    def update_job(self, progress, desc=''):
+        JobUtil.add_job_status(self.job_id, progress, desc)
+
+    def update_ns_status(self, status):
+        NSInstModel.objects.filter(id=self.ns_instance_id).update(status=status)
diff --git a/lcm/ns/swagger.json b/lcm/ns/swagger.json
index 430a04d8..11754206 100644
--- a/lcm/ns/swagger.json
+++ b/lcm/ns/swagger.json
@@ -510,6 +510,51 @@
                 }
             }
         },
+        "/ns/{ns_instance_id}/heal": {
+            "post": {
+                "tags": [
+                    "ns heal"
+                ],
+                "summary": "ns heal",
+                "description": "ns heal",
+                "operationId": "ns_heal",
+                "consumes": [
+                    "application/json"
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "parameters": [
+                    {
+                        "required": true,
+                        "type": "string",
+                        "description": "Identifier of the NS instance.",
+                        "name": "ns_instance_id",
+                        "in": "path"
+                    },
+                    {
+                        "in": "body",
+                        "name": "healVnfData",
+                        "description": "healVnfData",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/healVnfDataRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "202": {
+                        "description": "",
+                        "schema": {
+                            "$ref": "#/definitions/healVnfDataResponse"
+                        }
+                    },
+                    "500": {
+                        "description": "the url is invalid"
+                    }
+                }
+            }
+        },
         "/mandb/{modelName}": {
             "get": {
                 "tags": [
@@ -1005,6 +1050,44 @@
                 }
             }
         },
+        "healVnfDataRequest": {
+            "type": "object",
+            "properties": {
+                "vnfInstanceId": {
+                    "type": "string"
+                },
+                "cause": {
+                    "type": "string"
+                },
+                "additionalParams": {
+                    "type": "object",
+                    "properties": {
+                        "action": {
+                            "type": "string"
+                        },
+                        "actionvminfo": {
+                            "type": "object",
+                            "properties": {
+                                "vmid": {
+                                    "type": "string"
+                                },
+                                "vmname": {
+                                    "type": "string"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "healVnfDataResponse": {
+            "type": "object",
+            "properties": {
+                "jobId": {
+                    "type": "string"
+                }
+            }
+        },
         "TableInfo": {
             "type": "object",
             "properties": {
diff --git a/lcm/ns/tests/test_ns_heal.py b/lcm/ns/tests/test_ns_heal.py
new file mode 100644
index 00000000..87cd174d
--- /dev/null
+++ b/lcm/ns/tests/test_ns_heal.py
@@ -0,0 +1,65 @@
+# Copyright 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import uuid
+from rest_framework import status
+from django.test import TestCase
+from django.test import Client
+from lcm.pub.database.models import NSDModel, NSInstModel
+from lcm.pub.utils.jobutil import JobUtil, JOB_TYPE
+from lcm.ns.const import NS_INST_STATUS
+from lcm.pub.utils import restcall
+from lcm.ns.ns_heal import NSHealService
+
+
+class TestHealNsViews(TestCase):
+    def setUp(self):
+        self.nsd_id = str(uuid.uuid4())
+        self.ns_package_id = str(uuid.uuid4())
+        self.ns_inst_id = str(uuid.uuid4())
+        self.job_id = JobUtil.create_job("NS", JOB_TYPE.HEAL_VNF, self.ns_inst_id)
+        NSDModel(id=self.ns_package_id, nsd_id=self.nsd_id, name='name').save()
+
+        self.client = Client()
+        self.context = '{"vnfs": ["a", "b"], "sfcs": ["c"], "vls": ["d", "e", "f"]}'
+        NSInstModel(id=self.ns_inst_id, name="abc", nspackage_id="7", nsd_id="111").save()
+
+    def tearDown(self):
+        NSInstModel.objects.filter().delete()
+
+    @mock.patch.object(NSHealService, 'run')
+    def test_ns_heal(self, mock_run):
+        data = {
+            'nsdid': self.nsd_id,
+            'nsname': 'ns',
+            'description': 'description'}
+        response = self.client.post("/api/nslcm/v1/ns/%s/heal" % self.nsd_id, data=data)
+        self.failUnlessEqual(status.HTTP_202_ACCEPTED, response.status_code)
+
+    @mock.patch.object(restcall, 'call_req')
+    def test_ns_heal_thread(self, mock_call):
+
+        data = {
+            'nsdid': self.nsd_id,
+            'nsname': 'ns',
+            'description': 'description'
+        }
+
+        NSHealService(self.ns_inst_id, data, self.job_id).run()
+        self.assertEqual(NSInstModel.objects.get(id=self.ns_inst_id).status, NS_INST_STATUS.HEALING)
+
+    def test_swagger_ok(self):
+        resp = self.client.get("/api/nslcm/v1/swagger.json", format='json')
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
diff --git a/lcm/ns/tests/vnfs/tests.py b/lcm/ns/tests/vnfs/tests.py
index 8ac7c962..4b78963b 100644
--- a/lcm/ns/tests/vnfs/tests.py
+++ b/lcm/ns/tests/vnfs/tests.py
@@ -28,6 +28,7 @@ from lcm.pub.utils.timeutil import now_time
 from lcm.pub.utils.values import ignore_case_get
 from lcm.ns.vnfs.terminate_nfs import TerminateVnfs
 from lcm.ns.vnfs.scale_vnfs import NFManualScaleService
+from lcm.ns.vnfs.heal_vnfs import NFHealService
 from lcm.pub.utils.jobutil import JobUtil, JOB_TYPE
 
 
@@ -299,6 +300,48 @@ class TestScaleVnfViews(TestCase):
         else:
             self.failUnlessEqual(1, 0)
 
+
+class TestHealVnfViews(TestCase):
+    def setUp(self):
+        self.client = Client()
+        self.ns_inst_id = str(uuid.uuid4())
+        self.nf_inst_id = str(uuid.uuid4())
+        self.nf_uuid = '111'
+
+        NSInstModel(id=self.ns_inst_id, name="ns_name").save()
+        NfInstModel.objects.create(nfinstid=self.nf_inst_id, nf_name='name_1', vnf_id='1',
+                                   vnfm_inst_id='1', ns_inst_id='111,2-2-2',
+                                   max_cpu='14', max_ram='12296', max_hd='101', max_shd="20", max_net=10,
+                                   status='active', mnfinstid=self.nf_uuid, package_id='pkg1',
+                                   vnfd_model='{"metadata": {"vnfdId": "1","vnfdName": "PGW001",'
+                                              '"vnfProvider": "zte","vnfdVersion": "V00001","vnfVersion": "V5.10.20",'
+                                              '"productType": "CN","vnfType": "PGW",'
+                                              '"description": "PGW VNFD description",'
+                                              '"isShared":true,"vnfExtendType":"driver"}}')
+
+    def tearDown(self):
+        NSInstModel.objects.all().delete()
+        NfInstModel.objects.all().delete()
+
+    @mock.patch.object(restcall, "call_req")
+    def test_heal_vnf(self, mock_call_req):
+
+        req_data = {
+            "action": "vmReset",
+            "affectedvm": {
+                "vmid": 1,
+                "vduid": 1,
+                "vmname": "name",
+            }
+        }
+
+        NFHealService(self.ns_inst_id, req_data).run()
+        nsIns = NfInstModel.objects.filter(nfinstid=self.nf_inst_id)
+        if nsIns:
+            self.failUnlessEqual(1, 1)
+        else:
+            self.failUnlessEqual(1, 0)
+
 vnfd_model_dict = {
     'local_storages': [],
     'vdus': [
diff --git a/lcm/ns/urls.py b/lcm/ns/urls.py
index eeb3511c..a18efcc5 100644
--- a/lcm/ns/urls.py
+++ b/lcm/ns/urls.py
@@ -15,7 +15,7 @@ from django.conf.urls import patterns, url
 from rest_framework.urlpatterns import format_suffix_patterns
 
 from lcm.ns.views import CreateNSView, NSInstView, TerminateNSView, NSDetailView, SwaggerJsonView, NSInstPostDealView, \
-    NSManualScaleView
+    NSManualScaleView, NSHealView
 
 urlpatterns = patterns('',
                        url(r'^api/nslcm/v1/ns$', CreateNSView.as_view()),
@@ -29,6 +29,8 @@ urlpatterns = patterns('',
                            NSInstPostDealView.as_view()),
                        url(r'^api/nslcm/v1/ns/(?P<ns_instance_id>[0-9a-zA-Z_-]+)/scale$',
                            NSManualScaleView.as_view()),
+                       url(r'^api/nslcm/v1/ns/(?P<ns_instance_id>[0-9a-zA-Z_-]+)/heal$',
+                           NSHealView.as_view())
                        )
 
 urlpatterns = format_suffix_patterns(urlpatterns)
diff --git a/lcm/ns/views.py b/lcm/ns/views.py
index 88ca5428..8fa4ba77 100644
--- a/lcm/ns/views.py
+++ b/lcm/ns/views.py
@@ -24,6 +24,7 @@ from lcm.ns.ns_create import CreateNSService
 from lcm.ns.ns_get import GetNSInfoService
 from lcm.ns.ns_instant import InstantNSService
 from lcm.ns.ns_manual_scale import NSManualScaleService
+from lcm.ns.ns_heal import NSHealService
 from lcm.ns.ns_terminate import TerminateNsService, DeleteNsService
 from lcm.pub.database.models import NSInstModel, ServiceBaseInfoModel
 from lcm.pub.utils.jobutil import JobUtil, JOB_TYPE
@@ -78,6 +79,20 @@ class TerminateNSView(APIView):
         return Response(data=ret, status=status.HTTP_202_ACCEPTED)
 
 
+class NSHealView(APIView):
+    def post(self, request, ns_instance_id):
+        logger.debug("Enter HealNSView::post %s", request.data)
+        job_id = JobUtil.create_job("VNF", JOB_TYPE.HEAL_VNF, ns_instance_id)
+        try:
+            NSHealService(ns_instance_id, request.data, job_id).start()
+        except Exception as e:
+            logger.error("Exception in HealNSView: %s", e.message)
+            return Response(data={'error': e.message}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        ret = {'jobId': job_id}
+        logger.debug("Leave HealNSView::post ret=%s", ret)
+        return Response(data=ret, status=status.HTTP_202_ACCEPTED)
+
+
 class NSDetailView(APIView):
     def get(self, request, ns_instance_id):
         logger.debug("Enter NSDetailView::get ns(%s)", ns_instance_id)
diff --git a/lcm/ns/vnfs/const.py b/lcm/ns/vnfs/const.py
index 0bef8fe5..3775f9cf 100644
--- a/lcm/ns/vnfs/const.py
+++ b/lcm/ns/vnfs/const.py
@@ -14,7 +14,7 @@
 from lcm.pub.utils.enumutil import enum
 
 VNF_STATUS = enum(NULL='null', INSTANTIATING="instantiating", INACTIVE='inactive', ACTIVE="active", FAILED="failed",
-                  TERMINATING="terminating", SCALING="scaling")
+                  TERMINATING="terminating", SCALING="scaling", HEALING="healing")
 INST_TYPE = enum(VNF=0, VNFM=1)
 INST_TYPE_NAME = enum(VNF='VNF', VNFM='VNFM')
 PACKAGE_TYPE = enum(VNFD='VNFD', NSD='NSD')
diff --git a/lcm/ns/vnfs/heal_vnfs.py b/lcm/ns/vnfs/heal_vnfs.py
new file mode 100644
index 00000000..a8a1ed69
--- /dev/null
+++ b/lcm/ns/vnfs/heal_vnfs.py
@@ -0,0 +1,109 @@
+# Copyright 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import threading
+import traceback
+
+from lcm.ns.vnfs.const import VNF_STATUS
+from lcm.ns.vnfs.wait_job import wait_job_finish
+from lcm.pub.database.models import NfInstModel
+from lcm.pub.exceptions import NSLCMException
+from lcm.pub.msapi.vnfmdriver import send_nf_heal_request
+from lcm.pub.utils.jobutil import JobUtil, JOB_TYPE, JOB_MODEL_STATUS
+from lcm.pub.utils.values import ignore_case_get
+
+JOB_ERROR = 255
+
+logger = logging.getLogger(__name__)
+
+
+class NFHealService(threading.Thread):
+    def __init__(self, vnf_instance_id, data):
+        super(NFHealService, self).__init__()
+        self.vnf_instance_id = vnf_instance_id
+        self.data = data
+        self.job_id = JobUtil.create_job("NF", JOB_TYPE.HEAL_VNF, vnf_instance_id)
+
+        self.nf_model = {}
+        self.nf_additional_params = {}
+        self.nf_heal_params = {}
+        self.m_nf_inst_id = ''
+        self.vnfm_inst_id = ''
+
+    def run(self):
+        try:
+            self.do_biz()
+        except NSLCMException as e:
+            JobUtil.add_job_status(self.job_id, JOB_ERROR, e.message)
+        except:
+            logger.error(traceback.format_exc())
+            JobUtil.add_job_status(self.job_id, JOB_ERROR, 'nf heal fail')
+
+    def do_biz(self):
+        self.update_job(1, desc='nf heal start')
+        self.update_nf_status(VNF_STATUS.HEALING)
+        self.get_and_check_params()
+        self.send_nf_healing_request()
+        self.update_nf_status(VNF_STATUS.ACTIVE)
+        self.update_job(100, desc='nf heal success')
+
+    def get_and_check_params(self):
+        nf_info = NfInstModel.objects.filter(nfinstid=self.vnf_instance_id)
+        if not nf_info:
+            logger.error('NF instance[id=%s] does not exist' % self.vnf_instance_id)
+            raise NSLCMException('NF instance[id=%s] does not exist' % self.vnf_instance_id)
+        logger.debug('vnfd_model = %s, vnf_instance_id = %s' % (nf_info[0].vnfd_model, self.vnf_instance_id))
+        self.nf_model = json.loads(nf_info[0].vnfd_model)
+        self.m_nf_inst_id = nf_info[0].mnfinstid
+        self.vnfm_inst_id = nf_info[0].vnfm_inst_id
+        self.nf_additional_params = ignore_case_get(self.data, 'additionalParams')
+
+        if not self.nf_additional_params:
+            logger.error('additionalParams parameter does not exist or value incorrect')
+            raise NSLCMException('additionalParams parameter does not exist or value incorrect')
+
+        action = ignore_case_get(self.nf_additional_params, 'action')
+        if action is "restartvm":
+            action = "vmReset"
+        vmid = ignore_case_get(self.nf_additional_params, 'vmid')
+        vmname = ignore_case_get(self.nf_additional_params, 'vmname')
+        # TODO(sshank): Find how to get 'vduid'
+        vduid = ""
+
+        self.nf_heal_params = {
+            "action": action,
+            "affectedvm": {
+                "vmid": vmid,
+                "vduid": vduid,
+                "vmname": vmname,
+            }
+        }
+
+    def send_nf_healing_request(self):
+        req_param = json.JSONEncoder().encode(self.nf_heal_params)
+        rsp = send_nf_heal_request(self.vnfm_inst_id, self.m_nf_inst_id, req_param)
+        vnfm_job_id = ignore_case_get(rsp, 'jobId')
+        ret = wait_job_finish(self.vnfm_inst_id, self.job_id, vnfm_job_id, progress_range=None, timeout=1200,
+                              mode='1')
+        if ret != JOB_MODEL_STATUS.FINISHED:
+            logger.error('[NF heal] nf heal failed')
+            raise NSLCMException("nf heal failed")
+
+    def update_job(self, progress, desc=''):
+        JobUtil.add_job_status(self.job_id, progress, desc)
+
+    def update_nf_status(self, status):
+        NfInstModel.objects.filter(nfinstid=self.vnf_instance_id).update(status=status)
diff --git a/lcm/pub/msapi/vnfmdriver.py b/lcm/pub/msapi/vnfmdriver.py
index 2f450ff2..9e88e2de 100644
--- a/lcm/pub/msapi/vnfmdriver.py
+++ b/lcm/pub/msapi/vnfmdriver.py
@@ -67,3 +67,13 @@ def send_nf_scaling_request(vnfm_inst_id, vnf_inst_id, req_param):
         logger.error("Failed to send nf scale req:%s,%s", ret[2], ret[1])
         raise NSLCMException('Failed to send nf scale request to VNFM(%s)' % vnfm_inst_id)
     return json.JSONDecoder().decode(ret[1])
+
+
+def send_nf_heal_request(vnfm_inst_id, vnf_inst_id, req_param):
+    vnfm = get_vnfm_by_id(vnfm_inst_id)
+    uri = "/api/%s/v1/%s/vnfs/%s/heal" % (vnfm["type"], vnfm_inst_id, vnf_inst_id)
+    ret = req_by_msb(uri, "POST", req_param)
+    if ret[0] > 0:
+        logger.error("Failed to send nf heal req:%s,%s", ret[2], ret[1])
+        raise NSLCMException('Failed to send nf heal request to VNFM(%s)' % vnfm_inst_id)
+    return json.JSONDecoder().decode(ret[1])
diff --git a/lcm/pub/utils/jobutil.py b/lcm/pub/utils/jobutil.py
index 650d2afc..ae8ecacf 100644
--- a/lcm/pub/utils/jobutil.py
+++ b/lcm/pub/utils/jobutil.py
@@ -29,7 +29,8 @@ def enum(**enums):
 JOB_STATUS = enum(PROCESSING=0, FINISH=1)
 JOB_MODEL_STATUS = enum(STARTED='started', PROCESSING='processing', FINISHED='finished', ERROR='error',
                         TIMEOUT='timeout')
-JOB_TYPE = enum(CREATE_VNF="create vnf", TERMINATE_VNF="terminate vnf", GRANT_VNF="grant vnf", MANUAL_SCALE_VNF="manual scale vnf")
+JOB_TYPE = enum(CREATE_VNF="create vnf", TERMINATE_VNF="terminate vnf", GRANT_VNF="grant vnf", MANUAL_SCALE_VNF="manual scale vnf",
+                HEAL_VNF="heal vnf")
 
 
 class JobUtil(object):