Add Heal Api in GVNFM

Change-Id: If825c7a03cd38a6694d2cc0d28fbff391a3f4a95
Signed-off-by: Shobana Jothi<shobana.jothi@verizon.com>
Issue-ID: VFC-995

12 files changed, 616 insertions, 149 deletions

diff --git a/lcm/lcm/nf/biz/common.py b/lcm/lcm/nf/biz/common.py
new file mode 100644
index 00000000..308ba060
--- /dev/null
+++ b/lcm/lcm/nf/biz/common.py
@@ -0,0 +1,145 @@
+# Copyright 2017 ZTE Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+from lcm.pub.database.models import VmInstModel, NetworkInstModel, \
+    SubNetworkInstModel, PortInstModel, StorageInstModel, FlavourInstModel, VNFCInstModel
+from lcm.pub.utils.jobutil import JobUtil
+from lcm.pub.utils.values import ignore_case_get, get_none, get_boolean, get_integer
+
+
+def volume_save(job_id, nf_inst_id, ret):
+    JobUtil.add_job_status(job_id, 25, 'Create vloumns!')
+    StorageInstModel.objects.create(
+        storageid=str(uuid.uuid4()),
+        vimid=ignore_case_get(ret, "vimId"),
+        resourceid=ignore_case_get(ret, "id"),
+        name=ignore_case_get(ret, "name"),
+        tenant=ignore_case_get(ret, "tenantId"),
+        create_time=ignore_case_get(ret, "createTime"),
+        storagetype=get_none(ignore_case_get(ret, "type")),
+        size=ignore_case_get(ret, "size"),
+        insttype=0,
+        is_predefined=ignore_case_get(ret, "returnCode"),
+        nodeId=ignore_case_get(ret, "nodeId"),
+        instid=nf_inst_id)
+
+
+def network_save(job_id, nf_inst_id, ret):
+    JobUtil.add_job_status(job_id, 35, 'Create networks!')
+    NetworkInstModel.objects.create(
+        networkid=str(uuid.uuid4()),
+        name=ignore_case_get(ret, "name"),
+        vimid=ignore_case_get(ret, "vimId"),
+        resourceid=ignore_case_get(ret, "id"),
+        tenant=ignore_case_get(ret, "tenantId"),
+        segmentid=str(ignore_case_get(ret, "segmentationId")),
+        network_type=ignore_case_get(ret, "networkType"),
+        physicalNetwork=ignore_case_get(ret, "physicalNetwork"),
+        vlantrans=get_boolean(ignore_case_get(ret, "vlanTransparent")),
+        is_shared=get_boolean(ignore_case_get(ret, "shared")),
+        routerExternal=get_boolean(ignore_case_get(ret, "routerExternal")),
+        insttype=0,
+        is_predefined=ignore_case_get(ret, "returnCode"),
+        nodeId=ignore_case_get(ret, "nodeId"),
+        instid=nf_inst_id)
+
+
+def subnet_save(job_id, nf_inst_id, ret):
+    JobUtil.add_job_status(job_id, 40, 'Create subnets!')
+    SubNetworkInstModel.objects.create(
+        subnetworkid=str(uuid.uuid4()),
+        name=ignore_case_get(ret, "name"),
+        vimid=ignore_case_get(ret, "vimId"),
+        resourceid=ignore_case_get(ret, "id"),
+        tenant=ignore_case_get(ret, "tenantId"),
+        networkid=ignore_case_get(ret, "networkId"),
+        cidr=ignore_case_get(ret, "cidr"),
+        ipversion=ignore_case_get(ret, "ipversion"),
+        isdhcpenabled=ignore_case_get(ret, "enableDhcp"),
+        gatewayip=ignore_case_get(ret, "gatewayIp"),
+        dnsNameservers=ignore_case_get(ret, "dnsNameservers"),
+        hostRoutes=ignore_case_get(ret, "hostRoutes"),
+        allocationPools=ignore_case_get(ret, "allocationPools"),
+        insttype=0,
+        is_predefined=ignore_case_get(ret, "returnCode"),
+        instid=nf_inst_id)
+
+
+def port_save(job_id, nf_inst_id, ret):
+    JobUtil.add_job_status(job_id, 50, 'Create ports!')
+    PortInstModel.objects.create(
+        portid=str(uuid.uuid4()),
+        networkid=ignore_case_get(ret, "networkId"),
+        subnetworkid=ignore_case_get(ret, "subnetId"),
+        name=ignore_case_get(ret, "name"),
+        vimid=ignore_case_get(ret, "vimId"),
+        resourceid=ignore_case_get(ret, "id"),
+        tenant=ignore_case_get(ret, "tenantId"),
+        macaddress=ignore_case_get(ret, "macAddress"),
+        ipaddress=ignore_case_get(ret, "ip"),
+        typevirtualnic=ignore_case_get(ret, "vnicType"),
+        securityGroups=ignore_case_get(ret, "securityGroups"),
+        insttype=0,
+        is_predefined=ignore_case_get(ret, "returnCode"),
+        nodeId=ignore_case_get(ret, "nodeId"),
+        instid=nf_inst_id)
+
+
+def flavor_save(job_id, nf_inst_id, ret):
+    JobUtil.add_job_status(job_id, 60, 'Create flavors!')
+    FlavourInstModel.objects.create(
+        flavourid=str(uuid.uuid4()),
+        name=ignore_case_get(ret, "name"),
+        vimid=ignore_case_get(ret, "vimId"),
+        resourceid=ignore_case_get(ret, "id"),
+        tenant=ignore_case_get(ret, "tenantId"),
+        vcpu=get_integer(ignore_case_get(ret, "vcpu")),
+        memory=get_integer(ignore_case_get(ret, "memory")),
+        disk=get_integer(ignore_case_get(ret, "disk")),
+        ephemeral=get_integer(ignore_case_get(ret, "ephemeral")),
+        swap=get_integer(ignore_case_get(ret, "swap")),
+        isPublic=get_boolean(ignore_case_get(ret, "isPublic")),
+        extraspecs=ignore_case_get(ret, "extraSpecs"),
+        is_predefined=ret.get("returnCode", int(0)),
+        instid=nf_inst_id)
+
+
+def vm_save(job_id, nf_inst_id, ret):
+    JobUtil.add_job_status(job_id, 70, 'Create vms!')
+    vm_id = str(uuid.uuid4())
+    VmInstModel.objects.create(
+        vmid=vm_id,
+        vmname=ignore_case_get(ret, "name"),
+        vimid=ignore_case_get(ret, "vimId"),
+        resourceid=ignore_case_get(ret, "id"),
+        tenant=ignore_case_get(ret, "tenantId"),
+        nic_array=ignore_case_get(ret, "nicArray"),
+        metadata=ignore_case_get(ret, "metadata"),
+        volume_array=ignore_case_get(ret, "volumeArray"),
+        server_group=ignore_case_get(ret, "serverGroup"),
+        availability_zone=str(ignore_case_get(ret, "availabilityZone", "undefined")),
+        flavor_id=ignore_case_get(ret, "flavorId"),
+        security_groups=ignore_case_get(ret, "securityGroups"),
+        operationalstate=ignore_case_get(ret, "status"),
+        insttype=0,
+        is_predefined=ignore_case_get(ret, "returnCode"),
+        instid=nf_inst_id)
+    VNFCInstModel.objects.create(
+        vnfcinstanceid=str(uuid.uuid4()),
+        vduid=ignore_case_get(ret, "id"),
+        is_predefined=ignore_case_get(ret, "returnCode"),
+        instid=nf_inst_id,
+        vmid=vm_id)
diff --git a/lcm/lcm/nf/biz/create_vnf.py b/lcm/lcm/nf/biz/create_vnf.py
index de857b61..c86ec14c 100644
--- a/lcm/lcm/nf/biz/create_vnf.py
+++ b/lcm/lcm/nf/biz/create_vnf.py
@@ -57,7 +57,7 @@ class CreateVnf:
                                    version=version,
                                    vendor=provider,
                                    netype=netype,
-                                   vnfd_model=self.vnfd_info,
+                                   vnfd_model=json.dumps(self.vnfd_info),
                                    status='NOT_INSTANTIATED',
                                    nf_desc=self.description,
                                    vnfdid=self.csar_id,
diff --git a/lcm/lcm/nf/biz/grant_vnf.py b/lcm/lcm/nf/biz/grant_vnf.py
index 0889e6dc..92f22350 100644
--- a/lcm/lcm/nf/biz/grant_vnf.py
+++ b/lcm/lcm/nf/biz/grant_vnf.py
@@ -76,6 +76,24 @@ def grant_resource(data, nf_inst_id, job_id, grant_type, vdus):
             content_args['addResources'].append(res_def)
             res_index += 1
         content_args['additionalParams']['vimid'] = vim_id
+    elif grant_type == GRANT_TYPE.HEAL_RESTART:
+        res_index = 1
+        res_def = {
+            'type': 'VDU',
+            'resDefId': str(res_index),
+            'resDesId': vdus[0].resourceid}
+        content_args['updateResources'].append(res_def)
+        content_args['additionalParams']['vimid'] = vdus[0].vimid
+    elif grant_type == GRANT_TYPE.HEAL_CREATE:
+        vim_id = vdus[0]["properties"]["location_info"]["vimid"]
+        res_index = 1
+        res_def = {
+            'type': 'VDU',
+            'resDefId': str(res_index),
+            'resDesId': ignore_case_get(vdus[0], "vdu_id")
+        }
+        content_args['addResources'].append(res_def)
+        content_args['additionalParams']['vimid'] = vim_id
     elif grant_type == GRANT_TYPE.OPERATE:
         res_index = 1
         for vdu in vdus:
diff --git a/lcm/lcm/nf/biz/heal_vnf.py b/lcm/lcm/nf/biz/heal_vnf.py
new file mode 100644
index 00000000..bf8e34a0
--- /dev/null
+++ b/lcm/lcm/nf/biz/heal_vnf.py
@@ -0,0 +1,117 @@
+# Copyright (C) 2018 Verizon. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import traceback
+from threading import Thread
+
+from lcm.pub.database.models import NfInstModel, VmInstModel
+from lcm.pub.exceptions import NFLCMException
+from lcm.pub.msapi.gvnfmdriver import notify_lcm_to_nfvo, prepare_notification_data
+from lcm.pub.utils.jobutil import JobUtil
+from lcm.pub.utils.timeutil import now_time
+from lcm.pub.utils.values import ignore_case_get
+from lcm.pub.vimapi import adaptor
+from lcm.nf.biz.grant_vnf import grant_resource
+from lcm.nf.const import VNF_STATUS, GRANT_TYPE, HEAL_ACTION_TYPE, CHANGE_TYPE, OPERATION_TYPE
+from lcm.nf.biz import common
+
+logger = logging.getLogger(__name__)
+
+
+class HealVnf(Thread):
+    def __init__(self, data, nf_inst_id, job_id):
+        super(HealVnf, self).__init__()
+        self.data = data
+        self.nf_inst_id = nf_inst_id
+        self.job_id = job_id
+        self.affectedvm = ignore_case_get(ignore_case_get(self.data, "additionalParams"), "affectedvm")
+        # TODO: Check if we could move the action param into the list of affectedvm structure
+        self.action = ignore_case_get(ignore_case_get(self.data, "additionalParams"), "action")
+        self.grant_type = ""
+        if self.action == HEAL_ACTION_TYPE.START:
+            self.grant_type = GRANT_TYPE.HEAL_CREATE
+        elif self.action == HEAL_ACTION_TYPE.RESTART:
+            self.grant_type = GRANT_TYPE.HEAL_RESTART
+
+    def run(self):
+        try:
+            self.heal_pre()
+            self.apply_grant()
+            self.heal_resource()
+            JobUtil.add_job_status(self.job_id, 100, "Heal Vnf success.")
+            NfInstModel.objects.filter(nfinstid=self.nf_inst_id).update(status='INSTANTIATED', lastuptime=now_time())
+            self.lcm_notify()
+        except NFLCMException as e:
+            logger.error(e.message)
+            self.vnf_heal_failed_handle(e.message)
+        except Exception as e:
+            logger.error(e.message)
+            self.vnf_heal_failed_handle(traceback.format_exc())
+
+    def heal_pre(self):
+        if self.action not in (HEAL_ACTION_TYPE.START, HEAL_ACTION_TYPE.RESTART):
+            raise NFLCMException("Action type in Request in invalid. Should be %s or %s" % (HEAL_ACTION_TYPE.START, HEAL_ACTION_TYPE.RESTART))
+
+        self.vm_id = ignore_case_get(self.affectedvm, "vmid")
+        self.vdu_id = ignore_case_get(self.affectedvm, "vduid")
+        self.vm_name = ignore_case_get(self.affectedvm, "vmname")
+        if not (self.vm_id and self.vdu_id and self.vm_name):
+            raise NFLCMException("VM identifiers is not present in request.")
+
+        self.vnf_insts = NfInstModel.objects.filter(nfinstid=self.nf_inst_id)
+        self.vnfd_info = json.loads(self.vnf_insts[0].vnfd_model)
+
+    def apply_grant(self):
+        if self.action == HEAL_ACTION_TYPE.RESTART:
+            self.vdu = VmInstModel.objects.filter(instid=self.nf_inst_id, is_predefined=1, vmid=self.vm_id, vmname=self.vm_name)
+            if not self.vdu:
+                raise NFLCMException("VNF Vm does not exist.")
+            self.vimid = self.vdu[0].vimid
+            self.tenant = self.vdu[0].tenant
+        elif self.action == HEAL_ACTION_TYPE.START:
+            vdus = ignore_case_get(self.vnfd_info, "vdus")
+            self.vdu = [elem for elem in vdus if ignore_case_get(elem, "vdu_id") == self.vdu_id]
+            if not self.vdu:
+                raise NFLCMException("VNF Vm does not exist.")
+        apply_result = grant_resource(data=self.data, nf_inst_id=self.nf_inst_id, job_id=self.job_id,
+                                      grant_type=self.grant_type, vdus=self.vdu)
+        if self.action == HEAL_ACTION_TYPE.START:
+            self.vimid = ignore_case_get(apply_result, "vimid"),
+            self.tenant = ignore_case_get(apply_result, "tenant")
+        logger.info("Grant resource, response: %s" % apply_result)
+        JobUtil.add_job_status(self.job_id, 20, 'Nf Healing grant_resource finish')
+
+    def heal_resource(self):
+        logger.info('Heal resource begin')
+        data = {'action': self.action, 'vimid': self.vimid, 'tenant': self.tenant}
+        adaptor.heal_vim_res(self.vdu, self.vnfd_info, self.do_notify, data, json.loads(self.vnf_insts[0].vimInfo), json.loads(self.vnf_insts[0].resInfo))
+        logger.info('Heal resource complete')
+
+    def do_notify(self, res_type, ret):
+        logger.info('Creating [%s] resource' % res_type)
+        resource_save_method = getattr(common, res_type + '_save')
+        resource_save_method(self.job_id, self.nf_inst_id, ret)
+
+    def lcm_notify(self):
+        notification_content = prepare_notification_data(self.nf_inst_id, self.job_id, CHANGE_TYPE.MODIFIED, OPERATION_TYPE.HEAL)
+        logger.info('Notify request data = %s' % notification_content)
+        resp = notify_lcm_to_nfvo(json.dumps(notification_content))
+        logger.info('Lcm notify end, response %s' % resp)
+
+    def vnf_heal_failed_handle(self, error_msg):
+        logger.error('VNF Healing failed, detail message: %s' % error_msg)
+        NfInstModel.objects.filter(nfinstid=self.nf_inst_id).update(status=VNF_STATUS.FAILED, lastuptime=now_time())
+        JobUtil.add_job_status(self.job_id, 255, error_msg)
diff --git a/lcm/lcm/nf/biz/instantiate_vnf.py b/lcm/lcm/nf/biz/instantiate_vnf.py
index 73109c80..606dbab0 100644
--- a/lcm/lcm/nf/biz/instantiate_vnf.py
+++ b/lcm/lcm/nf/biz/instantiate_vnf.py
@@ -15,11 +15,9 @@
 import json
 import logging
 import traceback
-import uuid
 from threading import Thread
 
-from lcm.pub.database.models import NfInstModel, VmInstModel, NetworkInstModel, \
-    SubNetworkInstModel, PortInstModel, StorageInstModel, FlavourInstModel, VNFCInstModel
+from lcm.pub.database.models import NfInstModel
 from lcm.pub.exceptions import NFLCMException
 from lcm.pub.msapi.gvnfmdriver import prepare_notification_data
 # from lcm.pub.msapi.gvnfmdriver import notify_lcm_to_nfvo
@@ -27,10 +25,11 @@ from lcm.pub.msapi.sdc_run_catalog import query_vnfpackage_by_id
 from lcm.pub.utils.jobutil import JobUtil
 from lcm.pub.utils.timeutil import now_time
 from lcm.pub.utils.notificationsutil import NotificationsUtil
-from lcm.pub.utils.values import ignore_case_get, get_none, get_boolean, get_integer
+from lcm.pub.utils.values import ignore_case_get
 from lcm.pub.vimapi import adaptor
 from lcm.nf.biz.grant_vnf import grant_resource
 from lcm.nf.const import CHANGE_TYPE, GRANT_TYPE, OPERATION_TYPE
+from lcm.nf.biz import common
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +92,7 @@ class InstantiateVnf(Thread):
                    version=version,
                    vendor=vendor,
                    netype=netype,
-                   vnfd_model=self.vnfd_info,
+                   vnfd_model=json.dumps(self.vnfd_info),
                    status='NOT_INSTANTIATED',
                    vnfdid=self.vnfd_id,
                    localizationLanguage=ignore_case_get(self.data, 'localizationLanguage'),
@@ -126,8 +125,12 @@ class InstantiateVnf(Thread):
 
     def create_res(self):
         logger.info("Create resource start")
-        adaptor.create_vim_res(self.vnfd_info, self.do_notify)
+        vim_cache, res_cache = {}, {}
+        adaptor.create_vim_res(self.vnfd_info, self.do_notify, vim_cache=vim_cache, res_cache=res_cache)
         JobUtil.add_job_status(self.job_id, 70, '[NF instantiation] create resource finish')
+        NfInstModel.objects.filter(nfinstid=self.nf_inst_id).\
+            update(vimInfo=json.dumps(vim_cache),
+                   resInfo=json.dumps(res_cache))
         logger.info("Create resource finish")
 
     def lcm_notify(self):
@@ -144,7 +147,7 @@ class InstantiateVnf(Thread):
 
     def do_notify(self, res_type, ret):
         logger.info('Creating [%s] resource' % res_type)
-        resource_save_method = globals().get(res_type + '_save')
+        resource_save_method = getattr(common, res_type + '_save')
         resource_save_method(self.job_id, self.nf_inst_id, ret)
 
     def update_cps(self):
@@ -201,128 +204,3 @@ class InstantiateVnf(Thread):
                     subnet_ids.append(ignore_case_get(ip_address, "subnetId"))
         return subnet_ids
     '''
-
-
-def volume_save(job_id, nf_inst_id, ret):
-    JobUtil.add_job_status(job_id, 25, 'Create vloumns!')
-    StorageInstModel.objects.create(
-        storageid=str(uuid.uuid4()),
-        vimid=ignore_case_get(ret, "vimId"),
-        resourceid=ignore_case_get(ret, "id"),
-        name=ignore_case_get(ret, "name"),
-        tenant=ignore_case_get(ret, "tenantId"),
-        create_time=ignore_case_get(ret, "createTime"),
-        storagetype=get_none(ignore_case_get(ret, "type")),
-        size=ignore_case_get(ret, "size"),
-        insttype=0,
-        is_predefined=ignore_case_get(ret, "returnCode"),
-        nodeId=ignore_case_get(ret, "nodeId"),
-        instid=nf_inst_id)
-
-
-def network_save(job_id, nf_inst_id, ret):
-    JobUtil.add_job_status(job_id, 35, 'Create networks!')
-    NetworkInstModel.objects.create(
-        networkid=str(uuid.uuid4()),
-        name=ignore_case_get(ret, "name"),
-        vimid=ignore_case_get(ret, "vimId"),
-        resourceid=ignore_case_get(ret, "id"),
-        tenant=ignore_case_get(ret, "tenantId"),
-        segmentid=str(ignore_case_get(ret, "segmentationId")),
-        network_type=ignore_case_get(ret, "networkType"),
-        physicalNetwork=ignore_case_get(ret, "physicalNetwork"),
-        vlantrans=get_boolean(ignore_case_get(ret, "vlanTransparent")),
-        is_shared=get_boolean(ignore_case_get(ret, "shared")),
-        routerExternal=get_boolean(ignore_case_get(ret, "routerExternal")),
-        insttype=0,
-        is_predefined=ignore_case_get(ret, "returnCode"),
-        nodeId=ignore_case_get(ret, "nodeId"),
-        instid=nf_inst_id)
-
-
-def subnet_save(job_id, nf_inst_id, ret):
-    JobUtil.add_job_status(job_id, 40, 'Create subnets!')
-    SubNetworkInstModel.objects.create(
-        subnetworkid=str(uuid.uuid4()),
-        name=ignore_case_get(ret, "name"),
-        vimid=ignore_case_get(ret, "vimId"),
-        resourceid=ignore_case_get(ret, "id"),
-        tenant=ignore_case_get(ret, "tenantId"),
-        networkid=ignore_case_get(ret, "networkId"),
-        cidr=ignore_case_get(ret, "cidr"),
-        ipversion=ignore_case_get(ret, "ipversion"),
-        isdhcpenabled=ignore_case_get(ret, "enableDhcp"),
-        gatewayip=ignore_case_get(ret, "gatewayIp"),
-        dnsNameservers=ignore_case_get(ret, "dnsNameservers"),
-        hostRoutes=ignore_case_get(ret, "hostRoutes"),
-        allocationPools=ignore_case_get(ret, "allocationPools"),
-        insttype=0,
-        is_predefined=ignore_case_get(ret, "returnCode"),
-        instid=nf_inst_id)
-
-
-def port_save(job_id, nf_inst_id, ret):
-    JobUtil.add_job_status(job_id, 50, 'Create ports!')
-    PortInstModel.objects.create(
-        portid=str(uuid.uuid4()),
-        networkid=ignore_case_get(ret, "networkId"),
-        subnetworkid=ignore_case_get(ret, "subnetId"),
-        name=ignore_case_get(ret, "name"),
-        vimid=ignore_case_get(ret, "vimId"),
-        resourceid=ignore_case_get(ret, "id"),
-        tenant=ignore_case_get(ret, "tenantId"),
-        macaddress=ignore_case_get(ret, "macAddress"),
-        ipaddress=ignore_case_get(ret, "ip"),
-        typevirtualnic=ignore_case_get(ret, "vnicType"),
-        securityGroups=ignore_case_get(ret, "securityGroups"),
-        insttype=0,
-        is_predefined=ignore_case_get(ret, "returnCode"),
-        nodeId=ignore_case_get(ret, "nodeId"),
-        instid=nf_inst_id)
-
-
-def flavor_save(job_id, nf_inst_id, ret):
-    JobUtil.add_job_status(job_id, 60, 'Create flavors!')
-    FlavourInstModel.objects.create(
-        flavourid=str(uuid.uuid4()),
-        name=ignore_case_get(ret, "name"),
-        vimid=ignore_case_get(ret, "vimId"),
-        resourceid=ignore_case_get(ret, "id"),
-        tenant=ignore_case_get(ret, "tenantId"),
-        vcpu=get_integer(ignore_case_get(ret, "vcpu")),
-        memory=get_integer(ignore_case_get(ret, "memory")),
-        disk=get_integer(ignore_case_get(ret, "disk")),
-        ephemeral=get_integer(ignore_case_get(ret, "ephemeral")),
-        swap=get_integer(ignore_case_get(ret, "swap")),
-        isPublic=get_boolean(ignore_case_get(ret, "isPublic")),
-        extraspecs=ignore_case_get(ret, "extraSpecs"),
-        is_predefined=ret.get("returnCode", int(0)),
-        instid=nf_inst_id)
-
-
-def vm_save(job_id, nf_inst_id, ret):
-    JobUtil.add_job_status(job_id, 70, 'Create vms!')
-    vm_id = str(uuid.uuid4())
-    VmInstModel.objects.create(
-        vmid=vm_id,
-        vmname=ignore_case_get(ret, "name"),
-        vimid=ignore_case_get(ret, "vimId"),
-        resourceid=ignore_case_get(ret, "id"),
-        tenant=ignore_case_get(ret, "tenantId"),
-        nic_array=ignore_case_get(ret, "nicArray"),
-        metadata=ignore_case_get(ret, "metadata"),
-        volume_array=ignore_case_get(ret, "volumeArray"),
-        server_group=ignore_case_get(ret, "serverGroup"),
-        availability_zone=str(ignore_case_get(ret, "availabilityZone", "undefined")),
-        flavor_id=ignore_case_get(ret, "flavorId"),
-        security_groups=ignore_case_get(ret, "securityGroups"),
-        operationalstate=ignore_case_get(ret, "status"),
-        insttype=0,
-        is_predefined=ignore_case_get(ret, "returnCode"),
-        instid=nf_inst_id)
-    VNFCInstModel.objects.create(
-        vnfcinstanceid=str(uuid.uuid4()),
-        vduid=ignore_case_get(ret, "id"),
-        is_predefined=ignore_case_get(ret, "returnCode"),
-        instid=nf_inst_id,
-        vmid=vm_id)
diff --git a/lcm/lcm/nf/const.py b/lcm/lcm/nf/const.py
index 912b9c38..5e6cb675 100644
--- a/lcm/lcm/nf/const.py
+++ b/lcm/lcm/nf/const.py
@@ -330,13 +330,6 @@ vnfd_model_dict = {
                 }
             },
             "artifacts": [
-                {
-                    "artifact_name": "software_version_file",
-                    "type": "tosca.artifacts.Deployment",
-                    "file": "AppSoftwares/zte-cn-xgw-V5.16.11_NFV-version.zip",
-                    "repository": "",
-                    "deploy_path": ""
-                }
             ]
         }
     ],
@@ -499,6 +492,15 @@ c1_data_get_tenant_id = {
         }
     ]
 }
+c1_data_get_tenant_id_1 = {
+    "tenants": [
+        {
+            "id": "1",
+            "name": "tenantname"
+        }
+    ]
+}
+
 
 # create_volume
 c2_data_create_volume = {
diff --git a/lcm/lcm/nf/serializers/heal_vnf_req.py b/lcm/lcm/nf/serializers/heal_vnf_req.py
new file mode 100644
index 00000000..e020182a
--- /dev/null
+++ b/lcm/lcm/nf/serializers/heal_vnf_req.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2018 Verizon. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from rest_framework import serializers
+
+
+class HealVnfRequestSerializer(serializers.Serializer):
+    cause = serializers.CharField(help_text="Cause of NS heal", required=False, allow_null=True)
+    additionalParams = serializers.DictField(
+        help_text="Additional input parameters for the healing process, \
+        specific to the VNF being healed, \
+        as declared in the VNFD as part of HealVnfOpConfig.",
+        required=False,
+        allow_null=True)
diff --git a/lcm/lcm/nf/tests/test_heal_vnf.py b/lcm/lcm/nf/tests/test_heal_vnf.py
new file mode 100644
index 00000000..27cce093
--- /dev/null
+++ b/lcm/lcm/nf/tests/test_heal_vnf.py
@@ -0,0 +1,176 @@
+# Copyright (C) 2018 Verizon. All Rights Reserved.

+#

+# Licensed under the Apache License, Version 2.0 (the "License");

+# you may not use this file except in compliance with the License.

+# You may obtain a copy of the License at

+#

+#       http://www.apache.org/licenses/LICENSE-2.0

+#

+# Unless required by applicable law or agreed to in writing, software

+# distributed under the License is distributed on an "AS IS" BASIS,

+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+# See the License for the specific language governing permissions and

+# limitations under the License.

+

+import json

+

+import mock

+from django.test import TestCase

+from rest_framework import status

+from rest_framework.test import APIClient

+

+from lcm.nf.biz.heal_vnf import HealVnf

+from lcm.nf.const import c1_data_get_tenant_id_1, c9_data_create_vm, c10_data_get_vm, vnfd_model_dict

+from lcm.pub.database.models import NfInstModel, JobStatusModel, VmInstModel

+from lcm.pub.utils import restcall

+from lcm.pub.utils.jobutil import JobUtil

+from lcm.pub.utils.timeutil import now_time

+from lcm.pub.vimapi import api

+

+

+class TestNFInstantiate(TestCase):

+    def setUp(self):

+        self.client = APIClient()

+        self.grant_result = {

+            "vimid": 'vimid_1',

+            "tenant": 'tenantname_1',

+        }

+        self.getvmResult = {

+            "status": "ACTIVE",

+            "id": "11",

+        }

+

+    def tearDown(self):

+        pass

+

+    def assert_job_result(self, job_id, job_progress, job_detail):

+        jobs = JobStatusModel.objects.filter(jobid=job_id,

+                                             progress=job_progress,

+                                             descp=job_detail)

+        self.assertEqual(1, len(jobs))

+

+    def test_heal_vnf_not_found(self):

+        req_data = {}

+        response = self.client.post("/api/vnflcm/v1/vnf_instances/12/heal", data=req_data, format='json')

+        self.failUnlessEqual(status.HTTP_404_NOT_FOUND, response.status_code)

+

+    def test_heal_vnf_conflict(self):

+        req_data = {}

+        NfInstModel(nfinstid='12', nf_name='VNF1', status='NOT_INSTANTIATED').save()

+        response = self.client.post("/api/vnflcm/v1/vnf_instances/12/heal", data=req_data, format='json')

+        self.failUnlessEqual(status.HTTP_409_CONFLICT, response.status_code)

+        NfInstModel(nfinstid='12', nf_name='VNF1', status='NOT_INSTANTIATED').delete()

+

+    @mock.patch.object(HealVnf, 'run')

+    def test_heal_vnf_success(self, mock_run):

+        req_data = {}

+        NfInstModel(nfinstid='12', nf_name='VNF1', status='INSTANTIATED').save()

+        response = self.client.post("/api/vnflcm/v1/vnf_instances/12/heal", data=req_data, format='json')

+        mock_run.re.return_value = None

+        self.failUnlessEqual(status.HTTP_202_ACCEPTED, response.status_code)

+        NfInstModel(nfinstid='12', nf_name='VNF1', status='INSTANTIATED').delete()

+

+    @mock.patch.object(restcall, 'call_req')

+    @mock.patch.object(api, 'call')

+    def test_heal_vnf_success_reboot(self, mock_call, mock_call_req):

+        vim_cache = {}

+        res_cache = {}

+

+        NfInstModel.objects.create(nfinstid='1111',

+                                   nf_name='2222',

+                                   vnfminstid='1',

+                                   package_id='todo',

+                                   version='',

+                                   vendor='',

+                                   netype='',

+                                   vnfd_model=json.dumps(vnfd_model_dict),

+                                   status='INSTANTIATED',

+                                   nf_desc='',

+                                   vnfdid='',

+                                   vnfSoftwareVersion='',

+                                   vnfConfigurableProperties='todo',

+                                   localizationLanguage='EN_US',

+                                   create_time=now_time(),

+                                   resInfo=json.dumps(res_cache),

+                                   vimInfo=json.dumps(vim_cache))

+

+        VmInstModel.objects.create(vmid="1",

+                                   vimid="vimid_1",

+                                   resourceid="11",

+                                   insttype=0,

+                                   instid="1111",

+                                   vmname="vduinstname",

+                                   is_predefined=1,

+                                   tenant="tenantname_1",

+                                   operationalstate=1)

+        t1_apply_grant_result = [0, json.JSONEncoder().encode(self.grant_result), '200']

+        t2_lcm_notify_result = [0, json.JSONEncoder().encode(''), '200']

+        t3_action_get_vm = [0, json.JSONEncoder().encode(self.getvmResult), '202']

+        t4_action_vm_start_reboot = [0, json.JSONEncoder().encode(''), '202']

+        mock_call_req.side_effect = [t1_apply_grant_result, t2_lcm_notify_result, t3_action_get_vm, t4_action_vm_start_reboot]

+        mock_call.side_effect = [self.getvmResult, None]

+        req_data = {

+            "cause": "Error",

+            "additionalParams": {

+                "action": "vmReset",

+                "affectedvm": {

+                    "vmid": "1",

+                    "vduid": "vdu1Id",

+                    "vmname": "vduinstname"

+                }

+            }

+        }

+        self.nf_inst_id = '1111'

+        self.job_id = JobUtil.create_job('NF', 'HEAL', self.nf_inst_id)

+        JobUtil.add_job_status(self.job_id, 0, "HEAL_VNF_READY")

+        HealVnf(req_data, nf_inst_id=self.nf_inst_id, job_id=self.job_id).run()

+        self.assert_job_result(self.job_id, 100, "Heal Vnf success.")

+

+    @mock.patch.object(restcall, 'call_req')

+    @mock.patch.object(api, 'call')

+    def test_heal_vnf_success_start(self, mock_call, mock_call_req):

+        vim_cache = {}

+        res_cache = {"volume": {}, "flavor": {}, "port": {}}

+        res_cache["volume"]["volume_storage1"] = "vol1"

+        res_cache["flavor"]["vdu1Id"] = "flavor1"

+        res_cache["port"]["cpId1"] = "port1"

+

+        NfInstModel.objects.create(nfinstid='1111',

+                                   nf_name='2222',

+                                   vnfminstid='1',

+                                   package_id='todo',

+                                   version='',

+                                   vendor='',

+                                   netype='',

+                                   vnfd_model=json.dumps(vnfd_model_dict),

+                                   status='INSTANTIATED',

+                                   nf_desc='',

+                                   vnfdid='',

+                                   vnfSoftwareVersion='',

+                                   vnfConfigurableProperties='todo',

+                                   localizationLanguage='EN_US',

+                                   resInfo=json.dumps(res_cache),

+                                   vimInfo=json.dumps(vim_cache),

+                                   create_time=now_time())

+

+        t1_apply_grant_result = [0, json.JSONEncoder().encode(self.grant_result), '200']

+        t2_lcm_notify_result = [0, json.JSONEncoder().encode(''), '200']

+        t3_action_vm_start_create = [0, json.JSONEncoder().encode(''), '202']

+        mock_call_req.side_effect = [t1_apply_grant_result, t2_lcm_notify_result, t3_action_vm_start_create]

+        mock_call.side_effect = [c1_data_get_tenant_id_1, c9_data_create_vm, c10_data_get_vm]

+        req_data = {

+            "cause": "Error",

+            "additionalParams": {

+                "action": "vmCreate",

+                "affectedvm": {

+                    "vmid": "1",

+                    "vduid": "vdu1Id",

+                    "vmname": "vduinstname"

+                }

+            }

+        }

+        self.nf_inst_id = '1111'

+        self.job_id = JobUtil.create_job('NF', 'HEAL', self.nf_inst_id)

+        JobUtil.add_job_status(self.job_id, 0, "HEAL_VNF_READY")

+        HealVnf(req_data, nf_inst_id=self.nf_inst_id, job_id=self.job_id).run()

+        self.assert_job_result(self.job_id, 100, "Heal Vnf success.")

diff --git a/lcm/lcm/nf/urls.py b/lcm/lcm/nf/urls.py
index 58f5c392..fd0aaae9 100644
--- a/lcm/lcm/nf/urls.py
+++ b/lcm/lcm/nf/urls.py
@@ -18,6 +18,7 @@ from lcm.nf.views.curd_vnf_views import DeleteVnfAndQueryVnf, CreateVnfAndQueryV
 from lcm.nf.views.instantiate_vnf_view import InstantiateVnfView
 from lcm.nf.views.terminate_vnf_view import TerminateVnfView
 from lcm.nf.views.subscriptions_view import SubscriptionsView
+from lcm.nf.views.heal_vnf_view import HealVnfView
 from lcm.nf.views.operate_vnf_view import OperateVnfView
 from lcm.nf.views.lcm_op_occs_view import QueryMultiVnfLcmOpOccs, QuerySingleVnfLcmOpOcc
 
@@ -27,6 +28,7 @@ urlpatterns = [
     url(r'^api/vnflcm/v1/vnf_instances/(?P<instanceid>[0-9a-zA-Z_-]+)/instantiate$', InstantiateVnfView.as_view()),
     url(r'^api/vnflcm/v1/vnf_instances/(?P<instanceid>[0-9a-zA-Z_-]+)$', DeleteVnfAndQueryVnf.as_view()),
     url(r'^api/vnflcm/v1/vnf_instances/(?P<instanceid>[0-9a-zA-Z_-]+)/terminate$', TerminateVnfView.as_view()),
+    url(r'^api/vnflcm/v1/vnf_instances/(?P<instanceid>[0-9a-zA-Z_-]+)/heal$', HealVnfView.as_view()),
     url(r'^api/vnflcm/v1/vnf_instances/(?P<instanceid>[0-9a-zA-Z_-]+)/operate$', OperateVnfView.as_view()),
     url(r'^api/vnflcm/v1/vnf_lcm_op_occs$', QueryMultiVnfLcmOpOccs.as_view()),
     url(r'^api/vnflcm/v1/vnf_lcm_op_occs/(?P<lcmopoccid>[0-9a-zA-Z_-]+)$', QuerySingleVnfLcmOpOcc.as_view()),
diff --git a/lcm/lcm/nf/views/heal_vnf_view.py b/lcm/lcm/nf/views/heal_vnf_view.py
new file mode 100644
index 00000000..08daa98f
--- /dev/null
+++ b/lcm/lcm/nf/views/heal_vnf_view.py
@@ -0,0 +1,88 @@
+# Copyright (C) 2018 Verizon. All Rights Reserved.

+#

+# Licensed under the Apache License, Version 2.0 (the "License");

+# you may not use this file except in compliance with the License.

+# You may obtain a copy of the License at

+#

+#       http://www.apache.org/licenses/LICENSE-2.0

+#

+# Unless required by applicable law or agreed to in writing, software

+# distributed under the License is distributed on an "AS IS" BASIS,

+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+# See the License for the specific language governing permissions and

+# limitations under the License.

+

+import logging

+import traceback

+

+from drf_yasg.utils import swagger_auto_schema

+from rest_framework import status

+from rest_framework.response import Response

+from rest_framework.views import APIView

+

+from lcm.nf.biz.heal_vnf import HealVnf

+from lcm.nf.serializers.heal_vnf_req import HealVnfRequestSerializer

+from lcm.nf.serializers.response import ProblemDetailsSerializer

+from lcm.pub.exceptions import NFLCMException, NFLCMExceptionNotFound, NFLCMExceptionConflict

+from lcm.pub.utils.jobutil import JobUtil

+from lcm.pub.database.models import NfInstModel

+from lcm.nf.const import VNF_STATUS

+

+logger = logging.getLogger(__name__)

+

+

+class HealVnfView(APIView):

+    @swagger_auto_schema(

+        request_body=HealVnfRequestSerializer(),

+        responses={

+            status.HTTP_202_ACCEPTED: "Success",

+            status.HTTP_404_NOT_FOUND: ProblemDetailsSerializer(),

+            status.HTTP_409_CONFLICT: ProblemDetailsSerializer(),

+            status.HTTP_500_INTERNAL_SERVER_ERROR: "Internal error"

+        }

+    )

+    def post(self, request, instanceid):

+        logger.debug("HealVnf--post::> %s" % request.data)

+        try:

+            heal_vnf_request_serializer = HealVnfRequestSerializer(data=request.data)

+            if not heal_vnf_request_serializer.is_valid():

+                raise NFLCMException(heal_vnf_request_serializer.errors)

+

+            job_id = JobUtil.create_job('NF', 'HEAL', instanceid)

+            JobUtil.add_job_status(job_id, 0, "HEAL_VNF_READY")

+            self.heal_pre_check(instanceid, job_id)

+            HealVnf(heal_vnf_request_serializer.data, instanceid, job_id).start()

+            response = Response(data=None, status=status.HTTP_202_ACCEPTED)

+            response["Location"] = "/vnf_lc_ops/%s" % job_id

+            return response

+        except NFLCMExceptionNotFound as e:

+            probDetail = ProblemDetailsSerializer(data={"status": status.HTTP_404_NOT_FOUND, "detail": "VNF Instance not found"})

+            resp_isvalid = probDetail.is_valid()

+            if not resp_isvalid:

+                raise NFLCMException(probDetail.errors)

+            return Response(data=probDetail.data, status=status.HTTP_404_NOT_FOUND)

+        except NFLCMExceptionConflict as e:

+            probDetail = ProblemDetailsSerializer(data={"status": status.HTTP_409_CONFLICT, "detail": "VNF Instance not in Instantiated State"})

+            resp_isvalid = probDetail.is_valid()

+            if not resp_isvalid:

+                    raise NFLCMException(probDetail.errors)

+            return Response(data=probDetail.data, status=status.HTTP_409_CONFLICT)

+        except NFLCMException as e:

+            logger.error(e.message)

+            return Response(data={'error': '%s' % e.message}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

+        except Exception as e:

+            logger.error(e.message)

+            logger.error(traceback.format_exc())

+            return Response(data={'error': 'unexpected exception'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

+

+    def heal_pre_check(self, nf_inst_id, job_id):

+        vnf_insts = NfInstModel.objects.filter(nfinstid=nf_inst_id)

+        if not vnf_insts.exists():

+            raise NFLCMExceptionNotFound("VNF nf_inst_id does not exist.")

+

+        if vnf_insts[0].status != 'INSTANTIATED':

+            raise NFLCMExceptionConflict("VNF instantiationState is not INSTANTIATED.")

+

+        NfInstModel.objects.filter(nfinstid=nf_inst_id).update(status=VNF_STATUS.HEALING)

+        JobUtil.add_job_status(job_id, 15, 'Nf healing pre-check finish')

+        logger.info("Nf healing pre-check finish")

diff --git a/lcm/lcm/pub/database/models.py b/lcm/lcm/pub/database/models.py
index 47a22fdd..3ac4a832 100644
--- a/lcm/lcm/pub/database/models.py
+++ b/lcm/lcm/pub/database/models.py
@@ -40,6 +40,8 @@ class NfInstModel(models.Model):
     vnfConfigurableProperties = models.TextField(db_column='VNFCONFIGURABLEPROPERTIES', max_length=20000, blank=True, null=True)
     localizationLanguage = models.CharField(db_column='LOCALIZATIONLANGUAGE', max_length=255, null=True)
     operationState = models.CharField(db_column='OPERATIONSTATE', max_length=255, null=True)
+    resInfo = models.TextField(db_column='RESINFO', max_length=20000, blank=True, null=True)
+    vimInfo = models.TextField(db_column='VIMINFO', max_length=20000, blank=True, null=True)
 
 
 class JobModel(models.Model):
diff --git a/lcm/lcm/pub/vimapi/adaptor.py b/lcm/lcm/pub/vimapi/adaptor.py
index f47c3370..3427aaf4 100644
--- a/lcm/lcm/pub/vimapi/adaptor.py
+++ b/lcm/lcm/pub/vimapi/adaptor.py
@@ -20,7 +20,8 @@ from lcm.pub.utils.values import ignore_case_get, set_opt_val
 from lcm.pub.msapi.aai import get_flavor_info
 from . import api
 from .exceptions import VimException
-from lcm.nf.const import ACTION_TYPE
+from lcm.pub.exceptions import NFLCMException
+from lcm.nf.const import ACTION_TYPE, HEAL_ACTION_TYPE
 
 logger = logging.getLogger(__name__)
 
@@ -101,17 +102,30 @@ def operate_vim_res(data, changeStateTo, stopType, gracefulStopTimeout, do_notif
                         gracefulStopTimeout = 60
                     time.sleep(gracefulStopTimeout)
                 action_vm(ACTION_TYPE.STOP, res, res["vim_id"], res["tenant_id"])
-                # TODO check if the we should poll getvm to get the status or the action_vm api
-                # successful return should suffice to mark vm as Active/Inactive
                 do_notify_op("INACTIVE", res["id"])
         except VimException as e:
-            # TODO Have to update database appropriately on failure
-            logger.error("Failed to Heal %s(%s)", RES_VM, res["res_id"])
+            logger.error("Failed to Operate %s(%s)", RES_VM, res["res_id"])
             logger.error("%s:%s", e.http_code, e.message)
-
-
-def create_vim_res(data, do_notify):
-    vim_cache, res_cache = {}, {}
+            raise NFLCMException("Failed to Operate %s(%s)", RES_VM, res["res_id"])
+
+
+def heal_vim_res(vdus, vnfd_info, do_notify, data, vim_cache, res_cache):
+    try:
+        vimid = data["vimid"]
+        tenant = data["tenant"]
+        actionType = data["action"]
+        if actionType == HEAL_ACTION_TYPE.START:
+            create_vm(vim_cache, res_cache, vnfd_info, vdus[0], do_notify, RES_VM)
+        elif actionType == HEAL_ACTION_TYPE.RESTART:
+            vm_info = api.get_vm(vimid, tenant, vdus[0].resourceid)
+            action_vm(ACTION_TYPE.REBOOT, vm_info, vimid, tenant)
+    except VimException as e:
+        logger.error("Failed to Heal %s(%s)", RES_VM, vdus[0]["vdu_id"])
+        logger.error("%s:%s", e.http_code, e.message)
+        raise NFLCMException("Failed to Heal %s(%s)", RES_VM, vdus[0]["vdu_id"])
+
+
+def create_vim_res(data, do_notify, vim_cache={}, res_cache={}):
     for vol in ignore_case_get(data, "volume_storages"):
         create_volume(vim_cache, res_cache, vol, do_notify, RES_VOLUME)
     for network in ignore_case_get(data, "vls"):