path: root/django/engagementmanager/service/checklist_service.py

#  
# ============LICENSE_START========================================== 
# org.onap.vvp/engagementmgr
# ===================================================================
# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
# ===================================================================
#
# Unless otherwise specified, all software contained herein is licensed
# under the Apache License, Version 2.0 (the “License”);
# you may not use this software except in compliance with the License.
# You may obtain a copy of the License at
#
#             http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
#
# Unless otherwise specified, all documentation contained herein is licensed
# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
# you may not use this documentation except in compliance with the License.
# You may obtain a copy of the License at
#
#             https://creativecommons.org/licenses/by/4.0/
#
# Unless required by applicable law or agreed to in writing, documentation
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ============LICENSE_END============================================
#
# ECOMP is a trademark and service mark of AT&T Intellectual Property.
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django.db.models.aggregates import Max
from engagementmanager.apps import bus_service
from engagementmanager.bus.messages.activity_event_message import \
    ActivityEventMessage
from engagementmanager.git.git_manager import GitManager
from engagementmanager.models import ChecklistTemplate, Checklist, Engagement, \
    ChecklistAuditLog, ChecklistDecision, ChecklistLineItem, IceUserProfile, VF, \
    ChecklistSection, Role
from engagementmanager.serializers import ThinChecklistModelSerializer, \
    ThinChecklistAuditLogModelSerializer, ThinChecklistDecisionModelSerializer, \
    ThinPostChecklistResponseModelSerializer, ChecklistTemplateModelSerializer, \
    ChecklistSectionModelSerializer, ChecklistLineItemModelSerializer
from engagementmanager.service.checklist_audit_log_service import \
    addAuditLogToDecision, addAuditLogToChecklist
from engagementmanager.service.checklist_decision_service import setDecision
from engagementmanager.service.checklist_state_service import set_state
from engagementmanager.service.base_service import BaseSvc
from engagementmanager.utils.constants import Roles, CheckListState, \
    CheckListLineType
from engagementmanager.utils.activities_data import TestFinishedActivityData
from engagementmanager.utils.vvp_exceptions import VvpObjectNotAvailable
from engagementmanager.utils.request_data_mgr import request_data_mgr
from engagementmanager.utils.validator import logEncoding
from engagementmanager.vm_integration.vm_client import \
    send_jenkins_job_and_gitlab_repo_exists, executor
from validationmanager.em_integration.vm_api import get_jenkins_build_log
import json
import time
from engagementmanager.service.logging_service import LoggingServiceFactory

logger = LoggingServiceFactory.get_logger()


class CheckListSvc(BaseSvc):
    def retreive_cl_files_for_engagment(self, eng_uuid):
        vf_associated_files_list = []

        checklists_of_eng = Checklist.objects.filter(engagement__uuid=eng_uuid).exclude(
            state=CheckListState.archive.name).exclude(state=CheckListState.closed.name)  # @UndefinedVariable

        for checklistObj in checklists_of_eng:
            associated_files = json.loads(checklistObj.associated_files)
            vf_associated_files_list = vf_associated_files_list + associated_files
        return vf_associated_files_list

    def getDataForCreateNewChecklist(self, user, eng_uuid):
        engagement = Engagement.objects.get(uuid=eng_uuid)
        vf = engagement.vf

        if not send_jenkins_job_and_gitlab_repo_exists(vf):
            msg = "Jenkins job or gitlab repo is still missing"
            logger.error(msg)
            return False

        # Get all templates
        data = self.getChecklistTemplates()

        data['checkListAssociatedFiles'] = self.gitManager.getRepoAssociatedFilesForUser(
            eng_uuid)
        return data

    def getChecklistTemplates(self, templateUuid=None):
        """Return  checklist template with nested sections and their nested line items"""
        data = dict()
        if not templateUuid:
            checkListTemplates = ChecklistTemplate.objects.all()
            if (checkListTemplates):
                data['checkListTemplates'] = ChecklistTemplateModelSerializer(
                    checkListTemplates, many=True).data
            return data
        cl_template = ChecklistTemplate.objects.get(uuid=templateUuid)
        if (cl_template != None):
            data = ChecklistTemplateModelSerializer(
                cl_template, many=False).data
            sections = ChecklistSection.objects.filter(template=cl_template)
            if (sections != None):
                section_list = []
                for sec in sections:
                    section_data = dict()
                    section_data = ChecklistSectionModelSerializer(
                        sec, many=False).data

                    lineItems = ChecklistLineItem.objects.filter(section=sec)
                    section_data['lineItems'] = ChecklistLineItemModelSerializer(
                        lineItems, many=True).data
                    section_list.append(section_data)

            data['sections'] = section_list

        return data

    def createNewSection(self, section, templateObj):
        """Create new section for a given template """
        logger.debug(
            "Creating a new section. Section name = " + section['name'])

        weight = int(ChecklistSection.objects.filter(template=templateObj).aggregate(
            max_weight=Max('weight'))['max_weight'] or 0) + 1

        newSection = ChecklistSection.objects.create(
            name=section.get('name', None),
            description=section.get('description', None),
            validation_instructions=section.get(
                'validation_instructions', None),
            template=templateObj,
            weight=weight)

        return newSection

    def createNewLineItemForSection(self, newSectionObj, listItem, templateObj):
        """Create new line item for a given section and template """

        weight = int(ChecklistLineItem.objects.filter(section=newSectionObj).aggregate(
            max_weight=Max('weight'))['max_weight'] or 0) + 1

        ChecklistLineItem.objects.create(
            name=listItem['name'],
            description=listItem.get('description', None),
            # @UndefinedVariable
            line_type=listItem.get('line_type', CheckListLineType.auto.name),
            validation_instructions=listItem.get(
                'validation_instructions', None),
            section=newSectionObj,
            template=templateObj,
            weight=weight)

    def delete(self, dict_structure, query_set, entity, isDirty):
        """Generically find the xor result of the user input and the db data. Assumption: If entities exits in DB but not in user input they'll be deleted"""
        uuid_client = [dict['uuid'] for dict in dict_structure]
        uuid_db = [record.uuid for record in query_set]
        uuids_to_delete = set(uuid_db) - set(uuid_client)
        for u_uid in uuids_to_delete:
            entity.objects.filter(uuid=u_uid).delete()
            # Note: No need to delete ChecklistLineItem corresponding to this section
            # since there is a CASCADE operation on delete section
            isDirty[0] = True

    def editIfChanged(self, entity, uidict, fieldList):
        """Generic function to check that set of fields are modified on a certain entity"""
        isChanged = False
        for field in fieldList:
            if (field in uidict):
                if (isChanged != True and entity.__dict__[field] == uidict[field]):
                    isChanged = False
                else:
                    entity.__dict__[field] = uidict[field]
                    isChanged = True
        return isChanged

    def updateTemplateFields(self, clTemplate, checklistTemplate, isDirty):
        if (self.editIfChanged(clTemplate, checklistTemplate, ['name'])):
            clTemplate.save()
            isDirty[0] = True

    def updateSectionFields(self, section, sec, isDirty):
        if (self.editIfChanged(section, sec, ['name'])):
            section.save()
            isDirty[0] = True

    def updateLineItemFields(self, lineitem, li, isDirty):
        if (self.editIfChanged(lineitem, li, ['name', 'description', 'validation_instructions', 'line_type'])):
            lineitem.save()
            isDirty[0] = True

    def editChecklistTemplate(self, checklistTemplate):
        """edit the template+section+line-item of user input"""
        NEW_ENTITY = "newEntity"  # this is an indication on top of the provided json to create the entity
        templateObj = None
        isDirty = [False]
        if ('uuid' in checklistTemplate):
            templateUuid = checklistTemplate['uuid']
            templateObj = ChecklistTemplate.objects.get(uuid=templateUuid)
            self.updateTemplateFields(templateObj, checklistTemplate, isDirty)
            # SECTIONS
            if ('sections' in checklistTemplate):
                sections = checklistTemplate['sections']
                query_set = templateObj.checklistsection_set.all()
                self.delete(sections, query_set, ChecklistSection, isDirty)
                sectionObj = None
                for sec in sections:
                    if (sec['uuid'] == NEW_ENTITY):
                        sectionObj = self.createNewSection(sec, templateObj)
                        isDirty[0] = True
                    else:  # section was only updated
                        sectionObj = ChecklistSection.objects.get(
                            uuid=sec['uuid'])
                        self.updateSectionFields(sectionObj, sec, isDirty)
                    # LINE-ITEMS
                    if ('lineItems' in sec):
                        lineItems = sec['lineItems']
                        query_set = sectionObj.checklistlineitem_set.all()
                        self.delete(lineItems, query_set,
                                    ChecklistLineItem, isDirty)
                        for li in lineItems:
                            if (li['uuid'] == NEW_ENTITY):
                                self.createNewLineItemForSection(
                                    sectionObj, li, templateObj)
                                isDirty[0] = True
                            else:  # line-item was only updated
                                lineitem = ChecklistLineItem.objects.get(
                                    uuid=li['uuid'])
                                self.updateLineItemFields(lineitem, li, isDirty)

        executor.submit(self.decline_all_template_checklists,
                        isDirty[0], templateObj, request_data_mgr.get_user())

    def decline_all_template_checklists(self, isDirty, templateObj, user):
        request_data_mgr.set_user(user)
        checklists = None

        start = time.clock()
        try:
            if (isDirty == True):
                states_to_exclude = [CheckListState.archive.name, CheckListState.closed.name,
                                     CheckListState.pending.name]  # @UndefinedVariable
                checklists = Checklist.objects.filter(
                    template=templateObj).exclude(state__in=states_to_exclude)
                logger.debug("Number of checklists=" +
                                  str(len(checklists)))
                for checklist in checklists:
                    request_data_mgr.set_cl_uuid(checklist.uuid)
                    request_data_mgr.set_eng_uuid(checklist.engagement.uuid)
                    set_state(
                        True,  # means that the checklist will be declined and a cloned one is created in PENDING status
                        checklist.uuid,
                        isMoveToAutomation=True,
                        description="""Checklist {name} was rejected since its template ({template}) was edited/deleted""".format(
                            name=checklist.name, template=templateObj.name),  # means the checklist will be triggered into automation cycle
                    )
        except Exception as e:
            msg = """Something went wrong while trying to reject check-lists which its template was changed. template={template}. Error:""".format(
                template=templateObj.name)
            logger.error(msg + " " + str(e))
            raise e  # Don't remove try-except, it supports async invocation
        end = time.clock()
        logger.debug("TIME:" + str(end - start))

    def getChecklist(self, user, checklistUuid):

        data = dict()
        checklist = None

        if user.role.name == Roles.admin.name or user.role.name == Roles.admin_ro.name:  # @UndefinedVariable
            checklist = Checklist.objects.get(uuid=checklistUuid)
        else:
            checklist = Checklist.objects.get(
                Q(uuid=checklistUuid), Q(creator=user) | Q(owner=user))

        # CheckList
        if checklist.state == CheckListState.archive.name:  # @UndefinedVariable
            msg = "got a request for a checklist which is an archived one, might have been due to an admin edit of a checklist template."
            logger.error(msg)
            msg = "Requested checklist is archived, reloading checklists list"
            raise VvpObjectNotAvailable(msg)
        data['checklist'] = ThinChecklistModelSerializer(
            checklist, many=False).data
        vf = VF.objects.get(engagement=checklist.engagement)
        data['checklist']['associated_files'] = json.loads(
            data['checklist']['associated_files'])
        data['checklist']['jenkins_log'] = get_jenkins_build_log(
            vf, checklistUuid)
        data['checklist']['repo_associated_files'] = GitManager(
        ).getRepoAssociatedFilesForUser(checklist.engagement.uuid)
        # CheckList Audit Logs
        # here all fetched records should have decision==null
        checklistAuditLogs = ChecklistAuditLog.objects.filter(
            checklist__uuid=checklistUuid)
        serializedAuditLogsData = ThinChecklistAuditLogModelSerializer(
            checklistAuditLogs, many=True).data
        data['checklistAuditLogs'] = serializedAuditLogsData

        # CheckList Decisions + LineItems + Sections (The data is nested thanks
        # to the serializer)
        checklistDecisions = ChecklistDecision.objects.filter(
            checklist__uuid=checklistUuid)
        serializedDecisionsData = ThinChecklistDecisionModelSerializer(
            checklistDecisions, many=True).data
        checklistLineItems = {}
        for checklistDecision in serializedDecisionsData:
            section_uuid = checklistDecision['lineitem']['section']['uuid']
            section_weight = checklistDecision['lineitem']['section']['weight']
            section_key = str(section_weight) + "_" + section_uuid
            if section_key not in checklistLineItems:
                checklistLineItems[section_key] = {}
                checklistLineItems[section_key]['section'] = checklistDecision['lineitem']['section']
                checklistLineItems[section_key]['decisions'] = {}
                checklistLineItems[section_key]['weight'] = section_weight

            decision_uuid = checklistDecision['uuid']
            line_item_weight = checklistDecision['lineitem']['weight']
            decision_key = str(line_item_weight) + "_" + decision_uuid
            checklistLineItems[section_key]['decisions'][decision_key] = checklistDecision
            checklistLineItems[section_key]['decisions'][decision_key]['weight'] = line_item_weight

        data['checklistDecisions'] = checklistLineItems

        # Decision Audit Logs
        data['decisionAuditLogs'] = {}
        for checklistDecision in checklistDecisions:
            # here all fetched records should have checklist==null
            decisionAuditLogs = ChecklistAuditLog.objects.filter(
                decision=checklistDecision)
            if decisionAuditLogs.count() > 0:
                serializedAuditLogsData = ThinChecklistAuditLogModelSerializer(
                    decisionAuditLogs, many=True).data
                data['decisionAuditLogs'][checklistDecision.uuid] = serializedAuditLogsData

        logger.debug("get existing checklist has succeeded for checklist.uuid=" + str(checklist.uuid) +
                          ", user.uuid=" + str(user.uuid) + ", checklist.uuid=" + str(checklistUuid))

        return data

    def getEngagementFiles(self, eng_uuid):
        repo_files = self.gitManager.getRepoAssociatedFilesForUser(eng_uuid)
        return repo_files

    def createOrUpdateChecklist(self, checkListName, checkListTemplateUuid, checkListAssociatedFiles, checklistUuid=None):
        template = None
        checklist = None
        user = request_data_mgr.get_user()
        eng_uuid = request_data_mgr.get_eng_uuid()

        vf = VF.objects.get(engagement__uuid=eng_uuid)

        if not send_jenkins_job_and_gitlab_repo_exists(vf):
            msg = "Jenkins job or gitlab repo is still missing"
            logger.error(msg)
            msg = "Create or update checklist is not ready yet"
            raise Exception(msg)

        # associated_files may be delivered in this format
        #   [{"File": "bar"}, {"File": "baz"}, {"File": "foo"}, {"File": "quux"}]
        # but we want to store it in this format
        #   ["bar", "baz", "foo", "quux"]

        repo_files = self.gitManager.getRepoAssociatedFilesForUser(eng_uuid)
#         checklist_files_list = [f['File'] if isinstance(f, dict) else f for f in checkListAssociatedFiles]
        checklist_files_list = []
        for file in checkListAssociatedFiles:
            if isinstance(file, dict):
                checklist_files_list.append(file['File'])
            else:
                checklist_files_list.append(file)
        for added_name in checklist_files_list:
            if added_name not in repo_files:
                logger.error("Update checklist has failed. " +
                                  added_name + " doesnt exist in repo")
                msg = "Failed to create checklist, please select valid file"
                raise ValueError(msg)

        associated_files = json.dumps(checklist_files_list, ensure_ascii=False)

        engagement = Engagement.objects.get(uuid=eng_uuid)
        template = ChecklistTemplate.objects.get(uuid=checkListTemplateUuid)

        if (checklistUuid != None):  # Update Checklist
            checklist = Checklist.objects.get(uuid=checklistUuid)
            checklist.name = checkListName
            checklist.associated_files = associated_files
            checklist.template = template
            checklist.save()

            if (associated_files != None and len(checklist_files_list) > 0):
                set_state(
                    decline=True,
                    description="Checklist: " + checklist.name +
                    "in Pending state will transition to Automation because it has associated files",
                    isMoveToAutomation=True,  # means the checklist will be triggered into automation cycle
                    checklist_uuid=checklist.uuid
                )
        else:  # create ChcekList
            if (user.role.name == Roles.admin.name):  # @UndefinedVariable
                incharge_personal = engagement.reviewer
            else:
                incharge_personal = user
            vf = None
            vf = VF.objects.get(engagement=engagement)

            if (vf.git_repo_url == None):
                msg = "Can't create checklist since the attached VF (" + \
                    vf.name + ") doesn't contain git_repo_url"
                logger.error(
                    "Update checklist has failed. " + logEncoding(msg))
                raise ObjectDoesNotExist(msg)

            checklist = Checklist(name=checkListName, validation_cycle=1, associated_files=associated_files,
                                  state=CheckListState.pending.name, engagement=engagement, template=template, creator=user, owner=incharge_personal)  # @UndefinedVariable
            line_items_list = ChecklistLineItem.objects.filter(
                template=template)
            checklist.save()
            for lineitem in line_items_list:
                new_decision = ChecklistDecision(checklist=checklist,
                                                 template=template, lineitem=lineitem)
                new_decision.save()

            # When Checklist is created with files move it it automation
            if (associated_files != None and len(checklist_files_list) > 0):
                set_state(
                    decline=False,
                    checklist_uuid=checklist.uuid,
                    description="Checklist: " + checklist.name +
                    "in Pending state will transition to Automation because it has associated files",
                    isMoveToAutomation=True  # means the checklist will be triggered into automation cycle
                )

        logger.debug(
            "Create/Update checklist has succeeded for checklist.uuid=" + str(checklist.uuid))

        return ThinPostChecklistResponseModelSerializer(checklist).data

    def deleteChecklist(self, checklist_uuid):
        checklist = Checklist.objects.get(uuid=checklist_uuid)
        checklist.delete()

        logger.debug(
            "Delete checklist has succeeded for checklist.uuid=" + str(checklist_uuid))

    def setChecklistDecisionsFromValMgr(self, user, checklist_uuid, decisions, checklist_results_from_jenkins):
        checklist = Checklist.objects.get(uuid=checklist_uuid, owner=user,
                                          state=CheckListState.automation.name)  # @UndefinedVariable

        logger.debug("setChecklistDecisionsFromValMgr() checklist_uuid=%r, len(decisions)=%d",
                          checklist_uuid, len(decisions),)

        if ('error' in checklist_results_from_jenkins):
            el_role = Role.objects.get(name=Roles.el.name)
            admin_role = Role.objects.get(name=Roles.admin.name)
            el_admin_list = IceUserProfile.objects.all().filter(
                Q(role=el_role) | Q(role=admin_role))

            activity_data = TestFinishedActivityData(
                el_admin_list, checklist.engagement, checklist_results_from_jenkins['error'])
            bus_service.send_message(ActivityEventMessage(activity_data))

            msg = "test_finished signal from Jenkins has arrived with error: {}".format(
                checklist_results_from_jenkins['error'])
            logger.error(msg)
            set_state(True, checklist_uuid, isMoveToAutomation=False,
                      description=checklist_results_from_jenkins['error'])
            raise Exception(msg)

        ChecklistLineItem.objects.filter(template=checklist.template).update(
            line_type=CheckListLineType.manual.name)  # @UndefinedVariable

        for decision in decisions:
            lineitem_obj = ChecklistLineItem.objects.get(
                uuid=decision['line_item_id'])
            lineitem_obj.line_type = CheckListLineType.auto.name  # @UndefinedVariable
            lineitem_obj.save()

            decision_obj = ChecklistDecision.objects.get(
                checklist=checklist, lineitem=lineitem_obj)
            setDecision(decisionUuid=decision_obj.uuid,
                        user=user, value=decision['value'])

            if (decision['audit_log_text'] != '' and decision['audit_log_text'] != None):
                addAuditLogToDecision(decision=decision_obj,
                                      description=decision['audit_log_text'], user=user, category='')

        desc = "The {} validation test suite has completed. The decisions " +\
               "based on the test results have successfully been set in the " +\
               "checklist.".format(checklist.template.category)
        addAuditLogToChecklist(checklist=checklist, description=desc,
                               user=user, category='')
        checklistData = ThinChecklistModelSerializer(checklist, many=False).data
        set_state(False, checklist.uuid,
                  isMoveToAutomation=True, description="")

        return checklistData