refactor rest client to allow multipart formdata

Issue-ID: VNFSDK-350
Change-Id: I0581d562d53985450716c829dda6ca6e070283f1
Signed-off-by: Moshe <moshehoa@amdocs.com>

refactor rest client to allow multipart formdata

Issue-ID: VNFSDK-350
Change-Id: I2c60d1b28f010300f19f9bc02a94596060a58e05
Signed-off-by: Moshe <moshehoa@amdocs.com>

6 files changed, 36 insertions, 32 deletions

diff --git a/requirements.txt b/requirements.txt
index 3082d03..b85d3c7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -59,6 +59,7 @@ python-neutronclient==6.5.0     # OSI Approved  Apache Software License
 python-novaclient==9.1.1        # OSI Approved  Apache Software License
 pyzmq==16.0.2           # LGPL+BSD; OSI Approved  GNU Library or Lesser General Public License (LGPL); OSI Approved  BSD License
 requests==2.18.2        # Apache 2.0; OSI Approved  Apache Software License
+requests-toolbelt==0.8.0 # License: Apache 2.0
 requestsexceptions==1.3.0   # OSI Approved  Apache Software License
 scp==0.10.2             # LGPL
 shade==1.22.2           # OSI Approved  Apache Software License
diff --git a/vnftest/common/rest_client.py b/vnftest/common/rest_client.py
index bd938e4..b5ff567 100644
--- a/vnftest/common/rest_client.py
+++ b/vnftest/common/rest_client.py
@@ -14,24 +14,23 @@
 ##############################################################################
 
 import json
-
 import logging
 import os
 import urllib2
 import requests
-import sys
-import traceback
 from vnftest.common import utils
+from vnftest.common.yaml_loader import yaml_load
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 logger = logging.getLogger(__name__)
 os.putenv('PYTHONHTTPSVERIFY', "0")
 
 
-def post(url, headers, data, logger):
-    return call(url, 'POST', headers, data, logger)
+def post(url, headers, data):
+    return call(url, 'POST', headers, data)
 
 
-def call(url, method, headers, data, logger):
+def call(url, method, headers, data):
     data_json = json.dumps(data)
     f = None
     try:
@@ -49,7 +48,7 @@ def call(url, method, headers, data, logger):
         if 'application/xml' in content_type:
             response_body = utils.xml_to_dict(response_body)
         else:
-            response_body = json.loads(response_body)
+            response_body = yaml_load(response_body)
         result = {'return_code': return_code, 'body': response_body, 'headers': headers}
         return result
     except urllib2.HTTPError as e:
@@ -65,14 +64,17 @@ def call(url, method, headers, data, logger):
             f.close()
 
 
-def upload_file(url, headers, file, logger):
-    logger.debug("Upload file. URL: {}".format(url))
+def form_data(url, headers, form_data_content):
+    logger.debug("handle form-data. URL: {}".format(url))
     response = None
     try:
-        response = requests.post(url, headers=headers, files=file, verify=False)
-        return {'return_code': response.status_code, 'body': response.json(), 'headers': response.headers}
+        multipart_data = MultipartEncoder(fields=form_data_content)
+        headers['Content-Type'] = multipart_data.content_type
+        response = requests.post(url, headers=headers, data=multipart_data, verify=False)
+        body = yaml_load(response.text)
+        return {'return_code': response.status_code, 'body': body, 'headers': response.headers}
     except Exception as e:
-        message = "Error while uploading file to {}, exception: {}".format(url, e)
+        message = "Error handling form-data. url: {}, exception: {}".format(url, e)
         logger.exception(message)
         raise RuntimeError(message)
     finally:
diff --git a/vnftest/core/task.py b/vnftest/core/task.py
index d464d12..f1d2a4e 100644
--- a/vnftest/core/task.py
+++ b/vnftest/core/task.py
@@ -99,11 +99,11 @@ class Task(object):     # pragma: no cover
 
         if self.args.suite:
             # 1.parse suite, return suite_params info
-            task_files, task_args_list, task_args_fnames = \
-                parser.parse_suite()
+            task_files, task_args_list, task_args_fnames = parser.parse_suite()
+
         else:
             task_files = [parser.path]
-            task_args_list = [self.args.task_args]
+            task_args_list = [{}]
             task_args_fnames = [self.args.task_args_file]
 
         LOG.debug("task_files:%s, task_args_list:%s, task_args_fnames:%s",
@@ -120,9 +120,11 @@ class Task(object):     # pragma: no cover
                 task_args_file = task_args_fnames[i]
                 task_args = task_args_list[i]
                 try:
+                    inputs.update(parse_task_args("global_task_args", self.args.task_args))
                     if task_args_file:
                         with utils.load_resource(task_args_file) as f:
                             inputs.update(parse_task_args("task_args_file", f.read()))
+                    # task args from suite may override file args.
                     inputs.update(parse_task_args("task_args", task_args))
                 except TypeError:
                     raise TypeError()
diff --git a/vnftest/crawlers/base.py b/vnftest/crawlers/base.py
index c7813e8..733a846 100755
--- a/vnftest/crawlers/base.py
+++ b/vnftest/crawlers/base.py
@@ -41,7 +41,7 @@ class Crawler(object):
         output = {}
         for output_parameter in output_config:
             param_name = output_parameter['parameter_name']
-            param_value = output_parameter.get('value', "[]")
+            param_value = output_parameter.get('value', "[body]")
             crawler_type = output_parameter.get('type', 'default')
             crawler_class = Crawler.get_cls(crawler_type)
             crawler = crawler_class()
diff --git a/vnftest/steps/rest_call.py b/vnftest/steps/rest_call.py
index 3dde6f4..68631d9 100644
--- a/vnftest/steps/rest_call.py
+++ b/vnftest/steps/rest_call.py
@@ -13,15 +13,12 @@
 ##############################################################################
 from __future__ import absolute_import
 import copy
+import json
 import logging
 import time
-
-import os
 import yaml
-
 from vnftest.common import constants as consts, utils
 from vnftest.common import rest_client
-from vnftest.common.utils import dotdict
 from vnftest.common.exceptions import MandatoryKeyException, InputParameterMissing
 from vnftest.crawlers.base import Crawler
 from vnftest.steps import base
@@ -90,9 +87,7 @@ class RestCall(base.Step):
         params.update(copy.deepcopy(self.input_params))
         self.eval_input(params)
         execution_result = self.execute_operation(params)
-        result_body = execution_result['body']
-        result_body['headers'] = execution_result.get('headers', {})
-        output = Crawler.crawl(result_body, self.output_cfg)
+        output = Crawler.crawl(execution_result, self.output_cfg)
         result.update(output)
         return output
 
@@ -119,18 +114,22 @@ class RestCall(base.Step):
         LOG.info(url)
         LOG.info(headers)
         LOG.info(body)
-        if 'file' in operation:
-            file_conf = operation['file']
-            LOG.info(file_conf)
-            with utils.load_resource(file_conf['path']) as stream:
-                files = {file_conf['key']: stream}
-                result = rest_client.upload_file(url, headers, files, LOG)
+        if 'form-data' in operation:
+            form_data_content = {}
+            form_data_list = operation['form-data']
+            for form_data_item in form_data_list:
+                key = form_data_item['key']
+                if 'file' in form_data_item:
+                    form_data_content[key] = utils.load_resource(form_data_item['file'])
+                else:
+                    value_str = json.dumps(form_data_item['value'])
+                    form_data_content[key] = value_str
+            result = rest_client.form_data(url, headers, form_data_content)
         else:
             result = rest_client.call(url,
                                       operation['method'],
                                       headers,
-                                      body,
-                                      LOG)
+                                      body)
         if result['return_code'] >= 300:
             raise RuntimeError(
                 "Operation failed. return_code:{}, message:{}".format(result['return_code'], result['body']))
diff --git a/vnftest/tests/unit/onap/test_rest_call.py b/vnftest/tests/unit/onap/test_rest_call.py
index b100ecb..6735774 100644
--- a/vnftest/tests/unit/onap/test_rest_call.py
+++ b/vnftest/tests/unit/onap/test_rest_call.py
@@ -33,7 +33,7 @@ class RestCallTestCase(testtools.TestCase):
         'output':
         [{
           'parameter_name': "output1",
-          'value': "[value]"
+          'value': "[body][value]"
         }]},
       'sla': {
         'action': 'assert',