diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | pom.xml | 2 | ||||
| -rw-r--r-- | vio/requirements.txt | 3 | ||||
| -rw-r--r-- | vio/vio/middleware.py | 59 | ||||
| -rw-r--r-- | vio/vio/pub/config/config.py | 22 | ||||
| -rw-r--r-- | vio/vio/pub/config/log.yml | 8 | ||||
| -rw-r--r-- | vio/vio/pub/vim/vimapi/nova/OperateHypervisor.py | 4 | ||||
| -rw-r--r-- | vio/vio/settings.py | 2 | ||||
| -rw-r--r-- | vio/vio/swagger/urls.py | 7 | ||||
| -rw-r--r-- | vio/vio/swagger/views/capacity/__init__.py | 0 | ||||
| -rw-r--r-- | vio/vio/swagger/views/capacity/views.py | 123 | ||||
| -rw-r--r-- | vio/vio/tests/test_capacity_view.py | 104 | ||||
| -rw-r--r-- | vio/vio/urls.py | 10 |
13 files changed, 309 insertions, 36 deletions
@@ -5,6 +5,7 @@ target/ logs/*.log *.pyc +*.swp # Test related files vio/.coverage @@ -18,7 +18,7 @@ <parent> <groupId>org.onap.oparent</groupId> <artifactId>oparent</artifactId> - <version>0.1.1</version> + <version>1.1.0</version> <relativePath>../oparent</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/vio/requirements.txt b/vio/requirements.txt index 8d5a51c..32384a9 100644 --- a/vio/requirements.txt +++ b/vio/requirements.txt @@ -14,6 +14,7 @@ httplib2==0.9.2 # for call openstack api openstacksdk==0.9.14 +python-cinderclient==3.5.0 # for unit test django-nose>=1.4.0 @@ -22,4 +23,4 @@ mock==2.0.0 unittest_xml_reporting==1.12.0 # for onap logging -onappylog>=1.0.5 +onappylog>=1.0.6 diff --git a/vio/vio/middleware.py b/vio/vio/middleware.py new file mode 100644 index 0000000..161562e --- /dev/null +++ b/vio/vio/middleware.py @@ -0,0 +1,59 @@ +# Copyright (c) 2017-2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +import uuid +from onaplogging.mdcContext import MDC +from vio.pub.config.config import SERVICE_NAME +from vio.pub.config.config import FORWARDED_FOR_FIELDS + + +class LogContextMiddleware(object): + + # the last IP behind multiple proxies, if no exist proxies + # get local host ip. + def _getLastIp(self, request): + + ip = "" + try: + for field in FORWARDED_FOR_FIELDS: + if field in request.META: + if ',' in request.META[field]: + parts = request.META[field].split(',') + ip = parts[-1].strip().split(":")[0] + else: + ip = request.META[field].split(":")[0] + + if ip == "": + ip = request.META.get("HTTP_HOST").split(":")[0] + + except Exception: + pass + + return ip + + def process_request(self, request): + + ReqeustID = request.META.get("HTTP_X_TRANSACTIONID", None) + if ReqeustID is None: + ReqeustID = uuid.uuid3(uuid.NAMESPACE_URL, SERVICE_NAME) + MDC.put("requestID", ReqeustID) + InovocationID = uuid.uuid3(uuid.NAMESPACE_DNS, SERVICE_NAME) + MDC.put("invocationID", InovocationID) + MDC.put("serviceName", SERVICE_NAME) + MDC.put("serviceIP", self._getLastIp(request)) + return None + + def process_response(self, request, response): + + MDC.clear() + return response diff --git a/vio/vio/pub/config/config.py b/vio/vio/pub/config/config.py index 2b35e54..8f18ad5 100644 --- a/vio/vio/pub/config/config.py +++ b/vio/vio/pub/config/config.py @@ -24,7 +24,7 @@ ROOT_PATH = os.path.dirname(os.path.dirname( AAI_ADDR = "aai.api.simpledemo.openecomp.org" AAI_PORT = "8443" AAI_SERVICE_URL = 'https://%s:%s/aai' % (AAI_ADDR, AAI_PORT) -AAI_SCHEMA_VERSION = "v11" +AAI_SCHEMA_VERSION = "v13" AAI_USERNAME = 'AAI' AAI_PASSWORD = 'AAI' @@ -33,25 +33,13 @@ REDIS_HOST = '127.0.0.1' REDIS_PORT = '6379' REDIS_PASSWD = '' +# [MDC] +SERVICE_NAME = "multicloud-vio" +FORWARDED_FOR_FIELDS = ["HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_HOST", + "HTTP_X_FORWARDED_SERVER"] # [mysql] # DB_IP = '127.0.0.1' # DB_PORT = 3306 # DB_NAME = 'multivimvio' # DB_USER = 'root' # DB_PASSWD = 'password' - -# [register] -REG_TO_MSB_WHEN_START = False -REG_TO_MSB_REG_URL = "/api/microservices/v1/services" -REG_TO_MSB_REG_PARAM = { - "serviceName": "multicloud-vio", - "version": "v0", - "url": "/api/multicloud-vio/v0", - "protocol": "REST", - "visualRange": "1", - "nodes": [{ - "ip": "127.0.0.1", - "port": "9004", - "ttl": 0 - }] -} diff --git a/vio/vio/pub/config/log.yml b/vio/vio/pub/config/log.yml index 82eb814..5cced4f 100644 --- a/vio/vio/pub/config/log.yml +++ b/vio/vio/pub/config/log.yml @@ -12,14 +12,14 @@ handlers: class: "logging.handlers.RotatingFileHandler" filename: "/var/log/onap/multicloud/vio/vio.log" formatter: "mdcFormat" - maxBytes: 1024*1024*50 + maxBytes: 52428800 backupCount: 10 formatters: standard: - format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" + format: "%(asctime)s|||||%(name)s||%(thread)||%(funcName)s||%(levelname)s||%(message)s" mdcFormat: - format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:[%(mdc)s]: %(message)s" - mdcfmt: "{requestID}" + format: "%(asctime)s|||||%(name)s||%(thread)s||%(funcName)s||%(levelname)s||%(message)s||||%(mdc)s \t" + mdcfmt: "{requestID} {invocationID} {serviceName} {serviceIP}" datefmt: "%Y-%m-%d %H:%M:%S" (): onaplogging.mdcformatter.MDCFormatter diff --git a/vio/vio/pub/vim/vimapi/nova/OperateHypervisor.py b/vio/vio/pub/vim/vimapi/nova/OperateHypervisor.py index a9ad572..1249162 100644 --- a/vio/vio/pub/vim/vimapi/nova/OperateHypervisor.py +++ b/vio/vio/pub/vim/vimapi/nova/OperateHypervisor.py @@ -24,7 +24,7 @@ class OperateHypervisor(OperateNova): def get_hypervisor(self, data, hypervisor, **kwargs): try: return self.request('get_hypervisor', data, - project_id=data['project_id'], + project_id=data.get('project_id'), hypervisor=hypervisor, **kwargs) except exceptions.ResourceNotFound: @@ -33,7 +33,7 @@ class OperateHypervisor(OperateNova): def list_hypervisors(self, data, **query): try: return self.request('list_hypervisors', data, - project_id=data['project_id'], + project_id=data.get('project_id'), **query) except exceptions.ResourceNotFound: return None diff --git a/vio/vio/settings.py b/vio/vio/settings.py index e82b523..f1dad0f 100644 --- a/vio/vio/settings.py +++ b/vio/vio/settings.py @@ -51,6 +51,7 @@ MIDDLEWARE_CLASSES = [ 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'vio.middleware.LogContextMiddleware', ] ROOT_URLCONF = 'vio.urls' @@ -94,7 +95,6 @@ config.yamlConfig(filepath=LOGGING_FILE, watchDog=True) if 'test' in sys.argv: from vio.pub.config import config - config.REG_TO_MSB_WHEN_START = False DATABASES = {} DATABASES['default'] = { 'ENGINE': 'django.db.backends.sqlite3', diff --git a/vio/vio/swagger/urls.py b/vio/vio/swagger/urls.py index 89a5974..2dcae22 100644 --- a/vio/vio/swagger/urls.py +++ b/vio/vio/swagger/urls.py @@ -48,6 +48,9 @@ from vio.swagger.views.proxyplugin.dns.views import DesignateVersionLink from vio.swagger.views.registry.views import Registry from vio.swagger.views.registry.views import UnRegistry +# Capacity Check +from vio.swagger.views.capacity.views import CapacityCheck + # Extensions from vio.swagger.views.extensions.views import Extensions @@ -229,6 +232,10 @@ urlpatterns = [ url(r'^api/multicloud-vio/v0/(?P<vimid>[0-9a-z-A-Z\-\_]+)$', UnRegistry.as_view()), + # CapacityCheck + url(r'^api/multicloud-vio/v0/(?P<vimid>[0-9a-z-A-Z\-\_]+)/capacity_check$', + CapacityCheck.as_view()), + # proxy url(r'^api/multicloud-vio/v0/(?P<vimid>[0-9a-z-A-Z\-\_]+)/identity/v3', TokenView.as_view()), diff --git a/vio/vio/swagger/views/capacity/__init__.py b/vio/vio/swagger/views/capacity/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/vio/vio/swagger/views/capacity/__init__.py diff --git a/vio/vio/swagger/views/capacity/views.py b/vio/vio/swagger/views/capacity/views.py new file mode 100644 index 0000000..ce5d854 --- /dev/null +++ b/vio/vio/swagger/views/capacity/views.py @@ -0,0 +1,123 @@ +# Copyright (c) 2017-2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import json +import logging + +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView + +from vio.pub.exceptions import VimDriverVioException +from vio.pub.msapi import extsys +from vio.pub.vim.vimapi.nova import OperateHypervisor +from vio.pub.vim.vimapi.nova import OperateLimits + +from cinderclient import client + + +logger = logging.getLogger(__name__) + + +class CapacityCheck(APIView): + + def _check_capacity(self, hypervisor, requirement): + avail_vcpu = hypervisor.get("vcpus") - hypervisor.get("vcpus_used") + avail_mem = (hypervisor.get("memory_size") - hypervisor.get( + "memory_used"))/1024 + avail_disk = hypervisor.get("local_disk_size") - hypervisor.get( + "local_disk_used") + req_vcpu = requirement.get("vCPU") + req_mem = requirement.get("Memory") + req_disk = requirement.get("Storage") + if avail_vcpu < req_vcpu: + return False + if avail_mem < req_mem: + return False + if avail_disk < req_disk: + return False + return True + + def _check_nova_limits(self, limits, requirement): + avail_vcpu = ( + limits.absolute.total_cores - limits.absolute.total_cores_used) + avail_mem = ( + limits.absolute.total_ram - limits.absolute.total_ram_used)/1024 + req_vcpu = requirement.get("vCPU") + req_mem = requirement.get("Memory") + if avail_vcpu >= 0 and avail_vcpu < req_vcpu: + return False + if avail_mem >= 0 and avail_mem < req_mem: + return False + return True + + def _check_cinder_limits(self, limits, requirement): + absolute = limits['absolute'] + avail_disk = (absolute.get( + "maxTotalVolumeGigabytes") - absolute.get("totalGigabytesUsed")) + req_disk = requirement.get("Storage") + if avail_disk < req_disk: + return False + return True + + def post(self, request, vimid): + try: + requirement = json.loads(request.body) + except json.JSONDecodeError as ex: + return Response(data={'error': str(ex)}, + status=status.HTTP_400_BAD_REQUEST) + try: + vim_info = extsys.get_vim_by_id(vimid) + except VimDriverVioException as e: + return Response(data={'error': str(e)}, status=e.status_code) + auth_info = { + "username": vim_info['userName'], + "password": vim_info['password'], + "url": vim_info['url'], + "project_name": vim_info['tenant'] + } + + # check nova limits + servers_op = OperateLimits.OperateLimits() + try: + nova_limits = servers_op.get_limits(auth_info, None) + except Exception as e: + logger.exception("get nova limits error %(e)s", {"e": e}) + return Response(data={'error': str(e)}, status=e.status_code) + if not self._check_nova_limits(nova_limits, requirement): + return Response( + data={'result': False}, status=status.HTTP_200_OK) + # check cinder limits + cinder = client.Client( + "2", auth_info['username'], auth_info['password'], + auth_info['project_name'], auth_info['url'], insecure=True) + try: + limits = cinder.limits.get().to_dict() + except Exception as e: + logger.exception("get cinder limits error %(e)s", {"e": e}) + return Response(data={'error': str(e)}, status=e.status_code) + if not self._check_cinder_limits(limits, requirement): + return Response( + data={'result': False}, status=status.HTTP_200_OK) + # check hypervisor resources + hypervisor_op = OperateHypervisor.OperateHypervisor() + try: + hypervisors = hypervisor_op.list_hypervisors(auth_info) + except Exception as e: + logger.exception("get hypervisors error %(e)s", {"e": e}) + return Response(data={'error': str(e)}, status=e.status_code) + for hypervisor in hypervisors: + hyper = hypervisor_op.get_hypervisor(auth_info, hypervisor.id) + if self._check_capacity(hyper.to_dict(), requirement): + return Response(data={'result': True}, + status=status.HTTP_200_OK) + return Response(data={'result': False}, status=status.HTTP_200_OK) diff --git a/vio/vio/tests/test_capacity_view.py b/vio/vio/tests/test_capacity_view.py new file mode 100644 index 0000000..174d73e --- /dev/null +++ b/vio/vio/tests/test_capacity_view.py @@ -0,0 +1,104 @@ +# Copyright (c) 2017-2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import mock +import unittest + +from rest_framework import status + +from vio.pub.msapi import extsys +from vio.pub.vim.vimapi.nova import OperateHypervisor +from vio.pub.vim.vimapi.nova import OperateLimits +from vio.swagger.views.capacity.views import CapacityCheck + +from cinderclient import client + + +VIM_INFO = {'vimId': 1, 'name': 'openstack_regionone', 'userName': 'user1', + 'password': '1234', 'url': 'abc', 'tenant': 't1'} + + +class CapacityCheckTest(unittest.TestCase): + + def setUp(self): + self.view = CapacityCheck() + + @mock.patch.object(OperateHypervisor, "OperateHypervisor") + @mock.patch.object(OperateLimits, "OperateLimits") + @mock.patch.object(client, "Client") + @mock.patch.object(extsys, "get_vim_by_id") + def test_check_capacity_success(self, mock_get_vim, mock_cinder, + mock_limit, mock_hypervisor): + mock_get_vim.return_value = VIM_INFO + req = mock.Mock() + req.body = """{ + "vCPU": 1, + "Memory": 1, + "Storage": 500 + }""" + oplimits = mock.Mock() + absolute = mock.Mock( + total_cores=20, total_cores_used=1, + total_ram=128*1024, total_ram_used=4*1024) + oplimits.get_limits.return_value = mock.Mock(absolute=absolute) + mock_limit.return_value = oplimits + + climits = mock.Mock() + climits.to_dict.return_value = { + "absolute": { + "maxTotalVolumeGigabytes": 5000, + "totalGigabytesUsed": 100 + } + } + cclient = mock.Mock() + cclient.limits.get.return_value = climits + mock_cinder.return_value = cclient + + ophypervisor = mock.Mock() + ophypervisor.list_hypervisors.return_value = [ + mock.Mock(id="compute01")] + hyper = mock.Mock() + hyper.to_dict.return_value = { + "vcpus": 20, + "vcpus_used": 1, + "memory_size": 128*1024, + "memory_used": 4*1024, + "local_disk_size": 5000, + "local_disk_used": 100 + } + ophypervisor.get_hypervisor.return_value = hyper + mock_hypervisor.return_value = ophypervisor + resp = self.view.post(req, "openstack_regionone") + self.assertEqual(status.HTTP_200_OK, resp.status_code) + self.assertEqual({"result": True}, resp.data) + + @mock.patch.object(OperateLimits, "OperateLimits") + @mock.patch.object(extsys, "get_vim_by_id") + def test_check_capacity_nova_limits_failed( + self, mock_get_vim, mock_limit): + mock_get_vim.return_value = VIM_INFO + req = mock.Mock() + req.body = """{ + "vCPU": 1, + "Memory": 1, + "Storage": 500 + }""" + oplimits = mock.Mock() + absolute = mock.Mock( + total_cores=1, total_cores_used=1, + total_ram=128*1024, total_ram_used=4*1024) + oplimits.get_limits.return_value = mock.Mock(absolute=absolute) + mock_limit.return_value = oplimits + + resp = self.view.post(req, "openstack_regionone") + self.assertEqual(status.HTTP_200_OK, resp.status_code) + self.assertEqual({"result": False}, resp.data) diff --git a/vio/vio/urls.py b/vio/vio/urls.py index 9166774..85b3d9c 100644 --- a/vio/vio/urls.py +++ b/vio/vio/urls.py @@ -11,18 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. from django.conf.urls import include, url -from vio.pub.config.config import REG_TO_MSB_WHEN_START -from vio.pub.config.config import REG_TO_MSB_REG_URL -from vio.pub.config.config import REG_TO_MSB_REG_PARAM urlpatterns = [ url(r'^', include('vio.swagger.urls')), url(r'^', include('vio.samples.urls')), ] - -# regist to MSB when startup -if REG_TO_MSB_WHEN_START: - import json - from vio.pub.utils.restcall import req_by_msb - req_by_msb(REG_TO_MSB_REG_URL, "POST", - json.JSONEncoder().encode(REG_TO_MSB_REG_PARAM)) |