diff options
author | Arthur Martella <arthur.martella.1@att.com> | 2019-03-15 12:22:05 -0400 |
---|---|---|
committer | Arthur Martella <arthur.martella.1@att.com> | 2019-03-15 12:22:05 -0400 |
commit | 41656784593420d919be18951579be5df733ab0b (patch) | |
tree | ab8a0e6d428fe55e9ebf4405fb5ddbc5cc318094 | |
parent | 8410d8250bafc4ecb7b4206d8b60558e38c33557 (diff) |
Initial upload of F-GPS seed code 9/21
Includes:
Engine search filters
Change-Id: Ie5d94e1c41c6529a5c3c5247b449f8b6ef645ed3
Issue-ID: OPTFRA-440
Signed-off-by: arthur.martella.1@att.com
14 files changed, 1074 insertions, 0 deletions
diff --git a/engine/src/valet/engine/search/filters/__init__.py b/engine/src/valet/engine/search/filters/__init__.py new file mode 100644 index 0000000..bd50995 --- /dev/null +++ b/engine/src/valet/engine/search/filters/__init__.py @@ -0,0 +1,18 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/engine/src/valet/engine/search/filters/affinity_filter.py b/engine/src/valet/engine/search/filters/affinity_filter.py new file mode 100644 index 0000000..fb9aadc --- /dev/null +++ b/engine/src/valet/engine/search/filters/affinity_filter.py @@ -0,0 +1,69 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +from valet.engine.app_manager.group import Group + + +class AffinityFilter(object): + + def __init__(self): + self.name = "affinity" + + self.affinity_id = None + self.is_first = True + + self.status = None + + def init_condition(self): + self.affinity_id = None + self.is_first = True + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + if isinstance(_v, Group): + self.affinity_id = _v.vid + + if self.affinity_id in _avail_groups.keys(): + self.is_first = False + + if self.affinity_id is not None: + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + if self.is_first: + return _candidate_list + + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _candidate): + """Filter based on named affinity group.""" + + memberships = _candidate.get_all_memberships(_level) + for gk, gr in memberships.iteritems(): + if gr.group_type == "affinity" and gk == self.affinity_id: + return True + + return False diff --git a/engine/src/valet/engine/search/filters/aggregate_instance_filter.py b/engine/src/valet/engine/search/filters/aggregate_instance_filter.py new file mode 100644 index 0000000..316388e --- /dev/null +++ b/engine/src/valet/engine/search/filters/aggregate_instance_filter.py @@ -0,0 +1,106 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +import filter_utils +import six + + +_SCOPE = 'aggregate_instance_extra_specs' + + +class AggregateInstanceExtraSpecsFilter(object): + """AggregateInstanceExtraSpecsFilter works with InstanceType records.""" + + def __init__(self): + self.name = "aggregate-instance-extra-specs" + self.avail_hosts = {} + self.status = None + + def init_condition(self): + self.avail_hosts = {} + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + if len(_v.extra_specs_list) > 0: + self.avail_hosts = _avail_hosts + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, _v, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _v, _candidate): + """Check given candidate host if instance's extra specs matches to metadata.""" + + # If the candidate's host_type is not determined, skip the filter. + if _level == "host": + if len(_candidate.candidate_host_types) > 0: + return True + else: + # In rack level, if any host's host_type in the rack is not determined, + # skip the filter + for _, rh in self.avail_hosts.iteritems(): + if rh.rack_name == _candidate.rack_name: + if len(rh.candidate_host_types) > 0: + return True + + metadatas = filter_utils.aggregate_metadata_get_by_host(_level, _candidate) + + for extra_specs in _v.extra_specs_list: + for gk, metadata in metadatas.iteritems(): + if self._match_metadata(gk, extra_specs, metadata): + break + else: + return False + + return True + + def _match_metadata(self, _g_name, _extra_specs, _metadata): + """Match conditions + - No extra_specs + - Different SCOPE of extra_specs keys + - key of extra_specs exists in metadata & any value matches + """ + + for key, req in six.iteritems(_extra_specs): + scope = key.split(':', 1) + if len(scope) > 1: + if scope[0] != _SCOPE: + continue + else: + del scope[0] + key = scope[0] + + aggregate_vals = _metadata.get(key, None) + if not aggregate_vals: + return False + + for aggregate_val in aggregate_vals: + if filter_utils.match(aggregate_val, req): + break + else: + return False + + return True diff --git a/engine/src/valet/engine/search/filters/az_filter.py b/engine/src/valet/engine/search/filters/az_filter.py new file mode 100644 index 0000000..8902893 --- /dev/null +++ b/engine/src/valet/engine/search/filters/az_filter.py @@ -0,0 +1,74 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +import filter_utils + +from valet.engine.app_manager.group import Group +from valet.engine.app_manager.server import Server + + +class AvailabilityZoneFilter(object): + """Filters Hosts by availability zone. + + Works with aggregate metadata availability zones, using the key 'availability_zone'. + + Note: in theory a compute node can be part of multiple availability_zones + """ + + def __init__(self): + self.name = "availability-zone" + + self.status = None + + def init_condition(self): + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + if (isinstance(_v, Server) and _v.availability_zone is not None) or \ + (isinstance(_v, Group) and len(_v.availability_zone_list) > 0): + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, _v, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _v, _candidate): + az_request_list = [] + if isinstance(_v, Server): + az_request_list.append(_v.availability_zone) + else: + for az in _v.availability_zone_list: + az_request_list.append(az) + + if len(az_request_list) == 0: + return True + + availability_zone_list = filter_utils.availability_zone_get_by_host(_level, _candidate) + + for azr in az_request_list: + if azr not in availability_zone_list: + return False + + return True diff --git a/engine/src/valet/engine/search/filters/cpu_filter.py b/engine/src/valet/engine/search/filters/cpu_filter.py new file mode 100644 index 0000000..95dba8c --- /dev/null +++ b/engine/src/valet/engine/search/filters/cpu_filter.py @@ -0,0 +1,57 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +class CPUFilter(object): + + def __init__(self): + self.name = "cpu" + + self.status = None + + def init_condition(self): + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + return True + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, _v, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _v, _candidate): + """Return True if host has sufficient CPU cores.""" + + avail_vcpus = _candidate.get_vcpus(_level) + + instance_vcpus = _v.vCPUs + + # TODO: need to check against original CPUs? + # Do not allow an instance to overcommit against itself, + # only against other instances. + # if instance_vcpus > vCPUs: + # return False + + if avail_vcpus < instance_vcpus: + return False + + return True diff --git a/engine/src/valet/engine/search/filters/disk_filter.py b/engine/src/valet/engine/search/filters/disk_filter.py new file mode 100644 index 0000000..00fa93e --- /dev/null +++ b/engine/src/valet/engine/search/filters/disk_filter.py @@ -0,0 +1,50 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +class DiskFilter(object): + + def __init__(self): + self.name = "disk" + + self.status = None + + def init_condition(self): + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + return True + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, _v, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _v, _candidate): + """Filter based on disk usage.""" + + requested_disk = _v.local_volume_size + usable_disk = _candidate.get_local_disk(_level) + + if not usable_disk >= requested_disk: + return False + + return True diff --git a/engine/src/valet/engine/search/filters/diversity_filter.py b/engine/src/valet/engine/search/filters/diversity_filter.py new file mode 100644 index 0000000..882e11a --- /dev/null +++ b/engine/src/valet/engine/search/filters/diversity_filter.py @@ -0,0 +1,62 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +class DiversityFilter(object): + + def __init__(self): + self.name = "diversity" + + self.diversity_list = [] + + self.status = None + + def init_condition(self): + self.diversity_list = [] + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + if len(_v.diversity_groups) > 0: + for _, div_group in _v.diversity_groups.iteritems(): + if div_group.level == _level: + self.diversity_list.append(div_group.vid) + + if len(self.diversity_list) > 0: + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _candidate): + """Filter based on named diversity groups.""" + + memberships = _candidate.get_memberships(_level) + + for diversity_id in self.diversity_list: + for gk, gr in memberships.iteritems(): + if gr.group_type == "diversity" and gk == diversity_id: + return False + + return True diff --git a/engine/src/valet/engine/search/filters/dynamic_aggregate_filter.py b/engine/src/valet/engine/search/filters/dynamic_aggregate_filter.py new file mode 100644 index 0000000..709a9b9 --- /dev/null +++ b/engine/src/valet/engine/search/filters/dynamic_aggregate_filter.py @@ -0,0 +1,141 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +import copy + +from valet.engine.app_manager.server import Server +from valet.engine.search.filters.aggregate_instance_filter import AggregateInstanceExtraSpecsFilter +from valet.engine.search.filters.cpu_filter import CPUFilter +from valet.engine.search.filters.disk_filter import DiskFilter +from valet.engine.search.filters.mem_filter import MemFilter +from valet.engine.search.filters.numa_filter import NUMAFilter + + +class DynamicAggregateFilter(object): + + def __init__(self): + self.name = "dynamic-aggregate" + + self.avail_hosts = {} + self.avail_groups = {} + + self.aggr_filter = AggregateInstanceExtraSpecsFilter() + self.cpu_filter = CPUFilter() + self.mem_filter = MemFilter() + self.disk_filter = DiskFilter() + self.numa_filter = NUMAFilter() + + self.status = None + + def init_condition(self): + self.avail_hosts = {} + self.avail_groups = {} + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + if _level == "host" and isinstance(_v, Server): + self.avail_hosts = _avail_hosts + self.avail_groups = _avail_groups + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + specified_candidate_list = [] # candidates having specific host type + unspecified_candidate_list = [] # candidates not having specific host type + + for c in _candidate_list: + if len(c.candidate_host_types) == 0: + specified_candidate_list.append(c) + else: + unspecified_candidate_list.append(c) + + # Try to use existing hosts that have specific host type + if len(specified_candidate_list) > 0: + return specified_candidate_list + + # Take just one candidate + candidate = unspecified_candidate_list[0] + + # Get the host-aggregate of _v + flavor_type_list = _v.get_flavor_types() + if len(flavor_type_list) > 1: + self.status = "have more than one flavor type" + return [] + + ha = self.avail_groups[flavor_type_list[0]] + + # Add the host-aggregate into host and rack memberships. + # Adjust host with avail cpus, mem, disk, and numa + candidate.adjust_avail_resources(ha) + + # Change all others in the same rack. + for hrk, hr in self.avail_hosts.iteritems(): + if hrk != candidate.host_name: + if hr.rack_name == candidate.rack_name: + hr.adjust_avail_rack_resources(ha, + candidate.rack_avail_vCPUs, + candidate.rack_avail_mem, + candidate.rack_avail_local_disk) + + # Once the host type (ha) is determined, remove candidate_host_types + candidate.old_candidate_host_types = copy.deepcopy(candidate.candidate_host_types) + candidate.candidate_host_types.clear() + + # Filter against host-aggregate, cpu, mem, disk, numa + + self.aggr_filter.init_condition() + if self.aggr_filter.check_pre_condition(_level, _v, self.avail_hosts, self.avail_groups): + if not self.aggr_filter._check_candidate(_level, _v, candidate): + self.status = "host-aggregate violation" + + self.cpu_filter.init_condition() + if not self.cpu_filter._check_candidate(_level, _v, candidate): + self.status = "cpu violation" + + self.mem_filter.init_condition() + if not self.mem_filter._check_candidate(_level, _v, candidate): + self.status = "mem violation" + + self.disk_filter.init_condition() + if not self.disk_filter._check_candidate(_level, _v, candidate): + self.status = "disk violation" + + self.numa_filter.init_condition() + if self.numa_filter.check_pre_condition(_level, _v, self.avail_hosts, self.avail_groups): + if not self.numa_filter._check_candidate(_level, _v, candidate): + self.status = "numa violation" + + if self.status is None: + # Candidate not filtered. + return [candidate] + else: + # Rollback + candidate.rollback_avail_resources(ha) + candidate.candidate_host_types = copy.deepcopy(candidate.old_candidate_host_types) + candidate.old_candidate_host_types.clear() + + for hrk, hr in self.avail_hosts.iteritems(): + if hrk != candidate.host_name: + if hr.rack_name == candidate.rack_name: + hr.rollback_avail_rack_resources(ha, + candidate.rack_avail_vCPUs, + candidate.rack_avail_mem, + candidate.rack_avail_local_disk) + + return [] diff --git a/engine/src/valet/engine/search/filters/exclusivity_filter.py b/engine/src/valet/engine/search/filters/exclusivity_filter.py new file mode 100644 index 0000000..77efd97 --- /dev/null +++ b/engine/src/valet/engine/search/filters/exclusivity_filter.py @@ -0,0 +1,81 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +class ExclusivityFilter(object): + + def __init__(self): + self.name = "exclusivity" + + self.exclusivity_id = None + + self.status = None + + def init_condition(self): + self.exclusivity_id = None + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + exclusivities = _v.get_exclusivities(_level) + + if len(exclusivities) > 1: + self.status = "multiple exclusivities for node = " + _v.vid + return False + + if len(exclusivities) == 1: + ex_group = exclusivities[exclusivities.keys()[0]] + + if ex_group.level == _level: + self.exclusivity_id = ex_group.vid + + if self.exclusivity_id is not None: + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + + candidate_list = self._get_candidates(_level, _candidate_list) + + return candidate_list + + def _get_candidates(self, _level, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_exclusive_candidate(_level, c) is True or \ + self._check_empty(_level, c) is True: + candidate_list.append(c) + + return candidate_list + + def _check_exclusive_candidate(self, _level, _candidate): + memberships = _candidate.get_memberships(_level) + + for gk, gr in memberships.iteritems(): + if gr.group_type == "exclusivity" and gk == self.exclusivity_id: + return True + + return False + + def _check_empty(self, _level, _candidate): + num_of_placed_servers = _candidate.get_num_of_placed_servers(_level) + + if num_of_placed_servers == 0: + return True + + return False diff --git a/engine/src/valet/engine/search/filters/filter_utils.py b/engine/src/valet/engine/search/filters/filter_utils.py new file mode 100644 index 0000000..1005263 --- /dev/null +++ b/engine/src/valet/engine/search/filters/filter_utils.py @@ -0,0 +1,117 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +import collections +import operator + + +# 1. The following operations are supported: +# =, s==, s!=, s>=, s>, s<=, s<, <in>, <all-in>, <or>, ==, !=, >=, <= +# 2. Note that <or> is handled in a different way below. +# 3. If the first word in the extra_specs is not one of the operators, +# it is ignored. +op_methods = {'=': lambda x, y: float(x) >= float(y), + '<in>': lambda x, y: y in x, + '<all-in>': lambda x, y: all(val in x for val in y), + '==': lambda x, y: float(x) == float(y), + '!=': lambda x, y: float(x) != float(y), + '>=': lambda x, y: float(x) >= float(y), + '<=': lambda x, y: float(x) <= float(y), + 's==': operator.eq, + 's!=': operator.ne, + 's<': operator.lt, + 's<=': operator.le, + 's>': operator.gt, + 's>=': operator.ge} + + +def match(value, req): + words = req.split() + + op = method = None + if words: + op = words.pop(0) + method = op_methods.get(op) + + if op != '<or>' and not method: + return value == req + + if value is None: + return False + + if op == '<or>': # Ex: <or> v1 <or> v2 <or> v3 + while True: + if words.pop(0) == value: + return True + if not words: + break + words.pop(0) # remove a keyword <or> + if not words: + break + return False + + if words: + if op == '<all-in>': # requires a list not a string + return method(value, words) + return method(value, words[0]) + return False + + +def aggregate_metadata_get_by_host(_level, _host, _key=None): + """Returns a dict of all metadata based on a metadata key for a specific host. + + If the key is not provided, returns a dict of all metadata. + """ + + metadatas = {} + + groups = _host.get_memberships(_level) + + for gk, g in groups.iteritems(): + if g.group_type == "aggr": + if _key is None or _key in g.metadata: + metadata = collections.defaultdict(set) + for k, v in g.metadata.items(): + if k != "prior_metadata": + metadata[k].update(x.strip() for x in v.split(',')) + else: + # metadata[k] = v + if isinstance(g.metadata["prior_metadata"], dict): + for ik, iv in g.metadata["prior_metadata"].items(): + metadata[ik].update(y.strip() for y in iv.split(',')) + metadatas[gk] = metadata + + return metadatas + + +def availability_zone_get_by_host(_level, _host): + availability_zone_list = [] + + groups = _host.get_memberships(_level) + + for gk, g in groups.iteritems(): + if g.group_type == "az": + g_name_elements = gk.split(':', 1) + if len(g_name_elements) > 1: + g_name = g_name_elements[1] + else: + g_name = gk + + availability_zone_list.append(g_name) + + return availability_zone_list diff --git a/engine/src/valet/engine/search/filters/mem_filter.py b/engine/src/valet/engine/search/filters/mem_filter.py new file mode 100644 index 0000000..1b494c2 --- /dev/null +++ b/engine/src/valet/engine/search/filters/mem_filter.py @@ -0,0 +1,56 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +class MemFilter(object): + + def __init__(self): + self.name = "mem" + + self.status = None + + def init_condition(self): + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + return True + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, _v, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _v, _candidate): + """Only return hosts with sufficient available RAM.""" + + requested_ram = _v.mem # MB + usable_ram = _candidate.get_mem(_level) + + # TODO: need to check against original mem_cap? + # Do not allow an instance to overcommit against itself, + # only against other instances. + # if not total_ram >= requested_ram: + # return False + + if not usable_ram >= requested_ram: + return False + + return True diff --git a/engine/src/valet/engine/search/filters/no_exclusivity_filter.py b/engine/src/valet/engine/search/filters/no_exclusivity_filter.py new file mode 100644 index 0000000..43516fe --- /dev/null +++ b/engine/src/valet/engine/search/filters/no_exclusivity_filter.py @@ -0,0 +1,53 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +class NoExclusivityFilter(object): + + def __init__(self): + self.name = "no-exclusivity" + + self.status = None + + def init_condition(self): + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + exclusivities = _v.get_exclusivities(_level) + + if len(exclusivities) == 0: + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _candidate): + memberships = _candidate.get_memberships(_level) + + for _, g in memberships.iteritems(): + if g.group_type == "exclusivity" and g.level == _level: + return False + + return True diff --git a/engine/src/valet/engine/search/filters/numa_filter.py b/engine/src/valet/engine/search/filters/numa_filter.py new file mode 100644 index 0000000..3e095ec --- /dev/null +++ b/engine/src/valet/engine/search/filters/numa_filter.py @@ -0,0 +1,84 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +from valet.engine.app_manager.server import Server + + +_SCOPE = 'hw' + + +class NUMAFilter(object): + """Check NUMA alignment request in Flavor.""" + + def __init__(self): + """Define filter name and status.""" + + self.name = "numa" + + self.status = None + + def init_condition(self): + """Init variable.""" + + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + """Check if given server needs to check this filter.""" + + if _level == "host" and isinstance(_v, Server): + if _v.need_numa_alignment(): + return True + + return False + + def filter_candidates(self, _level, _v, _candidate_list): + """Check and filter one candidate at a time.""" + + candidate_list = [] + + for c in _candidate_list: + if self._check_candidate(_level, _v, c): + candidate_list.append(c) + + return candidate_list + + def _check_candidate(self, _level, _v, _candidate): + """Check given candidate host if it meets numa requirement.""" + + # servers = [] + # if isinstance(_v, Group): + # _v.get_servers(servers) + # else: + # servers.append(_v) + + # (vcpus_demand, mem_demand) = self._get_demand_with_numa(servers) + + return _candidate.NUMA.has_enough_resources(_v.vCPUs, _v.mem) + + def _get_demand_with_numa(self, _servers): + """Check numa and compute the amount of vCPUs and memory.""" + + vcpus = 0 + mem = 0 + + for s in _servers: + if s.need_numa_alignment(): + vcpus += s.vCPUs + mem += s.mem + + return vcpus, mem diff --git a/engine/src/valet/engine/search/filters/quorum_diversity_filter.py b/engine/src/valet/engine/search/filters/quorum_diversity_filter.py new file mode 100644 index 0000000..6388ebc --- /dev/null +++ b/engine/src/valet/engine/search/filters/quorum_diversity_filter.py @@ -0,0 +1,106 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +import math + + +class QuorumDiversityFilter(object): + + def __init__(self): + self.name = "quorum-diversity" + + self.quorum_diversity_group_list = [] + + self.status = None + + def init_condition(self): + self.quorum_diversity_group_list = [] + self.status = None + + def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups): + if len(_v.quorum_diversity_groups) > 0: + for _, qdiv_group in _v.quorum_diversity_groups.iteritems(): + if qdiv_group.level == _level: + self.quorum_diversity_group_list.append(qdiv_group) + + if len(self.quorum_diversity_group_list) > 0: + return True + else: + return False + + def filter_candidates(self, _level, _v, _candidate_list): + candidate_list = [] + + # First, try diversity rule. + + for c in _candidate_list: + if self._check_diversity_candidate(_level, c): + candidate_list.append(c) + + if len(candidate_list) > 0: + return candidate_list + + # Second, if no available hosts for diversity rule, try quorum rule. + + for c in _candidate_list: + if self._check_quorum_candidate(_level, c): + candidate_list.append(c) + + return candidate_list + + def _check_diversity_candidate(self, _level, _candidate): + """Filter based on named diversity groups.""" + + memberships = _candidate.get_memberships(_level) + + for qdiv in self.quorum_diversity_group_list: + for gk, gr in memberships.iteritems(): + if gr.group_type == "quorum-diversity" and gk == qdiv.vid: + return False + + return True + + def _check_quorum_candidate(self, _level, _candidate): + """Filter based on quorum-diversity rule.""" + + memberships = _candidate.get_memberships(_level) + hk = _candidate.get_resource_name(_level) + + for qdiv in self.quorum_diversity_group_list: + # Requested num of servers under this rule + total_num_of_servers = len(qdiv.server_list) + + num_of_placed_servers_in_candidate = -1 + + for gk, gr in memberships.iteritems(): + if gr.group_type == "quorum-diversity" and gk == qdiv.vid: + # Total num of servers under this rule + total_num_of_servers += gr.original_num_of_placed_servers + + if hk in gr.num_of_placed_servers_of_host.keys(): + num_of_placed_servers_in_candidate = gr.num_of_placed_servers_of_host[hk] + + break + + # Allowed maximum num of servers per host + quorum = max(math.ceil(float(total_num_of_servers) / 2.0 - 1.0), 1.0) + + if num_of_placed_servers_in_candidate >= quorum: + return False + + return True |