summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArthur Martella <arthur.martella.1@att.com>2019-03-15 12:22:05 -0400
committerArthur Martella <arthur.martella.1@att.com>2019-03-15 12:22:05 -0400
commit41656784593420d919be18951579be5df733ab0b (patch)
treeab8a0e6d428fe55e9ebf4405fb5ddbc5cc318094
parent8410d8250bafc4ecb7b4206d8b60558e38c33557 (diff)
Initial upload of F-GPS seed code 9/21
Includes: Engine search filters Change-Id: Ie5d94e1c41c6529a5c3c5247b449f8b6ef645ed3 Issue-ID: OPTFRA-440 Signed-off-by: arthur.martella.1@att.com
-rw-r--r--engine/src/valet/engine/search/filters/__init__.py18
-rw-r--r--engine/src/valet/engine/search/filters/affinity_filter.py69
-rw-r--r--engine/src/valet/engine/search/filters/aggregate_instance_filter.py106
-rw-r--r--engine/src/valet/engine/search/filters/az_filter.py74
-rw-r--r--engine/src/valet/engine/search/filters/cpu_filter.py57
-rw-r--r--engine/src/valet/engine/search/filters/disk_filter.py50
-rw-r--r--engine/src/valet/engine/search/filters/diversity_filter.py62
-rw-r--r--engine/src/valet/engine/search/filters/dynamic_aggregate_filter.py141
-rw-r--r--engine/src/valet/engine/search/filters/exclusivity_filter.py81
-rw-r--r--engine/src/valet/engine/search/filters/filter_utils.py117
-rw-r--r--engine/src/valet/engine/search/filters/mem_filter.py56
-rw-r--r--engine/src/valet/engine/search/filters/no_exclusivity_filter.py53
-rw-r--r--engine/src/valet/engine/search/filters/numa_filter.py84
-rw-r--r--engine/src/valet/engine/search/filters/quorum_diversity_filter.py106
14 files changed, 1074 insertions, 0 deletions
diff --git a/engine/src/valet/engine/search/filters/__init__.py b/engine/src/valet/engine/search/filters/__init__.py
new file mode 100644
index 0000000..bd50995
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/__init__.py
@@ -0,0 +1,18 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
diff --git a/engine/src/valet/engine/search/filters/affinity_filter.py b/engine/src/valet/engine/search/filters/affinity_filter.py
new file mode 100644
index 0000000..fb9aadc
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/affinity_filter.py
@@ -0,0 +1,69 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+from valet.engine.app_manager.group import Group
+
+
+class AffinityFilter(object):
+
+ def __init__(self):
+ self.name = "affinity"
+
+ self.affinity_id = None
+ self.is_first = True
+
+ self.status = None
+
+ def init_condition(self):
+ self.affinity_id = None
+ self.is_first = True
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ if isinstance(_v, Group):
+ self.affinity_id = _v.vid
+
+ if self.affinity_id in _avail_groups.keys():
+ self.is_first = False
+
+ if self.affinity_id is not None:
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ if self.is_first:
+ return _candidate_list
+
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _candidate):
+ """Filter based on named affinity group."""
+
+ memberships = _candidate.get_all_memberships(_level)
+ for gk, gr in memberships.iteritems():
+ if gr.group_type == "affinity" and gk == self.affinity_id:
+ return True
+
+ return False
diff --git a/engine/src/valet/engine/search/filters/aggregate_instance_filter.py b/engine/src/valet/engine/search/filters/aggregate_instance_filter.py
new file mode 100644
index 0000000..316388e
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/aggregate_instance_filter.py
@@ -0,0 +1,106 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+import filter_utils
+import six
+
+
+_SCOPE = 'aggregate_instance_extra_specs'
+
+
+class AggregateInstanceExtraSpecsFilter(object):
+ """AggregateInstanceExtraSpecsFilter works with InstanceType records."""
+
+ def __init__(self):
+ self.name = "aggregate-instance-extra-specs"
+ self.avail_hosts = {}
+ self.status = None
+
+ def init_condition(self):
+ self.avail_hosts = {}
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ if len(_v.extra_specs_list) > 0:
+ self.avail_hosts = _avail_hosts
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, _v, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _v, _candidate):
+ """Check given candidate host if instance's extra specs matches to metadata."""
+
+ # If the candidate's host_type is not determined, skip the filter.
+ if _level == "host":
+ if len(_candidate.candidate_host_types) > 0:
+ return True
+ else:
+ # In rack level, if any host's host_type in the rack is not determined,
+ # skip the filter
+ for _, rh in self.avail_hosts.iteritems():
+ if rh.rack_name == _candidate.rack_name:
+ if len(rh.candidate_host_types) > 0:
+ return True
+
+ metadatas = filter_utils.aggregate_metadata_get_by_host(_level, _candidate)
+
+ for extra_specs in _v.extra_specs_list:
+ for gk, metadata in metadatas.iteritems():
+ if self._match_metadata(gk, extra_specs, metadata):
+ break
+ else:
+ return False
+
+ return True
+
+ def _match_metadata(self, _g_name, _extra_specs, _metadata):
+ """Match conditions
+ - No extra_specs
+ - Different SCOPE of extra_specs keys
+ - key of extra_specs exists in metadata & any value matches
+ """
+
+ for key, req in six.iteritems(_extra_specs):
+ scope = key.split(':', 1)
+ if len(scope) > 1:
+ if scope[0] != _SCOPE:
+ continue
+ else:
+ del scope[0]
+ key = scope[0]
+
+ aggregate_vals = _metadata.get(key, None)
+ if not aggregate_vals:
+ return False
+
+ for aggregate_val in aggregate_vals:
+ if filter_utils.match(aggregate_val, req):
+ break
+ else:
+ return False
+
+ return True
diff --git a/engine/src/valet/engine/search/filters/az_filter.py b/engine/src/valet/engine/search/filters/az_filter.py
new file mode 100644
index 0000000..8902893
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/az_filter.py
@@ -0,0 +1,74 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+import filter_utils
+
+from valet.engine.app_manager.group import Group
+from valet.engine.app_manager.server import Server
+
+
+class AvailabilityZoneFilter(object):
+ """Filters Hosts by availability zone.
+
+ Works with aggregate metadata availability zones, using the key 'availability_zone'.
+
+ Note: in theory a compute node can be part of multiple availability_zones
+ """
+
+ def __init__(self):
+ self.name = "availability-zone"
+
+ self.status = None
+
+ def init_condition(self):
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ if (isinstance(_v, Server) and _v.availability_zone is not None) or \
+ (isinstance(_v, Group) and len(_v.availability_zone_list) > 0):
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, _v, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _v, _candidate):
+ az_request_list = []
+ if isinstance(_v, Server):
+ az_request_list.append(_v.availability_zone)
+ else:
+ for az in _v.availability_zone_list:
+ az_request_list.append(az)
+
+ if len(az_request_list) == 0:
+ return True
+
+ availability_zone_list = filter_utils.availability_zone_get_by_host(_level, _candidate)
+
+ for azr in az_request_list:
+ if azr not in availability_zone_list:
+ return False
+
+ return True
diff --git a/engine/src/valet/engine/search/filters/cpu_filter.py b/engine/src/valet/engine/search/filters/cpu_filter.py
new file mode 100644
index 0000000..95dba8c
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/cpu_filter.py
@@ -0,0 +1,57 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+class CPUFilter(object):
+
+ def __init__(self):
+ self.name = "cpu"
+
+ self.status = None
+
+ def init_condition(self):
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ return True
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, _v, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _v, _candidate):
+ """Return True if host has sufficient CPU cores."""
+
+ avail_vcpus = _candidate.get_vcpus(_level)
+
+ instance_vcpus = _v.vCPUs
+
+ # TODO: need to check against original CPUs?
+ # Do not allow an instance to overcommit against itself,
+ # only against other instances.
+ # if instance_vcpus > vCPUs:
+ # return False
+
+ if avail_vcpus < instance_vcpus:
+ return False
+
+ return True
diff --git a/engine/src/valet/engine/search/filters/disk_filter.py b/engine/src/valet/engine/search/filters/disk_filter.py
new file mode 100644
index 0000000..00fa93e
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/disk_filter.py
@@ -0,0 +1,50 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+class DiskFilter(object):
+
+ def __init__(self):
+ self.name = "disk"
+
+ self.status = None
+
+ def init_condition(self):
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ return True
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, _v, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _v, _candidate):
+ """Filter based on disk usage."""
+
+ requested_disk = _v.local_volume_size
+ usable_disk = _candidate.get_local_disk(_level)
+
+ if not usable_disk >= requested_disk:
+ return False
+
+ return True
diff --git a/engine/src/valet/engine/search/filters/diversity_filter.py b/engine/src/valet/engine/search/filters/diversity_filter.py
new file mode 100644
index 0000000..882e11a
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/diversity_filter.py
@@ -0,0 +1,62 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+class DiversityFilter(object):
+
+ def __init__(self):
+ self.name = "diversity"
+
+ self.diversity_list = []
+
+ self.status = None
+
+ def init_condition(self):
+ self.diversity_list = []
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ if len(_v.diversity_groups) > 0:
+ for _, div_group in _v.diversity_groups.iteritems():
+ if div_group.level == _level:
+ self.diversity_list.append(div_group.vid)
+
+ if len(self.diversity_list) > 0:
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _candidate):
+ """Filter based on named diversity groups."""
+
+ memberships = _candidate.get_memberships(_level)
+
+ for diversity_id in self.diversity_list:
+ for gk, gr in memberships.iteritems():
+ if gr.group_type == "diversity" and gk == diversity_id:
+ return False
+
+ return True
diff --git a/engine/src/valet/engine/search/filters/dynamic_aggregate_filter.py b/engine/src/valet/engine/search/filters/dynamic_aggregate_filter.py
new file mode 100644
index 0000000..709a9b9
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/dynamic_aggregate_filter.py
@@ -0,0 +1,141 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+import copy
+
+from valet.engine.app_manager.server import Server
+from valet.engine.search.filters.aggregate_instance_filter import AggregateInstanceExtraSpecsFilter
+from valet.engine.search.filters.cpu_filter import CPUFilter
+from valet.engine.search.filters.disk_filter import DiskFilter
+from valet.engine.search.filters.mem_filter import MemFilter
+from valet.engine.search.filters.numa_filter import NUMAFilter
+
+
+class DynamicAggregateFilter(object):
+
+ def __init__(self):
+ self.name = "dynamic-aggregate"
+
+ self.avail_hosts = {}
+ self.avail_groups = {}
+
+ self.aggr_filter = AggregateInstanceExtraSpecsFilter()
+ self.cpu_filter = CPUFilter()
+ self.mem_filter = MemFilter()
+ self.disk_filter = DiskFilter()
+ self.numa_filter = NUMAFilter()
+
+ self.status = None
+
+ def init_condition(self):
+ self.avail_hosts = {}
+ self.avail_groups = {}
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ if _level == "host" and isinstance(_v, Server):
+ self.avail_hosts = _avail_hosts
+ self.avail_groups = _avail_groups
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ specified_candidate_list = [] # candidates having specific host type
+ unspecified_candidate_list = [] # candidates not having specific host type
+
+ for c in _candidate_list:
+ if len(c.candidate_host_types) == 0:
+ specified_candidate_list.append(c)
+ else:
+ unspecified_candidate_list.append(c)
+
+ # Try to use existing hosts that have specific host type
+ if len(specified_candidate_list) > 0:
+ return specified_candidate_list
+
+ # Take just one candidate
+ candidate = unspecified_candidate_list[0]
+
+ # Get the host-aggregate of _v
+ flavor_type_list = _v.get_flavor_types()
+ if len(flavor_type_list) > 1:
+ self.status = "have more than one flavor type"
+ return []
+
+ ha = self.avail_groups[flavor_type_list[0]]
+
+ # Add the host-aggregate into host and rack memberships.
+ # Adjust host with avail cpus, mem, disk, and numa
+ candidate.adjust_avail_resources(ha)
+
+ # Change all others in the same rack.
+ for hrk, hr in self.avail_hosts.iteritems():
+ if hrk != candidate.host_name:
+ if hr.rack_name == candidate.rack_name:
+ hr.adjust_avail_rack_resources(ha,
+ candidate.rack_avail_vCPUs,
+ candidate.rack_avail_mem,
+ candidate.rack_avail_local_disk)
+
+ # Once the host type (ha) is determined, remove candidate_host_types
+ candidate.old_candidate_host_types = copy.deepcopy(candidate.candidate_host_types)
+ candidate.candidate_host_types.clear()
+
+ # Filter against host-aggregate, cpu, mem, disk, numa
+
+ self.aggr_filter.init_condition()
+ if self.aggr_filter.check_pre_condition(_level, _v, self.avail_hosts, self.avail_groups):
+ if not self.aggr_filter._check_candidate(_level, _v, candidate):
+ self.status = "host-aggregate violation"
+
+ self.cpu_filter.init_condition()
+ if not self.cpu_filter._check_candidate(_level, _v, candidate):
+ self.status = "cpu violation"
+
+ self.mem_filter.init_condition()
+ if not self.mem_filter._check_candidate(_level, _v, candidate):
+ self.status = "mem violation"
+
+ self.disk_filter.init_condition()
+ if not self.disk_filter._check_candidate(_level, _v, candidate):
+ self.status = "disk violation"
+
+ self.numa_filter.init_condition()
+ if self.numa_filter.check_pre_condition(_level, _v, self.avail_hosts, self.avail_groups):
+ if not self.numa_filter._check_candidate(_level, _v, candidate):
+ self.status = "numa violation"
+
+ if self.status is None:
+ # Candidate not filtered.
+ return [candidate]
+ else:
+ # Rollback
+ candidate.rollback_avail_resources(ha)
+ candidate.candidate_host_types = copy.deepcopy(candidate.old_candidate_host_types)
+ candidate.old_candidate_host_types.clear()
+
+ for hrk, hr in self.avail_hosts.iteritems():
+ if hrk != candidate.host_name:
+ if hr.rack_name == candidate.rack_name:
+ hr.rollback_avail_rack_resources(ha,
+ candidate.rack_avail_vCPUs,
+ candidate.rack_avail_mem,
+ candidate.rack_avail_local_disk)
+
+ return []
diff --git a/engine/src/valet/engine/search/filters/exclusivity_filter.py b/engine/src/valet/engine/search/filters/exclusivity_filter.py
new file mode 100644
index 0000000..77efd97
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/exclusivity_filter.py
@@ -0,0 +1,81 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+class ExclusivityFilter(object):
+
+ def __init__(self):
+ self.name = "exclusivity"
+
+ self.exclusivity_id = None
+
+ self.status = None
+
+ def init_condition(self):
+ self.exclusivity_id = None
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ exclusivities = _v.get_exclusivities(_level)
+
+ if len(exclusivities) > 1:
+ self.status = "multiple exclusivities for node = " + _v.vid
+ return False
+
+ if len(exclusivities) == 1:
+ ex_group = exclusivities[exclusivities.keys()[0]]
+
+ if ex_group.level == _level:
+ self.exclusivity_id = ex_group.vid
+
+ if self.exclusivity_id is not None:
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+
+ candidate_list = self._get_candidates(_level, _candidate_list)
+
+ return candidate_list
+
+ def _get_candidates(self, _level, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_exclusive_candidate(_level, c) is True or \
+ self._check_empty(_level, c) is True:
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_exclusive_candidate(self, _level, _candidate):
+ memberships = _candidate.get_memberships(_level)
+
+ for gk, gr in memberships.iteritems():
+ if gr.group_type == "exclusivity" and gk == self.exclusivity_id:
+ return True
+
+ return False
+
+ def _check_empty(self, _level, _candidate):
+ num_of_placed_servers = _candidate.get_num_of_placed_servers(_level)
+
+ if num_of_placed_servers == 0:
+ return True
+
+ return False
diff --git a/engine/src/valet/engine/search/filters/filter_utils.py b/engine/src/valet/engine/search/filters/filter_utils.py
new file mode 100644
index 0000000..1005263
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/filter_utils.py
@@ -0,0 +1,117 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+import collections
+import operator
+
+
+# 1. The following operations are supported:
+# =, s==, s!=, s>=, s>, s<=, s<, <in>, <all-in>, <or>, ==, !=, >=, <=
+# 2. Note that <or> is handled in a different way below.
+# 3. If the first word in the extra_specs is not one of the operators,
+# it is ignored.
+op_methods = {'=': lambda x, y: float(x) >= float(y),
+ '<in>': lambda x, y: y in x,
+ '<all-in>': lambda x, y: all(val in x for val in y),
+ '==': lambda x, y: float(x) == float(y),
+ '!=': lambda x, y: float(x) != float(y),
+ '>=': lambda x, y: float(x) >= float(y),
+ '<=': lambda x, y: float(x) <= float(y),
+ 's==': operator.eq,
+ 's!=': operator.ne,
+ 's<': operator.lt,
+ 's<=': operator.le,
+ 's>': operator.gt,
+ 's>=': operator.ge}
+
+
+def match(value, req):
+ words = req.split()
+
+ op = method = None
+ if words:
+ op = words.pop(0)
+ method = op_methods.get(op)
+
+ if op != '<or>' and not method:
+ return value == req
+
+ if value is None:
+ return False
+
+ if op == '<or>': # Ex: <or> v1 <or> v2 <or> v3
+ while True:
+ if words.pop(0) == value:
+ return True
+ if not words:
+ break
+ words.pop(0) # remove a keyword <or>
+ if not words:
+ break
+ return False
+
+ if words:
+ if op == '<all-in>': # requires a list not a string
+ return method(value, words)
+ return method(value, words[0])
+ return False
+
+
+def aggregate_metadata_get_by_host(_level, _host, _key=None):
+ """Returns a dict of all metadata based on a metadata key for a specific host.
+
+ If the key is not provided, returns a dict of all metadata.
+ """
+
+ metadatas = {}
+
+ groups = _host.get_memberships(_level)
+
+ for gk, g in groups.iteritems():
+ if g.group_type == "aggr":
+ if _key is None or _key in g.metadata:
+ metadata = collections.defaultdict(set)
+ for k, v in g.metadata.items():
+ if k != "prior_metadata":
+ metadata[k].update(x.strip() for x in v.split(','))
+ else:
+ # metadata[k] = v
+ if isinstance(g.metadata["prior_metadata"], dict):
+ for ik, iv in g.metadata["prior_metadata"].items():
+ metadata[ik].update(y.strip() for y in iv.split(','))
+ metadatas[gk] = metadata
+
+ return metadatas
+
+
+def availability_zone_get_by_host(_level, _host):
+ availability_zone_list = []
+
+ groups = _host.get_memberships(_level)
+
+ for gk, g in groups.iteritems():
+ if g.group_type == "az":
+ g_name_elements = gk.split(':', 1)
+ if len(g_name_elements) > 1:
+ g_name = g_name_elements[1]
+ else:
+ g_name = gk
+
+ availability_zone_list.append(g_name)
+
+ return availability_zone_list
diff --git a/engine/src/valet/engine/search/filters/mem_filter.py b/engine/src/valet/engine/search/filters/mem_filter.py
new file mode 100644
index 0000000..1b494c2
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/mem_filter.py
@@ -0,0 +1,56 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+class MemFilter(object):
+
+ def __init__(self):
+ self.name = "mem"
+
+ self.status = None
+
+ def init_condition(self):
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ return True
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, _v, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _v, _candidate):
+ """Only return hosts with sufficient available RAM."""
+
+ requested_ram = _v.mem # MB
+ usable_ram = _candidate.get_mem(_level)
+
+ # TODO: need to check against original mem_cap?
+ # Do not allow an instance to overcommit against itself,
+ # only against other instances.
+ # if not total_ram >= requested_ram:
+ # return False
+
+ if not usable_ram >= requested_ram:
+ return False
+
+ return True
diff --git a/engine/src/valet/engine/search/filters/no_exclusivity_filter.py b/engine/src/valet/engine/search/filters/no_exclusivity_filter.py
new file mode 100644
index 0000000..43516fe
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/no_exclusivity_filter.py
@@ -0,0 +1,53 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+class NoExclusivityFilter(object):
+
+ def __init__(self):
+ self.name = "no-exclusivity"
+
+ self.status = None
+
+ def init_condition(self):
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ exclusivities = _v.get_exclusivities(_level)
+
+ if len(exclusivities) == 0:
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _candidate):
+ memberships = _candidate.get_memberships(_level)
+
+ for _, g in memberships.iteritems():
+ if g.group_type == "exclusivity" and g.level == _level:
+ return False
+
+ return True
diff --git a/engine/src/valet/engine/search/filters/numa_filter.py b/engine/src/valet/engine/search/filters/numa_filter.py
new file mode 100644
index 0000000..3e095ec
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/numa_filter.py
@@ -0,0 +1,84 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+from valet.engine.app_manager.server import Server
+
+
+_SCOPE = 'hw'
+
+
+class NUMAFilter(object):
+ """Check NUMA alignment request in Flavor."""
+
+ def __init__(self):
+ """Define filter name and status."""
+
+ self.name = "numa"
+
+ self.status = None
+
+ def init_condition(self):
+ """Init variable."""
+
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ """Check if given server needs to check this filter."""
+
+ if _level == "host" and isinstance(_v, Server):
+ if _v.need_numa_alignment():
+ return True
+
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ """Check and filter one candidate at a time."""
+
+ candidate_list = []
+
+ for c in _candidate_list:
+ if self._check_candidate(_level, _v, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_candidate(self, _level, _v, _candidate):
+ """Check given candidate host if it meets numa requirement."""
+
+ # servers = []
+ # if isinstance(_v, Group):
+ # _v.get_servers(servers)
+ # else:
+ # servers.append(_v)
+
+ # (vcpus_demand, mem_demand) = self._get_demand_with_numa(servers)
+
+ return _candidate.NUMA.has_enough_resources(_v.vCPUs, _v.mem)
+
+ def _get_demand_with_numa(self, _servers):
+ """Check numa and compute the amount of vCPUs and memory."""
+
+ vcpus = 0
+ mem = 0
+
+ for s in _servers:
+ if s.need_numa_alignment():
+ vcpus += s.vCPUs
+ mem += s.mem
+
+ return vcpus, mem
diff --git a/engine/src/valet/engine/search/filters/quorum_diversity_filter.py b/engine/src/valet/engine/search/filters/quorum_diversity_filter.py
new file mode 100644
index 0000000..6388ebc
--- /dev/null
+++ b/engine/src/valet/engine/search/filters/quorum_diversity_filter.py
@@ -0,0 +1,106 @@
+#
+# -------------------------------------------------------------------------
+# Copyright (c) 2019 AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+import math
+
+
+class QuorumDiversityFilter(object):
+
+ def __init__(self):
+ self.name = "quorum-diversity"
+
+ self.quorum_diversity_group_list = []
+
+ self.status = None
+
+ def init_condition(self):
+ self.quorum_diversity_group_list = []
+ self.status = None
+
+ def check_pre_condition(self, _level, _v, _avail_hosts, _avail_groups):
+ if len(_v.quorum_diversity_groups) > 0:
+ for _, qdiv_group in _v.quorum_diversity_groups.iteritems():
+ if qdiv_group.level == _level:
+ self.quorum_diversity_group_list.append(qdiv_group)
+
+ if len(self.quorum_diversity_group_list) > 0:
+ return True
+ else:
+ return False
+
+ def filter_candidates(self, _level, _v, _candidate_list):
+ candidate_list = []
+
+ # First, try diversity rule.
+
+ for c in _candidate_list:
+ if self._check_diversity_candidate(_level, c):
+ candidate_list.append(c)
+
+ if len(candidate_list) > 0:
+ return candidate_list
+
+ # Second, if no available hosts for diversity rule, try quorum rule.
+
+ for c in _candidate_list:
+ if self._check_quorum_candidate(_level, c):
+ candidate_list.append(c)
+
+ return candidate_list
+
+ def _check_diversity_candidate(self, _level, _candidate):
+ """Filter based on named diversity groups."""
+
+ memberships = _candidate.get_memberships(_level)
+
+ for qdiv in self.quorum_diversity_group_list:
+ for gk, gr in memberships.iteritems():
+ if gr.group_type == "quorum-diversity" and gk == qdiv.vid:
+ return False
+
+ return True
+
+ def _check_quorum_candidate(self, _level, _candidate):
+ """Filter based on quorum-diversity rule."""
+
+ memberships = _candidate.get_memberships(_level)
+ hk = _candidate.get_resource_name(_level)
+
+ for qdiv in self.quorum_diversity_group_list:
+ # Requested num of servers under this rule
+ total_num_of_servers = len(qdiv.server_list)
+
+ num_of_placed_servers_in_candidate = -1
+
+ for gk, gr in memberships.iteritems():
+ if gr.group_type == "quorum-diversity" and gk == qdiv.vid:
+ # Total num of servers under this rule
+ total_num_of_servers += gr.original_num_of_placed_servers
+
+ if hk in gr.num_of_placed_servers_of_host.keys():
+ num_of_placed_servers_in_candidate = gr.num_of_placed_servers_of_host[hk]
+
+ break
+
+ # Allowed maximum num of servers per host
+ quorum = max(math.ceil(float(total_num_of_servers) / 2.0 - 1.0), 1.0)
+
+ if num_of_placed_servers_in_candidate >= quorum:
+ return False
+
+ return True