# Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Creates ARIA service template models based on the TOSCA presentation. Relies on many helper methods in the presentation classes. """ #pylint: disable=unsubscriptable-object import os import re from types import FunctionType from datetime import datetime from ruamel import yaml from aria.parser.validation import Issue from aria.utils.formatting import string_list_as_string from aria.utils.collections import (StrictDict, OrderedDict) from aria.orchestrator import WORKFLOW_DECORATOR_RESERVED_ARGUMENTS from aria.modeling.models import (Type, ServiceTemplate, NodeTemplate, RequirementTemplate, RelationshipTemplate, CapabilityTemplate, GroupTemplate, PolicyTemplate, SubstitutionTemplate, SubstitutionTemplateMapping, InterfaceTemplate, OperationTemplate, ArtifactTemplate, Metadata, Input, Output, Property, Attribute, Configuration, PluginSpecification) from .parameters import coerce_parameter_value from .constraints import (Equal, GreaterThan, GreaterOrEqual, LessThan, LessOrEqual, InRange, ValidValues, Length, MinLength, MaxLength, Pattern) from ..data_types import coerce_value # These match the first un-escaped ">" # See: http://stackoverflow.com/a/11819111/849021 IMPLEMENTATION_PREFIX_REGEX = re.compile(r'(?') def create_service_template_model(context): # pylint: disable=too-many-locals,too-many-branches model = ServiceTemplate(created_at=datetime.now(), main_file_name=os.path.basename(str(context.presentation.location))) model.description = context.presentation.get('service_template', 'description', 'value') # Metadata metadata = context.presentation.get('service_template', 'metadata') if metadata is not None: create_metadata_models(context, model, metadata) # Types model.node_types = Type(variant='node') create_types(context, model.node_types, context.presentation.get('service_template', 'node_types')) model.group_types = Type(variant='group') create_types(context, model.group_types, context.presentation.get('service_template', 'group_types')) model.policy_types = Type(variant='policy') create_types(context, model.policy_types, context.presentation.get('service_template', 'policy_types')) model.relationship_types = Type(variant='relationship') create_types(context, model.relationship_types, context.presentation.get('service_template', 'relationship_types')) model.capability_types = Type(variant='capability') create_types(context, model.capability_types, context.presentation.get('service_template', 'capability_types')) model.interface_types = Type(variant='interface') create_types(context, model.interface_types, context.presentation.get('service_template', 'interface_types')) model.artifact_types = Type(variant='artifact') create_types(context, model.artifact_types, context.presentation.get('service_template', 'artifact_types')) # Topology template topology_template = context.presentation.get('service_template', 'topology_template') if topology_template is not None: model.inputs.update( create_input_models_from_values(topology_template._get_input_values(context))) model.outputs.update( create_output_models_from_values(topology_template._get_output_values(context))) # Plugin specifications policies = context.presentation.get('service_template', 'topology_template', 'policies') if policies: for policy in policies.itervalues(): role = model.policy_types.get_descendant(policy.type).role if role == 'plugin': plugin_specification = create_plugin_specification_model(context, policy) model.plugin_specifications[plugin_specification.name] = plugin_specification elif role == 'workflow': operation_template = create_workflow_operation_template_model(context, model, policy) model.workflow_templates[operation_template.name] = operation_template # Node templates node_templates = context.presentation.get('service_template', 'topology_template', 'node_templates') if node_templates: for node_template in node_templates.itervalues(): node_template_model = create_node_template_model(context, model, node_template) model.node_templates[node_template_model.name] = node_template_model for node_template in node_templates.itervalues(): fix_node_template_model(context, model, node_template) # Group templates groups = context.presentation.get('service_template', 'topology_template', 'groups') if groups: for group in groups.itervalues(): group_template_model = create_group_template_model(context, model, group) model.group_templates[group_template_model.name] = group_template_model # Policy templates policies = context.presentation.get('service_template', 'topology_template', 'policies') if policies: for policy in policies.itervalues(): policy_template_model = create_policy_template_model(context, model, policy) model.policy_templates[policy_template_model.name] = policy_template_model # Substitution template substitution_mappings = context.presentation.get('service_template', 'topology_template', 'substitution_mappings') if substitution_mappings: model.substitution_template = create_substitution_template_model(context, model, substitution_mappings) return model def create_metadata_models(context, service_template, metadata): service_template.meta_data['template_name'] = Metadata(name='template_name', value=metadata.template_name) service_template.meta_data['template_author'] = Metadata(name='template_author', value=metadata.template_author) service_template.meta_data['template_version'] = Metadata(name='template_version', value=metadata.template_version) custom = metadata.custom if custom: for name, value in custom.iteritems(): service_template.meta_data[name] = Metadata(name=name, value=value) def create_node_template_model(context, service_template, node_template): node_type = node_template._get_type(context) node_type = service_template.node_types.get_descendant(node_type._name) model = NodeTemplate(name=node_template._name, type=node_type) if node_template.description: model.description = node_template.description.value if node_template.directives: model.directives = node_template.directives model.properties.update(create_property_models_from_values( template_properties=node_template._get_property_values(context))) model.attributes.update(create_attribute_models_from_values( template_attributes=node_template._get_attribute_default_values(context))) create_interface_template_models(context, service_template, model.interface_templates, node_template._get_interfaces(context)) artifacts = node_template._get_artifacts(context) if artifacts: for artifact_name, artifact in artifacts.iteritems(): model.artifact_templates[artifact_name] = \ create_artifact_template_model(context, service_template, artifact) capabilities = node_template._get_capabilities(context) if capabilities: for capability_name, capability in capabilities.iteritems(): model.capability_templates[capability_name] = \ create_capability_template_model(context, service_template, capability) if node_template.node_filter: model.target_node_template_constraints = [] create_node_filter_constraints(context, node_template.node_filter, model.target_node_template_constraints) return model def fix_node_template_model(context, service_template, node_template): # Requirements have to be created after all node templates have been created, because # requirements might reference another node template model = service_template.node_templates[node_template._name] requirements = node_template._get_requirements(context) if requirements: for _, requirement in requirements: model.requirement_templates.append(create_requirement_template_model(context, service_template, requirement)) def create_group_template_model(context, service_template, group): group_type = group._get_type(context) group_type = service_template.group_types.get_descendant(group_type._name) model = GroupTemplate(name=group._name, type=group_type) if group.description: model.description = group.description.value model.properties.update(create_property_models_from_values(group._get_property_values(context))) create_interface_template_models(context, service_template, model.interface_templates, group._get_interfaces(context)) members = group.members if members: for member in members: node_template = service_template.node_templates[member] assert node_template model.node_templates.append(node_template) return model def create_policy_template_model(context, service_template, policy): policy_type = policy._get_type(context) policy_type = service_template.policy_types.get_descendant(policy_type._name) model = PolicyTemplate(name=policy._name, type=policy_type) if policy.description: model.description = policy.description.value model.properties.update( create_property_models_from_values(policy._get_property_values(context))) node_templates, groups = policy._get_targets(context) if node_templates: for target in node_templates: node_template = service_template.node_templates[target._name] assert node_template model.node_templates.append(node_template) if groups: for target in groups: group_template = service_template.group_templates[target._name] assert group_template model.group_templates.append(group_template) return model def create_requirement_template_model(context, service_template, requirement): model = {'name': requirement._name} node, node_variant = requirement._get_node(context) if node is not None: if node_variant == 'node_type': node_type = service_template.node_types.get_descendant(node._name) model['target_node_type'] = node_type else: node_template = service_template.node_templates[node._name] model['target_node_template'] = node_template capability, capability_variant = requirement._get_capability(context) if capability is not None: if capability_variant == 'capability_type': capability_type = \ service_template.capability_types.get_descendant(capability._name) model['target_capability_type'] = capability_type else: model['target_capability_name'] = capability._name model = RequirementTemplate(**model) if requirement.node_filter: model.target_node_template_constraints = [] create_node_filter_constraints(context, requirement.node_filter, model.target_node_template_constraints) relationship = requirement.relationship if relationship is not None: model.relationship_template = \ create_relationship_template_model(context, service_template, relationship) model.relationship_template.name = requirement._name return model def create_relationship_template_model(context, service_template, relationship): relationship_type, relationship_type_variant = relationship._get_type(context) if relationship_type_variant == 'relationship_type': relationship_type = service_template.relationship_types.get_descendant( relationship_type._name) model = RelationshipTemplate(type=relationship_type) else: relationship_template = relationship_type relationship_type = relationship_template._get_type(context) relationship_type = service_template.relationship_types.get_descendant( relationship_type._name) model = RelationshipTemplate(type=relationship_type) if relationship_template.description: model.description = relationship_template.description.value create_parameter_models_from_assignments(model.properties, relationship.properties, model_cls=Property) create_interface_template_models(context, service_template, model.interface_templates, relationship.interfaces) return model def create_capability_template_model(context, service_template, capability): capability_type = capability._get_type(context) capability_type = service_template.capability_types.get_descendant(capability_type._name) model = CapabilityTemplate(name=capability._name, type=capability_type) capability_definition = capability._get_definition(context) if capability_definition.description: model.description = capability_definition.description.value occurrences = capability_definition.occurrences if occurrences is not None: model.min_occurrences = occurrences.value[0] if occurrences.value[1] != 'UNBOUNDED': model.max_occurrences = occurrences.value[1] valid_source_types = capability_definition.valid_source_types if valid_source_types: for valid_source_type in valid_source_types: # TODO: handle shortcut type names node_type = service_template.node_types.get_descendant(valid_source_type) model.valid_source_node_types.append(node_type) create_parameter_models_from_assignments(model.properties, capability.properties, model_cls=Property) return model def create_interface_template_model(context, service_template, interface): interface_type = interface._get_type(context) interface_type = service_template.interface_types.get_descendant(interface_type._name) model = InterfaceTemplate(name=interface._name, type=interface_type) if interface_type.description: model.description = interface_type.description create_parameter_models_from_assignments(model.inputs, interface.inputs, model_cls=Input) operations = interface.operations if operations: for operation_name, operation in operations.iteritems(): model.operation_templates[operation_name] = \ create_operation_template_model(context, service_template, operation) return model if model.operation_templates else None def create_operation_template_model(context, service_template, operation): model = OperationTemplate(name=operation._name) if operation.description: model.description = operation.description.value implementation = operation.implementation if implementation is not None: primary = implementation.primary extract_implementation_primary(context, service_template, operation, model, primary) relationship_edge = operation._get_extensions(context).get('relationship_edge') if relationship_edge is not None: if relationship_edge == 'source': model.relationship_edge = False elif relationship_edge == 'target': model.relationship_edge = True dependencies = implementation.dependencies configuration = OrderedDict() if dependencies: for dependency in dependencies: key, value = split_prefix(dependency) if key is not None: # Special ARIA prefix: signifies configuration parameters # Parse as YAML try: value = yaml.load(value) except yaml.parser.MarkedYAMLError as e: context.validation.report( 'YAML parser {0} in operation configuration: {1}' .format(e.problem, value), locator=implementation._locator, level=Issue.FIELD) continue # Coerce to intrinsic functions, if there are any value = coerce_parameter_value(context, implementation, None, value).value # Support dot-notation nesting set_nested(configuration, key.split('.'), value) else: if model.dependencies is None: model.dependencies = [] model.dependencies.append(dependency) # Convert configuration to Configuration models for key, value in configuration.iteritems(): model.configurations[key] = Configuration.wrap(key, value, description='Operation configuration.') create_parameter_models_from_assignments(model.inputs, operation.inputs, model_cls=Input) return model def create_artifact_template_model(context, service_template, artifact): artifact_type = artifact._get_type(context) artifact_type = service_template.artifact_types.get_descendant(artifact_type._name) model = ArtifactTemplate(name=artifact._name, type=artifact_type, source_path=artifact.file) if artifact.description: model.description = artifact.description.value model.target_path = artifact.deploy_path repository = artifact._get_repository(context) if repository is not None: model.repository_url = repository.url credential = repository._get_credential(context) if credential: model.repository_credential = {} for k, v in credential.iteritems(): model.repository_credential[k] = v model.properties.update( create_property_models_from_values(artifact._get_property_values(context))) return model def create_substitution_template_model(context, service_template, substitution_mappings): node_type = service_template.node_types.get_descendant(substitution_mappings.node_type) model = SubstitutionTemplate(node_type=node_type) capabilities = substitution_mappings.capabilities if capabilities: for mapped_capability_name, capability in capabilities.iteritems(): name = 'capability.' + mapped_capability_name node_template_model = service_template.node_templates[capability.node_template] capability_template_model = \ node_template_model.capability_templates[capability.capability] model.mappings[name] = \ SubstitutionTemplateMapping(name=name, capability_template=capability_template_model) requirements = substitution_mappings.requirements if requirements: for mapped_requirement_name, requirement in requirements.iteritems(): name = 'requirement.' + mapped_requirement_name node_template_model = service_template.node_templates[requirement.node_template] requirement_template_model = None for a_model in node_template_model.requirement_templates: if a_model.name == requirement.requirement: requirement_template_model = a_model break model.mappings[name] = \ SubstitutionTemplateMapping(name=name, requirement_template=requirement_template_model) return model def create_plugin_specification_model(context, policy): properties = policy.properties def get(name, default=None): prop = properties.get(name) return prop.value if prop is not None else default model = PluginSpecification(name=policy._name, version=get('version'), enabled=get('enabled', True)) return model def create_workflow_operation_template_model(context, service_template, policy): model = OperationTemplate(name=policy._name) # since we use backpopulates, these fields are populated upon commit, we get a weird(temporary) # behavior where in previous code service_template.workflow_templates is a dict which has None # as key for the value of model. service_template.workflow_templates[model.name] = model if policy.description: model.description = policy.description.value properties = policy._get_property_values(context) for prop_name, prop in properties.iteritems(): if prop_name == 'implementation': model.function = prop.value else: input_model = create_parameter_model_from_value(prop, prop_name, model_cls=Input) input_model.required = prop.required model.inputs[prop_name] = input_model used_reserved_names = WORKFLOW_DECORATOR_RESERVED_ARGUMENTS.intersection(model.inputs.keys()) if used_reserved_names: context.validation.report('using reserved arguments in workflow policy "{0}": {1}' .format( policy._name, string_list_as_string(used_reserved_names)), locator=policy._locator, level=Issue.EXTERNAL) return model # # Utils # def create_types(context, root, types): if types is None: return def added_all(): for name in types: if root.get_descendant(name) is None: return False return True while not added_all(): for name, the_type in types.iteritems(): if root.get_descendant(name) is None: parent_type = the_type._get_parent(context) model = Type(name=the_type._name, role=the_type._get_extension('role')) if the_type.description: model.description = the_type.description.value if parent_type is None: model.parent = root model.variant = root.variant root.children.append(model) else: container = root.get_descendant(parent_type._name) if container is not None: model.parent = container model.variant = container.variant container.children.append(model) def create_input_models_from_values(template_inputs): model_inputs = {} if template_inputs: for template_input_name, template_input in template_inputs.iteritems(): model_input = create_parameter_model_from_value(template_input, template_input_name, model_cls=Input) model_input.required = template_input.required model_inputs[model_input.name] = model_input return model_inputs def create_output_models_from_values(template_outputs): model_outputs = {} for template_output_name, template_output in template_outputs.iteritems(): model_outputs[template_output_name] = \ create_parameter_model_from_value(template_output, template_output_name, model_cls=Output) return model_outputs def create_property_models_from_values(template_properties): model_properties = {} for template_property_name, template_property in template_properties.iteritems(): model_properties[template_property_name] = \ create_parameter_model_from_value(template_property, template_property_name, model_cls=Property) return model_properties def create_attribute_models_from_values(template_attributes): model_attributes = {} for template_attribute_name, template_attribute in template_attributes.iteritems(): model_attributes[template_attribute_name] = \ create_parameter_model_from_value(template_attribute, template_attribute_name, model_cls=Attribute) return model_attributes def create_parameter_model_from_value(template_parameter, template_parameter_name, model_cls): return model_cls(name=template_parameter_name, type_name=template_parameter.type, value=template_parameter.value, description=template_parameter.description) def create_parameter_models_from_assignments(properties, source_properties, model_cls): if source_properties: for property_name, prop in source_properties.iteritems(): properties[property_name] = model_cls(name=property_name, # pylint: disable=unexpected-keyword-arg type_name=prop.value.type, value=prop.value.value, description=prop.value.description) def create_interface_template_models(context, service_template, interfaces, source_interfaces): if source_interfaces: for interface_name, interface in source_interfaces.iteritems(): interface = create_interface_template_model(context, service_template, interface) if interface is not None: interfaces[interface_name] = interface def create_node_filter_constraints(context, node_filter, target_node_template_constraints): properties = node_filter.properties if properties is not None: for property_name, constraint_clause in properties: constraint = create_constraint(context, node_filter, constraint_clause, property_name, None) target_node_template_constraints.append(constraint) capabilities = node_filter.capabilities if capabilities is not None: for capability_name, capability in capabilities: properties = capability.properties if properties is not None: for property_name, constraint_clause in properties: constraint = create_constraint(context, node_filter, constraint_clause, property_name, capability_name) target_node_template_constraints.append(constraint) def create_constraint(context, node_filter, constraint_clause, property_name, capability_name): # pylint: disable=too-many-return-statements constraint_key = constraint_clause._raw.keys()[0] the_type = constraint_clause._get_type(context) def coerce_constraint(constraint): if the_type is not None: return coerce_value(context, node_filter, the_type, None, None, constraint, constraint_key) else: return constraint def coerce_constraints(constraints): if the_type is not None: return tuple(coerce_constraint(constraint) for constraint in constraints) else: return constraints if constraint_key == 'equal': return Equal(property_name, capability_name, coerce_constraint(constraint_clause.equal)) elif constraint_key == 'greater_than': return GreaterThan(property_name, capability_name, coerce_constraint(constraint_clause.greater_than)) elif constraint_key == 'greater_or_equal': return GreaterOrEqual(property_name, capability_name, coerce_constraint(constraint_clause.greater_or_equal)) elif constraint_key == 'less_than': return LessThan(property_name, capability_name, coerce_constraint(constraint_clause.less_than)) elif constraint_key == 'less_or_equal': return LessOrEqual(property_name, capability_name, coerce_constraint(constraint_clause.less_or_equal)) elif constraint_key == 'in_range': return InRange(property_name, capability_name, coerce_constraints(constraint_clause.in_range)) elif constraint_key == 'valid_values': return ValidValues(property_name, capability_name, coerce_constraints(constraint_clause.valid_values)) elif constraint_key == 'length': return Length(property_name, capability_name, coerce_constraint(constraint_clause.length)) elif constraint_key == 'min_length': return MinLength(property_name, capability_name, coerce_constraint(constraint_clause.min_length)) elif constraint_key == 'max_length': return MaxLength(property_name, capability_name, coerce_constraint(constraint_clause.max_length)) elif constraint_key == 'pattern': return Pattern(property_name, capability_name, coerce_constraint(constraint_clause.pattern)) else: raise ValueError('malformed node_filter: {0}'.format(constraint_key)) def split_prefix(string): """ Splits the prefix on the first non-escaped ">". """ split = IMPLEMENTATION_PREFIX_REGEX.split(string, 1) if len(split) < 2: return None, None return split[0].strip(), split[1].strip() def set_nested(the_dict, keys, value): """ If the ``keys`` list has just one item, puts the value in the the dict. If there are more items, puts the value in a sub-dict, creating sub-dicts as necessary for each key. For example, if ``the_dict`` is an empty dict, keys is ``['first', 'second', 'third']`` and value is ``'value'``, then the_dict will be: ``{'first':{'second':{'third':'value'}}}``. :param the_dict: Dict to change :type the_dict: {} :param keys: Keys :type keys: [basestring] :param value: Value """ key = keys.pop(0) if len(keys) == 0: the_dict[key] = value else: if key not in the_dict: the_dict[key] = StrictDict(key_class=basestring) set_nested(the_dict[key], keys, value) def extract_implementation_primary(context, service_template, presentation, model, primary): prefix, postfix = split_prefix(primary) if prefix: # Special ARIA prefix model.plugin_specification = service_template.plugin_specifications.get(prefix) model.function = postfix if model.plugin_specification is None: context.validation.report( 'no policy for plugin "{0}" specified in operation implementation: {1}' .format(prefix, primary), locator=presentation._get_child_locator('properties', 'implementation'), level=Issue.BETWEEN_TYPES) else: # Standard TOSCA artifact with default plugin model.implementation = primary