# First attempt at microservice definitions in tosca-yaml. # Ted Johnson, Dec. 2015 ## Ralph Knag, config additions , Feb. 2016 tosca_definitions_version: tosca_simple_yaml_1_0_0 metadata: template_version: 1.0.2 data_types: ######################################################## ### data types related to schemas baseFieldDefinition: description: base class for field definitions properties: name: required: true type: string type: required: true type: string constraints: - valid_values: [ string, int, float, ipv4, ipv6, date, date_time, list, map, other] format: required: false type: string default_value: required: false type: string comment: required: false type: string constraints: description: Properties that the field must satisfy required: false type: list entry_schema: type: string constraints: list_delimiter: description: delimiter when list data packed into a string type. required: false type: string # subtype can be used for policy override allowed? subType: required: false type : string constraints: - valid_values: [ policyOveride, noPolicyOverride ] positionalFieldDefinition: derived_from: baseFieldDefinition description: fields with a fixed-position schema properties: position: required: true type: integer constraints: - greater_or_equal: 0 pathFieldDefinition: derived_from: baseFieldDefinition description: fields accessed via navigation properties: path: required: true type: string fieldDefinition: derived_from: baseFieldDefinition description: positional or path field properties: path: required: false type: string position: required: false type: integer constraints: - greater_or_equal: 0 comment: required: false type: string length: required: false type: integer constraints: - greater_than: 0 fieldList: description: fields in a schema properties: fieldList: required: true type: list entry_schema: type: fieldDefinition ######################################################## ### data types related to data properties sortedBy: description: sorting properties of a data set properties: fields: required: true type: list entry_schema: type: fieldDefinition sorting: required: true type: string default: fullySorted constraints: - valid_values: [ fullySorted, partiallySorted ] lag: description: maximum disorder in a partially sorted required: false type: integer constraints: - greater_or_equal: 0 keyFields: description: key fields of a data set properties: fields: required: true type: list entry_schema: type: fieldDefinition timestamp: required: false type: integer ############################################################### ############################################################### node_types: dcaeCatalogNodes: derived_from: tosca.nodes.Root properties: UUID: required: true type: string name: required: true type: string version: required: true type: string constraints: - pattern: \d+[.]\d+[.]\d+ schemaVersion: required: false type: integer default: 1 ######################################################## ### node types related to schemas baseDataFormat: description: base data format derived_from: dcaeCatalogNodes properties: fieldList: required: true type: list entry_schema: type: fieldDefinition formatType: type: string required: true constraints: - valid_values: [ fixed_format, variable_format, json ] formatSubType: required: true type: string fullList: type: string required: true constraints: - valid_values: [ yes, no ] default: yes capabilities: dataProperties: type: dcaeCatalog.capabilities.dataFormat baseVariableDataFormat: derived_from: baseDataFormat description: base class for fixed-position data formats properties: formatType: type: string required: true constraints: - equal: variable_format # probably more needed here - not sure baseDataFormat is sufficient...rhk jsonDataFormat: derived_from: baseVariableDataFormat description: JSON format properties: formatSubType: required: true type: string default: json constraints: - equal: json endOfRecord: required: false type: string numFields: required: false type: integer baseFixedDataFormat: derived_from: baseDataFormat description: base class for fixed-position data formats properties: formatType: type: string required: true constraints: - equal: fixed_format delimitedAsciiDataFormat: derived_from: baseFixedDataFormat description: delimited ascii format properties: formatSubType: required: true type: string default: delimitedAscii constraints: - equal: delimitedAscii delimiter: required: true type: string default: | endOfRecord: required: false type: string numFields: required: false type: integer fixedWidthAsciiDataFormat: derived_from: baseFixedDataFormat description: delimited ascii format properties: formatSubType: required: true type: string default: fixedWidthAscii constraints: - equal: fixedWidthAscii endOfRecord: required: false type: string numFields: required: false type: integer fixedBinaryDataFormat: derived_from: baseFixedDataFormat description: binary data, e.g. thrift messages, protocol buffers properties: formatSubType: required: true type: string default: fixedBinaryData constraints: - equal: fixedBinaryData binaryFormat: required: true type: string ############################################### ### Node types related to data properties dataProperties: description: Essential non-schema information about a data set derived_from: dcaeCatalogNodes properties: sort: required: false type: sortedBy keys: required: false type: keyFields timestamp: description: the field indicating the time when corresponding event occurred required: false type: string duration: description: the field indicating the length of the described measurement required: false type: string period: description: how often the data is generated. 0 means continuously. required: true type: integer default: 0 constraints: - greater_or_equal: 0 capabilities: dataProperties: type: dcaeCatalog.capabilities.dataProperties keyedDataProperties: derived_from: dataProperties description: Data sets that are required to hav a key properties: keys: required: true type: keyFields referenceDataProperties: derived_from: dataProperties description: data set from external source that is joined against streaming data properties: updateType: description: how the reference data is refreshed - snapshots, incremental, or both required: true type: string keyedReferenceDataProperties: derived_from: referenceDataProperties description: reference data that maps a key to a value, generally with a valid time interval properties: keys: required: true type: keyFields startTimeField: description: the time when the key --> value mapping became valid required: false type: string endTimeField: description: time when the key --> value mapping ceased to be valid required: false type: string ####################################################################### #### Things which describe how a data source can be accessed dataDelivery: description: the means of connecting to a data source. very incomplete. derived_from: dcaeCatalogNodes properties: deliveryMechanism: description: the general method by which data is delivered from a source. required: true type: string constraints: - valid_values: [stream, file, database, table ] polling: description: push vs. pull required: true type: string constraints: - valid_values: [ push, pull] capabilities: dataDelivery: type: dcaeCatalog.capabilities.dataDelivery streamDataDelivery: description: a data source that is a streaming source. derived_from: dataDelivery properties: deliveryMechanism: description: the general method by which data is delivered from a source. required: true type: string constraints: - equal: stream polling: description: push vs. pull required: true type: string constraints: - equal: push genericCDAPDelivery: description: a streaming data source delivered via the CDAP generic derived_from: streamDataDelivery # A UEB stream is kafka plus security TBD dmaapMessageDelivery: description: a streaming data source delivered via a kafka queue derived_from: streamDataDelivery properties: servers: description: the servers to contact to access the kafka queue. required: true type: list entry_schema: type: string constraints: topic: description: the stream identifier (topic in kafka terminology) required: true type: string numPartitions: description: the number of partitions for the kafka topic required: false default: 1 type: integer PartitioningMethod: description: the mechanism that assigns records to partitions required: false type: string partitionSubscriptions: description: if specified, the set of partitions that contain a particular substream required: false type: list entry_schema: type: integer dmaapDataRouterDelivery: description: a data source that is delivered via files. derived_from: dataDelivery properties: deliveryMechanism: description: the general method by which data is delivered from a source. required: true type: string constraints: - valid_values: [file] fileName: description: a name (or naming scheme) of the file required: true type: string fileDataDelivery: description: a data source that is delivered via files. derived_from: dataDelivery properties: deliveryMechanism: description: the general method by which data is delivered from a source. required: true type: string constraints: - valid_values: [file] fileName: description: a name (or naming scheme) of the file required: true type: string pulledFileDelivery: description: a data source that is delivered via files pulled from an external source derived_from: fileDataDelivery properties: uri: description: the thing to access to get a new file required: true type: string trigger: description: a description of the thing that triggers pulling a new file. required: true type: string #################################### ### Description of a data source (data output) dataSource: description: a generic data source description, link to schema, properties, delivery derived_from: dcaeCatalogNodes # properties: requirements: - dataFormat: capability: dcaeCatalog.capabilities.dataFormat - dataDelivery: capability: dcaeCatalog.capabilities.dataDelivery - dataProperties: capability: dcaeCatalog.capabilities.dataProperties capabilities: dataSource: type: dcaeCatalog.capabilities.dataSource occurrences: [0, UNBOUNDED] ########################## ### Description of a data requirement (data input) dataRequirement: description: a generic data source description, link to schema, properties, delivery derived_from: dcaeCatalogNodes # properties: requirements: - dataFormat: capability: dcaeCatalog.capabilities.dataFormat occurrences: [1, UNBOUNDED] - dataProperties: capability: dcaeCatalog.capabilities.dataProperties - dataDelivery: capability: dcaeCatalog.capabilities.dataDelivery capabilities: dataRequirement: type: dcaeCatalog.capabilities.dataRequirement occurrences: [0, UNBOUNDED] #################################### ### Description of a processingModuleParameters (module parameters excluding input/output descriptions ### allows any baseDataFormat derived input processingModuleParameters: description: derived_from: baseDataFormat properties: dataType: required: true type: string capabilities: moduleParameters: type: dcaeCatalog.capabilities.processingModuleParameters ################################ ### Module Dependencies instantiationDependency: description: code dependencies (possible transitive) derived_from: dcaeCatalogNodes properties: dependencyType: required: false type: string constraints: - valid_values: [R, Python, CDAP] dependencyValue: required: true type: string capabilities: dataProperties: type: dcaeCatalog.capabilities.instantiationDependency occurrences: [0, UNBOUNDED] requirements: - dependencies: capability: dcaeCatalog.capabilities.instantiationDependency occurrences: [0, UNBOUNDED] #################################### ### Description of a processingModuleDeployment - deployment attributes processingModuleDeployment: description: deployment attributes derived_from: dcaeCatalogNodes properties: language: required: true type: string version: type: string required: true deploymentType: required: true type: string capabilities: moduleDeployment: type: dcaeCatalog.capabilities.processingModuleDeployment occurrences: [0, UNBOUNDED] requirements: - dependencies: capability: dcaeCatalog.capabilities.instantiationDependency occurrences: [0, UNBOUNDED] #################################### ### Description of a processingModuleMetaData - search tags processingModuleMetadata: description: metadata tags derived_from: dcaeCatalogNodes properties: metaDataList: required: true type: list entry_schema: type: string constraints: capabilities: metadata: type: dcaeCatalog.capabilities.processingModuleMetadata occurrences: [0, UNBOUNDED] #################################### ### Description of a processingModule processingModule: description: full processing module description derived_from: dcaeCatalogNodes properties: moduleDescription: required: true type: string requirements: - processingModuleparameters: capability: dcaeCatalog.capabilities.processingModuleParameters occurrences: [0, UNBOUNDED] - processingModuleDeployment: capability: dcaeCatalog.capabilities.processingModuleDeployment occurrences: [1, UNBOUNDED] - processingModuleMetadata: capability: dcaeCatalog.capabilities.processingModuleMetadata occurrences: [1, UNBOUNDED] - dataRequirementsList: capability: dcaeCatalog.capabilities.dataRequirement occurrences: [1, UNBOUNDED] - dataSourcesList: capability: dcaeCatalog.capabilities.dataSource occurrences: [1, UNBOUNDED] capability_types: dcaeCatalog.capabilities.dataFormat: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.dataProperties: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.dataDelivery: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.instantiationDependency: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.processingModuleMetadata: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.processingModuleParameters: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.processingModuleDeployment: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.dataRequirement: derived_from: tosca.capabilities.Root dcaeCatalog.capabilities.dataSource: derived_from: tosca.capabilities.Root