diff options
56 files changed, 4257 insertions, 3576 deletions
diff --git a/docs/sections/components/architecture/service-discovery.rst b/docs/sections/components/architecture/service-discovery.rst index 1eeaef88..315f4520 100755 --- a/docs/sections/components/architecture/service-discovery.rst +++ b/docs/sections/components/architecture/service-discovery.rst @@ -11,7 +11,7 @@ discovery <http://microservices.io/patterns/server-side-discovery.html>`__ and is using `Consul <https://www.consul.io/>`__ as the service registry
solution.
-Service registration
+Service Registration
--------------------
All components are required to register with Consul in order to be
diff --git a/docs/sections/components/architecture/services.rst b/docs/sections/components/architecture/services.rst deleted file mode 100755 index 9cfd084c..00000000 --- a/docs/sections/components/architecture/services.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License.
-.. http://creativecommons.org/licenses/by/4.0
-
-‘Services’ Overview
-===================
-
-DCAE Services run on the DCAE platform. A service performs either
-monitors the virtualized network services or does analytics. A service
-is composed of one or more components, and performs a business need.
-
-Services are created in a ‘Service Design and Creation’ tool called
-‘Service Assurance Flow Designer’ by a Service Designer. This is done by
-\* 1. ‘compose’ing a service from the available SDC catalog components
-(actually from the TOSCA models representing the components), then \* 2.
-’submit’ing the service, which generates a Cloudify blueprint, is then
-automatically uploaded to INVENTORY, and then deployed by DEPLOYMENT
-HANDLER (once a DTI Event is triggered for the service). It should be
-noted that some service flows, specifally ’closed-loop’ flows, are not
-initiated by DTI, but are done by CLAMP.
diff --git a/docs/sections/components/component-json-schema.rst b/docs/sections/components/component-json-schema.rst index 9cd5e4f9..9ad54565 100644 --- a/docs/sections/components/component-json-schema.rst +++ b/docs/sections/components/component-json-schema.rst @@ -9,737 +9,745 @@ DCAE Component JSON Schema :: { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Component specification schema", - "type": "object", - "properties": { - "self": { - "type": "object", - "properties": { - "version": { - "$ref": "#/definitions/version" - }, - "description": { - "type": "string" - }, - "component_type": { - "type": "string", - "enum": [ - "docker", - "cdap" - ] - }, - "name": { - "$ref": "#/definitions/name" - } - }, - "required": [ - "version", - "name", - "description", - "component_type" - ] - }, - "streams": { - "type": "object", - "properties": { - "publishes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/publisher_http" }, - { "$ref": "#/definitions/publisher_message_router" }, - { "$ref": "#/definitions/publisher_data_router" } - ] - } - }, - "subscribes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/subscriber_http" }, - { "$ref": "#/definitions/subscriber_message_router" }, - { "$ref": "#/definitions/subscriber_data_router" } - ] - } - } - }, - "required": [ - "publishes", - "subscribes" - ] - }, - "services": { - "type": "object", - "properties": { - "calls": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/caller" - } - }, - "provides": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/provider" - } - } - }, - "required": [ - "calls", - "provides" - ] - }, - "parameters" : { - "anyOf" : [ - {"$ref": "#/definitions/docker-parameters"}, - {"$ref": "#/definitions/cdap-parameters"} - ] - }, - "auxilary": { - "oneOf" : [ - {"$ref": "#/definitions/auxilary_cdap"}, - {"$ref": "#/definitions/auxilary_docker"} - ] - }, - "artifacts": { - "type": "array", - "description": "List of component artifacts", - "items": { - "$ref": "#/definitions/artifact" - } - } - }, - "required": [ - "self", - "streams", - "services", - "parameters", - "auxilary", - "artifacts" - ], - "additionalProperties": false, - "definitions": { - "cdap-parameters": { - "description" : "There are three seperate ways to pass parameters to CDAP: app config, app preferences, program preferences. These are all treated as optional.", - "type": "object", - "properties" : { - "program_preferences": { - "description" : "A list of {program_id, program_type, program_preference} objects where program_preference is an object passed into program_id of type program_type", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/program_preference" - } - }, - "app_preferences" : { - "description" : "Parameters Passed down to the CDAP preference API", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "app_config" : { - "description" : "Parameters Passed down to the CDAP App Config", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - } - }, - "program_preference": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/program_type" - }, - "program_id": { - "type": "string" - }, - "program_pref":{ - "description" : "Parameters that the CDAP developer wants pushed to this program's preferences API. Optional", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - }, - "required": ["program_type", "program_id", "program_pref"] - }, - "program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "docker-parameters": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "parameter": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "used for complex data type in the future. 'type' must be map or array for entry_schema to kick_in. ", - "type": "string" - }, - "designer_editable": { - "description": "An optional key that declares a parameter to be editable by designer (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "policy_editable": { - "description": "An optional key that declares a parameter to be editable by policy (true) or not (false). Default is true.", - "type": "boolean", - "default": false - }, - "sourced_at_deployment": { - "description": "An optional key that declares a parameter's value to be assigned at deployment time (true). Default is false.", - "type": "boolean", - "default": false - }, - "policy_schema" :{ - "type": "array", - "uniqueItems": true, - "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "value", - "description" - ], - "additionalProperties": false, - "dependencies": { "policy_schema": ["policy_editable"]} - }, - "policy_schema_parameter": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime", "list", "map" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "The optional key that is used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is either list or map. If the type is list and the entry type is a simple type (string, number, boolean, datetime), follow with a simple string to describe the entry type. If the type is list and the entry type is a map, follow with an array to describe the keys for the entry map. If the type is list and the entry type is also list, this is not currently supported here. If the type is map, then follow with an array to describe the keys for this map. ", - "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/policy_schema_parameter"} - } + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Component specification schema", + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "version": { + "$ref": "#/definitions/version" }, - "required": [ - "name", - "type" - ], - "additionalProperties": false - }, - "parameter-constraints": { - "type": "object", - "additionalProperties": false, - "properties": { - "equal": { - "description": "Constrains a property or parameter to a value equal to (‘=’) the value declared." - }, - "greater_than": { - "description": "Constrains a property or parameter to a value greater than (‘>’) the value declared.", - "type": "number" - }, - "greater_or_equal": { - "description": "Constrains a property or parameter to a value greater than or equal to (‘>=’) the value declared.", - "type": "number" - }, - "less_than": { - "description": "Constrains a property or parameter to a value less than (‘<’) the value declared.", - "type": "number" - }, - "less_or_equal": { - "description": "Constrains a property or parameter to a value less than or equal to (‘<=’) the value declared.", - "type": "number" - }, - "valid_values": { - "description": "Constrains a property or parameter to a value that is in the list of declared values.", - "type": "array" - }, - "length": { - "description": "Constrains the property or parameter to a value of a given length.", - "type": "number" - }, - "min_length": { - "description": "Constrains the property or parameter to a value to a minimum length.", - "type": "number" - }, - "max_length": { - "description": "Constrains the property or parameter to a value to a maximum length.", - "type": "number" - } - } - }, - "stream_message_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "message router", "message_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "publisher_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "subscriber_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "route", - "type" - ] - }, - "subscriber_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "subscriber_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - }, - "config_key": { - "description": "Data router subscribers require config info to setup their endpoints to handle requests. For example, needs username and password", - "type": "string" - } - }, - "required": [ - "format", - "version", - "route", - "type", - "config_key" - ] - }, - "provider" : { - "oneOf" : [ - {"$ref": "#/definitions/docker-provider"}, - {"$ref": "#/definitions/cdap-provider"} - ] - }, - "cdap-provider" : { - "type": "object", - "properties" : { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "service_name" : { - "type" : "string" - }, - "service_endpoint" : { - "type" : "string" - }, - "verb" : { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } - }, - "required" : [ - "request", - "response", - "service_name", - "service_endpoint", - "verb" - ] - }, - "docker-provider": { - "type": "object", - "properties": { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "route": { - "type": "string" - }, - "verb": { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } + "description": { + "type": "string" + }, + "component_type": { + "type": "string", + "enum": [ + "docker", + "cdap" + ] + }, + "name": { + "$ref": "#/definitions/name" + } + }, + "required": [ + "version", + "name", + "description", + "component_type" + ] + }, + "streams": { + "type": "object", + "properties": { + "publishes": { + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + { "$ref": "#/definitions/publisher_http" }, + { "$ref": "#/definitions/publisher_message_router" }, + { "$ref": "#/definitions/publisher_data_router" } + ] + } + }, + "subscribes": { + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + { "$ref": "#/definitions/subscriber_http" }, + { "$ref": "#/definitions/subscriber_message_router" }, + { "$ref": "#/definitions/subscriber_data_router" } + ] + } + } + }, + "required": [ + "publishes", + "subscribes" + ] + }, + "services": { + "type": "object", + "properties": { + "calls": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/caller" + } + }, + "provides": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/provider" + } + } + }, + "required": [ + "calls", + "provides" + ] + }, + "parameters" : { + "anyOf" : [ + {"$ref": "#/definitions/docker-parameters"}, + {"$ref": "#/definitions/cdap-parameters"} + ] + }, + "auxilary": { + "oneOf" : [ + {"$ref": "#/definitions/auxilary_cdap"}, + {"$ref": "#/definitions/auxilary_docker"} + ] + }, + "artifacts": { + "type": "array", + "description": "List of component artifacts", + "items": { + "$ref": "#/definitions/artifact" + } + } + }, + "required": [ + "self", + "streams", + "services", + "parameters", + "auxilary", + "artifacts" + ], + "additionalProperties": false, + "definitions": { + "cdap-parameters": { + "description" : "There are three seperate ways to pass parameters to CDAP: app config, app preferences, program preferences. These are all treated as optional.", + "type": "object", + "properties" : { + "program_preferences": { + "description" : "A list of {program_id, program_type, program_preference} objects where program_preference is an object passed into program_id of type program_type", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/program_preference" + } + }, + "app_preferences" : { + "description" : "Parameters Passed down to the CDAP preference API", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + }, + "app_config" : { + "description" : "Parameters Passed down to the CDAP App Config", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + } + } + }, + "program_preference": { + "type": "object", + "properties": { + "program_type": { + "$ref": "#/definitions/program_type" + }, + "program_id": { + "type": "string" + }, + "program_pref":{ + "description" : "Parameters that the CDAP developer wants pushed to this program's preferences API. Optional", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + } + }, + "required": ["program_type", "program_id", "program_pref"] + }, + "program_type": { + "type": "string", + "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] + }, + "docker-parameters": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + }, + "parameter": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "The required data type for the parameter.", + "type": "string", + "enum": [ "string", "number", "boolean", "datetime" ] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "entry_schema": { + "description": "used for complex data type in the future. 'type' must be map or array for entry_schema to kick_in. ", + "type": "string" + }, + "designer_editable": { + "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", + "type": "boolean" + }, + "policy_editable": { + "description": "A required property that declares a parameter as editable by DevOps in Policy UI (true) or not (false).", + "type": "boolean" + }, + "sourced_at_deployment": { + "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", + "type": "boolean" + }, + "policy_schema" :{ + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/policy_schema_parameter"} + } + }, + "required": [ + "name", + "value", + "description", + "designer_editable", + "policy_editable", + "sourced_at_deployment" + ], + "additionalProperties": false, + "dependencies": { "policy_schema": ["policy_editable"]} + }, + "policy_schema_parameter": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "The required data type for the parameter.", + "type": "string", + "enum": [ "string", "number", "boolean", "datetime", "list", "map" ] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "entry_schema": { + "description": "The optional key that is used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is either list or map. If the type is list and the entry type is a simple type (string, number, boolean, datetime), follow with a simple string to describe the entry type. If the type is list and the entry type is a map, follow with an array to describe the keys for the entry map. If the type is list and the entry type is also list, this is not currently supported here. If the type is map, then follow with an array to describe the keys for this map. ", + "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/policy_schema_parameter"} + } }, - "required": [ - "request", - "response", - "route" - ] - }, - "caller": { - "type": "object", - "properties": { + "required": [ + "name", + "type" + ], + "additionalProperties": false + }, + "parameter-constraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "equal": { + "description": "Constrains a property or parameter to a value equal to ('=') the value declared." + }, + "greater_than": { + "description": "Constrains a property or parameter to a value greater than ('>') the value declared.", + "type": "number" + }, + "greater_or_equal": { + "description": "Constrains a property or parameter to a value greater than or equal to ('>=') the value declared.", + "type": "number" + }, + "less_than": { + "description": "Constrains a property or parameter to a value less than '<') the value declared.", + "type": "number" + }, + "less_or_equal": { + "description": "Constrains a property or parameter to a value less than or equal to ('<=') the value declared.", + "type": "number" + }, + "valid_values": { + "description": "Constrains a property or parameter to a value that is in the list of declared values.", + "type": "array" + }, + "length": { + "description": "Constrains the property or parameter to a value of a given length.", + "type": "number" + }, + "min_length": { + "description": "Constrains the property or parameter to a value to a minimum length.", + "type": "number" + }, + "max_length": { + "description": "Constrains the property or parameter to a value to a maximum length.", + "type": "number" + } + } + }, + "stream_message_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "message router", "message_router" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "publisher_http": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "http", + "https" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "publisher_message_router": { + "$ref": "#/definitions/stream_message_router" + }, + "publisher_data_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "data router", "data_router" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "subscriber_http": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "route": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "http", + "https" + ] + } + }, + "required": [ + "format", + "version", + "route", + "type" + ] + }, + "subscriber_message_router": { + "$ref": "#/definitions/stream_message_router" + }, + "subscriber_data_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "route": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "data router", "data_router" + ] + }, + "config_key": { + "description": "Data router subscribers require config info to setup their endpoints to handle requests. For example, needs username and password", + "type": "string" + } + }, + "required": [ + "format", + "version", + "route", + "type", + "config_key" + ] + }, + "provider" : { + "oneOf" : [ + {"$ref": "#/definitions/docker-provider"}, + {"$ref": "#/definitions/cdap-provider"} + ] + }, + "cdap-provider" : { + "type": "object", + "properties" : { "request": { "$ref": "#/definitions/formatPair" }, "response": { "$ref": "#/definitions/formatPair" }, - "config_key": { - "type": "string" - } - }, - "required": [ - "request", - "response", - "config_key" - ] - }, - "formatPair": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - } - } - }, - "name": { - "type": "string" - }, - "version": { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "artifact": { - "type": "object", - "description": "Component artifact object", - "properties": { - "uri": { - "type": "string", - "description": "Uri to artifact" - }, - "type": { - "type": "string", - "enum": ["jar", "docker image"] - } - }, - "required": ["uri", "type"] - }, - - "auxilary_cdap": { - "title": "cdap component specification schema", - "type": "object", - "properties": { - "streamname": { - "type": "string" - }, - "artifact_name" : { - "type": "string" + "service_name" : { + "type" : "string" }, - "artifact_version" : { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + "service_endpoint" : { + "type" : "string" }, - "namespace":{ + "verb" : { "type": "string", - "description" : "optional" - }, - "programs": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/cdap_program" - } - } - }, - "required": [ - "streamname", - "programs", - "artifact_name", - "artifact_version" - ] - }, - "cdap_program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "cdap_program": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/cdap_program_type" - }, - "program_id": { - "type": "string" - } - }, - "required": ["program_type", "program_id"] - }, - - "auxilary_docker": { - "title": "Docker component specification schema", - "type": "object", - "properties": { - "healthcheck": { - "description": "Define the health check that Consul should perfom for this component", - "type": "object", - "oneOf": [ - { "$ref": "#/definitions/docker_healthcheck_http" }, - { "$ref": "#/definitions/docker_healthcheck_script" } - ] - }, - "ports": { - "description": "Port mapping to be used for Docker containers. Each entry is of the format <container port>:<host port>.", - "type": "array", - "items": { - "type": "string" - } - }, - "volumes": { - "description": "Volume mapping to be used for Docker containers. Each entry is of the format below", - "type": "array", - "items": { - "type": "object", - "properties": { - "host":{ - "type":"object", - "path": {"type": "string"} - }, - "container":{ - "type":"object", - "bind": { "type": "string"}, - "mode": { "type": "string"} - } + "enum": ["GET", "PUT", "POST", "DELETE"] + } + }, + "required" : [ + "request", + "response", + "service_name", + "service_endpoint", + "verb" + ] + }, + "docker-provider": { + "type": "object", + "properties": { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "route": { + "type": "string" + }, + "verb": { + "type": "string", + "enum": ["GET", "PUT", "POST", "DELETE"] + } + }, + "required": [ + "request", + "response", + "route" + ] + }, + "caller": { + "type": "object", + "properties": { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "config_key": { + "type": "string" + } + }, + "required": [ + "request", + "response", + "config_key" + ] + }, + "formatPair": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + } + } + }, + "name": { + "type": "string" + }, + "version": { + "type": "string", + "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + }, + "artifact": { + "type": "object", + "description": "Component artifact object", + "properties": { + "uri": { + "type": "string", + "description": "Uri to artifact" + }, + "type": { + "type": "string", + "enum": ["jar", "docker image"] + } + }, + "required": ["uri", "type"] + }, + + "auxilary_cdap": { + "title": "cdap component specification schema", + "type": "object", + "properties": { + "streamname": { + "type": "string" + }, + "artifact_name" : { + "type": "string" + }, + "artifact_version" : { + "type": "string", + "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + }, + "namespace":{ + "type": "string", + "description" : "optional" + }, + "programs": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/cdap_program" + } + } + }, + "required": [ + "streamname", + "programs", + "artifact_name", + "artifact_version" + ] + }, + "cdap_program_type": { + "type": "string", + "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] + }, + "cdap_program": { + "type": "object", + "properties": { + "program_type": { + "$ref": "#/definitions/cdap_program_type" + }, + "program_id": { + "type": "string" + } + }, + "required": ["program_type", "program_id"] + }, + + "auxilary_docker": { + "title": "Docker component specification schema", + "type": "object", + "properties": { + "healthcheck": { + "description": "Define the health check that Consul should perfom for this component", + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/docker_healthcheck_http" }, + { "$ref": "#/definitions/docker_healthcheck_script" } + ] + }, + "ports": { + "description": "Port mapping to be used for Docker containers. Each entry is of the format <container port>:<host port>.", + "type": "array", + "items": { + "type": "string" + } + }, + "reconfigs": { + "properties": { + "dti": { + "description": "Script command that will be executed for reconfiguration", + "type": "string" } - } - } - }, - "required": [ - "healthcheck" - ], - "additionalProperties": false - }, - "docker_healthcheck_http": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "http", - "https" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "endpoint": { - "description": "Relative endpoint used by Consul to check health by making periodic HTTP GET calls", - "type": "string" - } - }, - "required": [ - "type", - "endpoint" - ] - }, - "docker_healthcheck_script": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "script", - "docker" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "script": { - "description": "Script command that will be executed by Consul to check health", - "type": "string" - } - }, - "required": [ - "type", - "script" - ] - } - } - } + } + }, + "volumes": { + "description": "Volume mapping to be used for Docker containers. Each entry is of the format below", + "type": "array", + "items": { + "type": "object", + "properties": { + "host":{ + "type":"object", + "path": {"type": "string"} + }, + "container":{ + "type":"object", + "bind": { "type": "string"}, + "mode": { "type": "string"} + } + } + } + } + }, + "required": [ + "healthcheck" + ], + "additionalProperties": false + }, + "docker_healthcheck_http": { + "properties": { + "type": { + "description": "Consul health check type", + "type": "string", + "enum": [ + "http", + "https" + ] + }, + "interval": { + "description": "Interval duration in seconds i.e. 10s", + "default": "15s", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds i.e. 10s", + "default": "1s", + "type": "string" + }, + "endpoint": { + "description": "Relative endpoint used by Consul to check health by making periodic HTTP GET calls", + "type": "string" + } + }, + "required": [ + "type", + "endpoint" + ] + }, + "docker_healthcheck_script": { + "properties": { + "type": { + "description": "Consul health check type", + "type": "string", + "enum": [ + "script", + "docker" + ] + }, + "interval": { + "description": "Interval duration in seconds i.e. 10s", + "default": "15s", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds i.e. 10s", + "default": "1s", + "type": "string" + }, + "script": { + "description": "Script command that will be executed by Consul to check health", + "type": "string" + } + }, + "required": [ + "type", + "script" + ] + } + } + } diff --git a/docs/sections/components/component-specification/cdap-specification.rst b/docs/sections/components/component-specification/cdap-specification.rst index 022b46d7..2b26442a 100755 --- a/docs/sections/components/component-specification/cdap-specification.rst +++ b/docs/sections/components/component-specification/cdap-specification.rst @@ -23,8 +23,7 @@ Current Limitations and TODOs -----------------------------
- The integration of DMD is likely to significantly change the
- :any:`Interfaces <interfaces>` section in this specification, see
- :any:`DMaaP abstraction <dmaap-abstraction>`.
+ :any:`Interfaces <interfaces>` section in this specification.
.. _parameters:
@@ -34,7 +33,7 @@ Parameters There is a ``parameters`` section in your component specification. This
section contains three optional keys: `app_config <#appconfig>`__,
`app_preferences <#apppreferences>`__, and
-`propram_preferences <#programpreferences>`__:
+`program_preferences <#programpreferences>`__:
::
@@ -146,7 +145,7 @@ Each ``program_pref`` JSON is passed into the CDAP API as the preference for ``program_id``.
NOTE: for CDAP, this section is very likely to change when DMD is
-available. The *future* vision, as per :any:`DMaaP intentionally abstracted <dmaap-abstraction>` is
+available. The *future* vision is
that you would publish your data as a series of files on HDFS, and DMD
will pick them up and send them to the appropriate DMaaP feeds or
directly when needed.
diff --git a/docs/sections/components/component-specification/common-specification.rst b/docs/sections/components/component-specification/common-specification.rst index a29610f5..ae846bd0 100755 --- a/docs/sections/components/component-specification/common-specification.rst +++ b/docs/sections/components/component-specification/common-specification.rst @@ -11,6 +11,32 @@ common to both Docker and CDAP components. Differences for each are pointed out below. Elements that are very different, and described in
the CDAP or Docker specific pages.
+.. _metaschema:
+
+Meta Schema Definition
+----------------------
+
+The component specification is represented (and validated) against this
+`Component Spec json
+schema <https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob;f=component-json-schemas/component-specification/dcae-cli-v1/component-spec-schema.json;h=27d0403af67eac00e03ab89437d5f07aa06fbee3;hb=HEAD>`__
+and described below:
+
+The “Meta Schema” implementation defines how component specification
+JSON schemas can be written to define user input. It is itself a JSON
+schema (thus it is a “meta schema”). It requires the name of the
+component entry, component type (either ‘cdap’ or ‘docker’) and a
+description under “self” object. The meta schema version must be
+specified as the value of the “version” key. Then the input schema
+itself is described.
+
+There are four types of schema descriptions objects - jsonschema for
+inline standard JSON Schema definitions of JSON inputs, delimitedschema
+for delimited data input using a JSON description defined by AT&T,
+unstructured for unstructured text, and reference that allows a pointer
+to another artifact for a schema. The reference allows for XML and Protocol Buffer schema,
+but can be used as a pointer to JSON, Delimited Format, and Unstructured
+schemas as well.
+
.. _metadata:
Component Metadata
@@ -27,7 +53,7 @@ Example: "self": {
"version": "1.0.0",
- "name": "asimov.component.kpi_anomaly",
+ "name": "yourapp.component.kpi_anomaly",
"description": "Classifies VNF KPI data as anomalous",
"component_type": "docker"
},
@@ -109,7 +135,9 @@ Streams feed, it could be a direct stream of data being routed via HTTP too.
It abstractly refers to a sequence of data leaving a publisher.
- Streams have anonymous publish/subscribe semantics, which decouples
- the production of information from its consumption.
+ the production of information from its consumption. Like the component
+ specification, the data format specification is represented/validated against this
+ `Data Format json schema <https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;h=be1568291300305c7adb9a8d244d39f9e6ddadbd;hb=HEAD>`__
- In general, components are not aware of who they are communicating
with.
- Instead, components that are interested in data, subscribe to the
@@ -178,7 +206,7 @@ Example: ...
}
-This describes that ``asimov.component.kpi_anomaly`` exposes an HTTP
+This describes that ``yourapp.component.kpi_anomaly`` exposes an HTTP
endpoint called ``/data`` which accepts requests that have the data
format of ``dcae.vnf.kpi`` version ``1.0.0``.
@@ -290,7 +318,7 @@ be used to construct the delivery url needed to register the subscriber to the provisioned feed. Developers must also provide a ``config_key``
because there is dynamic configuration information associated with the
feed that the application will need e.g. username and password. See the
-page on :doc:`DMaaP connection objects <../dcae-cli/dmaap-connection-objects>` for more details on
+page on :any:`DMaaP connection objects <dmaap-data-router>` for more details on
the configuration information.
Example (not tied to the larger example):
@@ -318,16 +346,16 @@ Example: "streams": {
...
"publishes": [{
- "format": "asimov.format.integerClassification",
+ "format": "yourapp.format.integerClassification",
"version": "1.0.0",
"config_key": "prediction",
"type": "http"
}]
},
-This describes that ``asimov.component.kpi_anomaly`` publishes by making
+This describes that ``yourapp.component.kpi_anomaly`` publishes by making
POST requests to streams that support the data format
-``asimov.format.integerClassification`` version ``1.0.0``.
+``yourapp.format.integerClassification`` version ``1.0.0``.
``publishes`` Schema:
@@ -408,8 +436,8 @@ Message router Message router publishers are http clients of DMaap message_router.
Developers must provide a ``config_key`` because there is dynamic
configuration information associated with the feed that the application
-will need to receive e.g. topic url, username, password. See the page on
-:doc:`DMaaP connection objects <../dcae-cli/dmaap-connection-objects>` for more details on
+needs to receive e.g. topic url, username, password. See the page on
+:any:`DMaaP connection objects <dmaap-message-router>` for more details on
the configuration information.
Example (not tied to the larger example):
@@ -435,7 +463,7 @@ Data router publishers are http clients that make ``PUT`` requests to data router. Developers must also provide a ``config_key`` because there
is dynamic configuration information associated with the feed that the
application will need to receive e.g. publish url, username, password.
-See the page on :doc:`DMaaP connection objects <../dcae-cli/dmaap-connection-objects>` for more details on
+See the page on :any:`DMaaP connection objects <dmaap-data-router>` for more details on
the configuration information.
Example (not tied to the larger example):
@@ -537,7 +565,7 @@ Example: ...
}
-This describes that ``asimov.component.kpi_anomaly`` will make HTTP
+This describes that ``yourapp.component.kpi_anomaly`` will make HTTP
calls to a downstream component that accepts requests of data format
``dcae.vnf.meta`` version ``1.0.0`` and is expecting the response to be
``dcae.vnf.kpi`` version ``1.0.0``.
@@ -649,17 +677,17 @@ Example: "version": "1.0.0"
},
"response": {
- "format": "asimov.format.integerClassification",
+ "format": "yourapp.format.integerClassification",
"version": "1.0.0"
}
}]
},
-This describes that ``asimov.component.kpi_anomaly`` provides a service
+This describes that ``yourapp.component.kpi_anomaly`` provides a service
interface and it is exposed on the ``/score-vnf`` HTTP endpoint. The
endpoint accepts requests that have the data format ``dcae.vnf.meta``
version ``1.0.0`` and gives back a response of
-``asimov.format.integerClassification`` version ``1.0.0``.
+``yourapp.format.integerClassification`` version ``1.0.0``.
``provides`` Schema for a Docker component:
@@ -807,6 +835,8 @@ services: | | | ‘POST’ |
+-------------+----+-----------+
+.. _common-specification-parameters:
+
Parameters
----------
@@ -926,7 +956,7 @@ Parameter object has the following available properties: | | | at this | |
| | | time | |
+--------------+----+----------+------+
-| designer_ed\ | bo\| An | true |
+| designer_ed\ | bo\| An | |
| itable | ol\| optional | |
| | ea\| key that | |
| | n | declares | |
@@ -941,8 +971,8 @@ Parameter object has the following available properties: | | | or not | |
| | | (false) | |
+--------------+----+----------+------+
-| sourced_at_d\| bo\| An | fals\|
-| eployment | ol\| optional | e |
+| sourced_at_d\| bo\| An | |
+| eployment | ol\| optional | |
| | ea\| key that | |
| | n | declares | |
| | | a | |
@@ -957,7 +987,7 @@ Parameter object has the following available properties: | | | time | |
| | | (true) | |
+--------------+----+----------+------+
-| policy_edit\ | bo\| An | true |
+| policy_edit\ | bo\| An | |
| able | ol\| optional | |
| | ea\| key that | |
| | n | declares | |
@@ -1374,23 +1404,8 @@ policy_schema object: Generated Application Configuration
-----------------------------------
-The above example for component ``asimov.component.kpi_anomaly`` will
-get transformed into the following application configuration JSON that
-is fully resolved and provided at runtime by calling the
-``config binding service``:
+The Platform generates configuration for the component (based on the component spec) at deployment time. The generated application configuration will be made up of the Parameters, Streams, and Services sections, after any provisioning for streams and services. The component developer can see what this configuration will look like by reviewing the :any:`component dev command <dcae-cli-run-the-dev-command>`.
-.. code:: json
-
- {
- "streams_publishes": {
- "prediction": ["10.100.1.100:32567"]
- },
- "streams_subscribes": {},
- "threshold": 0.75,
- "services_calls": {
- "vnf-db": ["10.100.1.101:32890"]
- }
- }
.. _artifacts:
@@ -1416,3 +1431,12 @@ the Docker image. For CDAP, this is the full path to the CDAP jar. +---------------+--------+--------------------------------------------+
| type | string | *Required*. ``docker image`` or ``jar`` |
+---------------+--------+--------------------------------------------+
+
+.. _component_spec:
+
+Working with Component Specs
+============================
+
+Components can be added to the onboarding catalog (which first validates the component spec) by using the :doc:`dcae_cli Tool <../dcae-cli/quickstart/>`
+Here you can also list the components, show the contents of a component, publish the component, validate the generated configuration for the component, deploy (run) and undeploy the component. For a list of these capabilities, see :any:`Component Commands <dcae_cli_component_commands>`
+
diff --git a/docs/sections/components/component-specification/component-specification.rst b/docs/sections/components/component-specification/component-specification.rst index 759f3dc9..56649d31 100644 --- a/docs/sections/components/component-specification/component-specification.rst +++ b/docs/sections/components/component-specification/component-specification.rst @@ -10,7 +10,7 @@ Component Specification ./common-specification.rst ./cdap-specification.rst ./docker-specification.rst - ./generated-configuration.rst + ./dmaap-connection-objects.rst ./streams-grid.rst ./configuration-grid.rst diff --git a/docs/sections/components/component-specification/configuration-grid.rst b/docs/sections/components/component-specification/configuration-grid.rst index 956e1ff2..da9b4ab7 100755 --- a/docs/sections/components/component-specification/configuration-grid.rst +++ b/docs/sections/components/component-specification/configuration-grid.rst @@ -4,312 +4,118 @@ Configuration Quick Reference
=============================
-The following types of configuration are supported by the DCAE Platform.
+Default Values
+^^^^^^^^^^^^^^
-+---------+---------+----------+---------+-----------+---+
-| Input | Default | Designer | Clamp | Policy | R\|
-| Sources | Values | Input | Input | Input | u\|
-| | | | | | n\|
-| | | | | | t\|
-| | | | | | i\|
-| | | | | | m\|
-| | | | | | e |
-| | | | | | I\|
-| | | | | | n\|
-| | | | | | p\|
-| | | | | | u\|
-| | | | | | t |
-+=========+=========+==========+=========+===========+===+
-| Notes | | This | This | | |
-| | | applies | applies | | |
-| | | only to | only to | | |
-| | | componen\| compone\| | |
-| | | ts | nts | | |
-| | | that are | that | | |
-| | | self-ser\| are | | |
-| | | vice | part of | | |
-| | | (support\| a | | |
-| | | ed | closed-\| | |
-| | | by SDC) | loop | | |
-| | | | interfa\| | |
-| | | | ce | | |
-+---------+---------+----------+---------+-----------+---+
-| Who | Compone\| Service | CLAMP | Operatio\ | R\|
-| provide\| nt | Designer | | ns | u\|
-| s? | Develop | | | | n\|
-| | er | | | | t\|
-| | | | | | i\|
-| | | | | | m\|
-| | | | | | e |
-| | | | | | P\|
-| | | | | | l\|
-| | | | | | a\|
-| | | | | | t\|
-| | | | | | f\|
-| | | | | | o\|
-| | | | | | r\|
-| | | | | | m |
-+---------+---------+----------+---------+-----------+---+
-| When/Wh\| During | At | At | Anytime | W\|
-| ere | onboard\| design | install\| – in the | h\|
-| it is | ing | time – | ation | POLICY | e\|
-| provide | – in | in the | – in | GUI | n |
-| d | the | SDC UI | the | | t\|
-| | compone\| | CLAMP | | h\|
-| | nt | | UI | | e |
-| | specifi\| | | | c\|
-| | cation | | | | o\|
-| | | | | | m\|
-| | | | | | p\|
-| | | | | | o\|
-| | | | | | n\|
-| | | | | | e\|
-| | | | | | n\|
-| | | | | | t |
-| | | | | | i\|
-| | | | | | s |
-| | | | | | d\|
-| | | | | | e\|
-| | | | | | p\|
-| | | | | | l\|
-| | | | | | o\|
-| | | | | | y\|
-| | | | | | e\|
-| | | | | | d |
-+---------+---------+----------+---------+-----------+---+
-| Compone\| For | ‘designe\| | ‘policy-\ | ‘\|
-| nt | CDAP: | r-editab\| | editable\ | s\|
-| Specifi\| ‘value’ | le’ | | ’ | o\|
-| cation | Name | must be | | must be | u\|
-| Details | and KV | set to | | set to | r\|
-| | pairs | ‘true’ | | ‘true’ | c\|
-| | in | for | | and | e\|
-| | AppConf\| variable | | ‘policy_s\| d\|
-| | ig | in | | chema’ | _\|
-| | or | ‘paramet\| | must be | a\|
-| | AppPref\| er’ | | provided | t\|
-| | erences | section. | | for | _\|
-| | For | | | variable | \ |
-| | Docker: | | | in | d\|
-| | ‘value’ | | | ‘paramet\ | e\|
-| | is | | | er’ | p\|
-| | provide | | | section | l\|
-| | d | | | | o\|
-| | for | | | | y\|
-| | variabl\| | | | m\|
-| | e | | | | e\|
-| | in | | | | n\|
-| | ‘parame\| | | | t\|
-| | ter’ | | | | ’ |
-| | section | | | | m\|
-| | | | | | u\|
-| | | | | | s\|
-| | | | | | t |
-| | | | | | b\|
-| | | | | | e |
-| | | | | | s\|
-| | | | | | e\|
-| | | | | | t |
-| | | | | | t\|
-| | | | | | o |
-| | | | | | ‘\|
-| | | | | | t\|
-| | | | | | r\|
-| | | | | | u\|
-| | | | | | e |
-| | | | | | ’\|
-| | | | | | f\|
-| | | | | | o\|
-| | | | | | r |
-| | | | | | v\|
-| | | | | | a\|
-| | | | | | r\|
-| | | | | | i\|
-| | | | | | a\|
-| | | | | | b\|
-| | | | | | l\|
-| | | | | | e\|
-| | | | | | i\|
-| | | | | | n\|
-| | | | | | ‘ |
-| | | | | | p\|
-| | | | | | a\|
-| | | | | | r\|
-| | | | | | a\|
-| | | | | | m\|
-| | | | | | e\|
-| | | | | | t\|
-| | | | | | e\|
-| | | | | | r\|
-| | | | | | ’ |
-| | | | | | s\|
-| | | | | | e\|
-| | | | | | c\|
-| | | | | | t\|
-| | | | | | i\|
-| | | | | | o\|
-| | | | | | n |
-+---------+---------+----------+---------+-----------+---+
+The component developer can provide default values for any ``parameter``
+in the component specification. These defaults will be passed to the
+component in its generated configuration.
+
+Overridden/Entered Values
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Depending on the other properties set for the parameter, the default
+value can be overridden at ‘design-time’, ‘deploy-time’ or once the
+microservice is running (‘run-time’). (*In the future, when Policy is
+supported, configuration will also be able to be provided/changed in the
+Policy UI at any time).*
+
+
++--------+--------+--------+--------+------------+
+| | Design\| CLAMP | Policy | Deploy-Time|
+| | -Time | Input | Input | Input |
+| | Input | | (futur\| |
+| | | | e) | |
++========+========+========+========+============+
+| Descri\| Applie\| Applie\| (not | Applies to |
+| ption | s | s | yet | manually |
+| | to SDC | to | suppor\| deployed |
+| | self-s\| compon\| ted) | services |
+| | ervice | ents | | |
+| | compon\| deploy\| | |
+| | ents | ed | | |
+| | | by | | |
+| | | CLAMP | | |
++--------+--------+--------+--------+------------+
+| Input | Servic\| CLAMP | Operat\| DevOps |
+| provid\| e | | ions | |
+| ed | Design | | | |
+| by | er | | | |
++--------+--------+--------+--------+------------+
+| How it | In the | In the | In the | In the DCAE|
+| is | SDC UI | CLAMP | POLICY | Dashboard |
+| provid\| | UI | GUI | (or Jenkins|
+| ed | | | | job) |
++--------+--------+--------+--------+------------+
+| Compon\| ‘desig\| None. | ‘polic\| ‘sourced\ |
+| ent | ner-ed\| Develo\| y_edit\| _at_deploy\|
+| Specif\| itable\| per | able’ | ment’ must |
+| icatio\| ’ | provid\| must | be set to |
+| n | set to | es | be set | ‘true’ |
+| Detail\| ‘true’ | CLAMP | to | |
+| s | | an | ‘true’ | |
+| | | email | and | |
+| | | with | ‘polic\| |
+| | | parame\| y_sche\| |
+| | | ters | ma’ | |
+| | | to be | must | |
+| | | suppor\| be | |
+| | | ted | provid\| |
+| | | | ed | |
+| | | | | |
+| | | | | |
++--------+--------+--------+--------+------------+
+| Additi\| | | For | |
+| onal | | | Docker | |
+| Info | | | only: | |
+| for | | | In the | |
+| Compon\| | | auxili\| |
+| ent | | | ary | |
+| Develo\| | | sectio\| |
+| per | | | n: | |
+| | | | {“poli\| |
+| | | | cy”: | |
+| | | | {“trig\| |
+| | | | ger_ty\| |
+| | | | pe”: | |
+| | | | “polic\| |
+| | | | y”,“sc\| |
+| | | | ript_p\| |
+| | | | ath”: | |
+| | | | “/opt/\| |
+| | | | app/re\| |
+| | | | config\| |
+| | | | ure.sh | |
+| | | | ”} | |
+| | | | } | |
+| | | | Script | |
+| | | | interf\| |
+| | | | ace | |
+| | | | would | |
+| | | | then | |
+| | | | be | |
+| | | | “/opt/\| |
+| | | | app/re\| |
+| | | | config\| |
+| | | | ure.sh | |
+| | | | ” | |
+| | | | $trigg\| |
+| | | | er_typ\| |
+| | | | e | |
+| | | | $updat\| |
+| | | | ed_pol\| |
+| | | | icy" | |
+| | | | where | |
+| | | | $updat\| |
+| | | | ed_pol\| |
+| | | | icy | |
+| | | | is | |
+| | | | json | |
+| | | | provid\| |
+| | | | ed | |
+| | | | by the | |
+| | | | Policy | |
+| | | | Handle\| |
+| | | | r. | |
++--------+--------+--------+--------+------------+
-+---------+---------+----------+---------+-----------+---+
-| How it | This is | This | This | This | T\|
-| is used | passed | override\| overrid\| override\ | h\|
-| | to the | s | es | s | i\|
-| | compone\| any | any | any | s |
-| | nt | values | values | values | o\|
-| | in the | previous\| previou\| previous\ | v\|
-| | generat\| ly | sly | ly | e\|
-| | ed | set, but | set, | set, but | r\|
-| | configu\| can be | but can | can be | r\|
-| | ration | overridd\| be | overridd\ | i\|
-| | if not | en | overrid\| en | d\|
-| | overrid\| by CLAMP | den | at any | e\|
-| | den. | or | by | point | s |
-| | | POLICY. | POLICY. | thereaft\ | a\|
-| | | | | er. | n\|
-| | | | | | y |
-| | | | | | v\|
-| | | | | | a\|
-| | | | | | l\|
-| | | | | | u\|
-| | | | | | e\|
-| | | | | | s |
-| | | | | | p\|
-| | | | | | r\|
-| | | | | | e\|
-| | | | | | v\|
-| | | | | | i\|
-| | | | | | o\|
-| | | | | | u\|
-| | | | | | s\|
-| | | | | | l\|
-| | | | | | y |
-| | | | | | s\|
-| | | | | | e\|
-| | | | | | t\|
-| | | | | | , |
-| | | | | | b\|
-| | | | | | u\|
-| | | | | | t |
-| | | | | | c\|
-| | | | | | a\|
-| | | | | | n |
-| | | | | | b\|
-| | | | | | e |
-| | | | | | o\|
-| | | | | | v\|
-| | | | | | e\|
-| | | | | | r\|
-| | | | | | r\|
-| | | | | | i\|
-| | | | | | d\|
-| | | | | | d\|
-| | | | | | e\|
-| | | | | | n |
-| | | | | | a\|
-| | | | | | t |
-| | | | | | a\|
-| | | | | | n\|
-| | | | | | y |
-| | | | | | p\|
-| | | | | | o\|
-| | | | | | i\|
-| | | | | | n\|
-| | | | | | t |
-| | | | | | t\|
-| | | | | | h\|
-| | | | | | e\|
-| | | | | | r\|
-| | | | | | e\|
-| | | | | | a\|
-| | | | | | f\|
-| | | | | | t\|
-| | | | | | e\|
-| | | | | | r |
-| | | | | | b\|
-| | | | | | y |
-| | | | | | P\|
-| | | | | | o\|
-| | | | | | l\|
-| | | | | | i\|
-| | | | | | c\|
-| | | | | | y\|
-| | | | | | . |
-+---------+---------+----------+---------+-----------+---+
-| Additio\| For | | | For | |
-| nal | CDAP: | | | Docker: | |
-| Info | ‘value’ | | | In the | |
-| for | is | | | auxiliar\ | |
-| Compone\| provide\| | | y | |
-| nt | d | | | section: | |
-| Develop\| for | | | {“policy | |
-| er | variabl\| | | ”: | |
-| | e | | | {“trigge\ | |
-| | in the | | | r_type”: | |
-| | ‘AppCon\| | | “policy” | |
-| | fig’ | | | ,“script\ | |
-| | or | | | _path”: | |
-| | ‘AppPre\| | | “/opt/ap\ | |
-| | ference\| | | p/reconf\ | |
-| | s’ | | | igure.sh\ | |
-| | section\| | | ”} | |
-| | s | | | } Script | |
-| | For | | | interfac\ | |
-| | Docker: | | | e | |
-| | ‘value’ | | | must be | |
-| | is | | | “opt/app\ | |
-| | provide\| | | /reconfi\ | |
-| | d | | | gure.sh” | |
-| | for | | | $trigger\ | |
-| | variabl\| | | _type | |
-| | e | | | $updated\ | |
-| | in | | | _policie\ | |
-| | ‘parame\| | | s | |
-| | ter’ | | | $updated\ | |
-| | section | | | _appl_co\ | |
-| | | | | nfig" | |
-| | | | | where | |
-| | | | | $updated\ | |
-| | | | | _policie\ | |
-| | | | | s | |
-| | | | | is a | |
-| | | | | json | |
-| | | | | provided | |
-| | | | | by the | |
-| | | | | Policy | |
-| | | | | Handler | |
-| | | | | and | |
-| | | | | $update_a\| |
-| | | | | ppl_con\ | |
-| | | | | fig | |
-| | | | | is the | |
-| | | | | post-mer\ | |
-| | | | | ged | |
-| | | | | appl\ | |
-| | | | | config | |
-| | | | | which | |
-| | | | | may | |
-| | | | | contain | |
-| | | | | unresolv\ | |
-| | | | | ed | |
-| | | | | configur\ | |
-| | | | | ation | |
-| | | | | that | |
-| | | | | didn’t | |
-| | | | | come | |
-| | | | | from | |
-| | | | | policy. | |
-| | | | | Suggesti\ | |
-| | | | | on | |
-| | | | | is for | |
-| | | | | script | |
-| | | | | to call | |
-| | | | | CONFIG | |
-| | | | | BINDING | |
-| | | | | SERVICE | |
-| | | | | to | |
-| | | | | resolve | |
-| | | | | any | |
-| | | | | configur\ | |
-| | | | | ation. | |
-+---------+---------+----------+---------+-----------+---+
diff --git a/docs/sections/components/component-specification/dmaap-connection-objects.rst b/docs/sections/components/component-specification/dmaap-connection-objects.rst new file mode 100755 index 00000000..dfd4980f --- /dev/null +++ b/docs/sections/components/component-specification/dmaap-connection-objects.rst @@ -0,0 +1,276 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+
+.. _dmaap-connection-objects:
+
+DMaaP connection objects
+========================
+
+DMaaP Connection Objects are JSON objects that:
+
+1. At Runtime - this is generated by the DCAE Platform and passed to the
+ component in its application_configuration to be used to connect to
+ the DMaaP feed or topic. Components will receive the entire object
+ with all properties populated (default will be \`null) unless
+ specified otherwise.
+2. During dcae_cli testing - this is provided through the command-line
+ argument ``--dmaap-file`` to test the component with manually
+ provisioned feeds and topics. Developers are not required to provide
+ the entire object. The required properties are labeled below with
+ “*Required as input*”.
+
+.. _dmaap-message-router:
+
+Message Router
+--------------
+
+Publishers and subscribers have the same generated
+``Dmaap Connection Object`` structure. Here’s an example for any given
+config-key: (This is what will be in application_configuration)
+
+.. code:: json
+
+ {
+ "type": "message_router",
+ "aaf_username": "some-user",
+ "aaf_password": "some-password",
+ "dmaap_info": {
+ "client_role": "com.dcae.member",
+ "client_id": "1500462518108",
+ "location": "mtc00",
+ "topic_url": "https://we-are-message-router.us:3905/events/some-topic"
+ }
+ }
+
+At the top-level:
+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| type | string | *Required as input*. Must |
+| | | be ``message_router`` for |
+| | | message router topics |
++--------------------------------+---------+---------------------------+
+| aaf_username | string | AAF username message |
+| | | router clients use to |
+| | | authenticate with secure |
+| | | topics |
++--------------------------------+---------+---------------------------+
+| aaf_password | string | AAF password message |
+| | | router clients use to |
+| | | authenticate with secure |
+| | | topics |
++--------------------------------+---------+---------------------------+
+| dmaap_info | JSON | *Required as input*. |
+| | object | Contains the topic |
+| | | connection details |
++--------------------------------+---------+---------------------------+
+
+The ``dmaap_info`` object contains:
+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| client_role | string | AAF client role that’s |
+| | | requesting publish or |
+| | | subscribe access to the |
+| | | topic |
++--------------------------------+---------+---------------------------+
+| client_id | string | Client id for given AAF |
+| | | client |
++--------------------------------+---------+---------------------------+
+| location | string | DCAE location for the |
+| | | publisher or subscriber, |
+| | | used to set up routing |
++--------------------------------+---------+---------------------------+
+| topic_url | string | *Required as input*. URL |
+| | | for accessing the topic |
+| | | to publish or receive |
+| | | events |
++--------------------------------+---------+---------------------------+
+
+The –dmaap-file argument (to the component ``run`` or ``dev`` command),
+must minimally contain:
+
+.. code:: json
+
+ {
+ "type": "message_router",
+ "dmaap_info": {
+ "topic_url": "https://we-are-message-router.us:3905/events/some-topic"
+ }
+ }
+
+.. _dmaap-data-router:
+
+Data Router
+-----------
+
+Publisher
+~~~~~~~~~
+
+Here’s an example of what the generated ``Dmaap Connection Object`` for
+Data Router Publisher looks like: (This is what will be in
+application_configuration)
+
+.. code:: json
+
+ {
+ "type": "data_router",
+ "dmaap_info": {
+ "location": "mtc00",
+ "publish_url": "https://we-are-data-router.us/feed/xyz",
+ "log_url": "https://we-are-data-router.us/feed/xyz/logs",
+ "username": "some-user",
+ "password": "some-password",
+ "publisher_id": "123456"
+ }
+ }
+
+At the top-level:
+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| type | string | *Required as input*. Must |
+| | | be ``data_router`` for |
+| | | data router feeds |
++--------------------------------+---------+---------------------------+
+| dmaap_info | JSON | *Required as input*. |
+| | object | Contains the feed |
+| | | connection details |
++--------------------------------+---------+---------------------------+
+
+The ``dmaap_info`` object contains:
+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| location | string | DCAE location for the |
+| | | publisher, used to set up |
+| | | routing |
++--------------------------------+---------+---------------------------+
+| publish_url | string | *Required as input*. URL |
+| | | to which the publisher |
+| | | makes Data Router publish |
+| | | requests |
++--------------------------------+---------+---------------------------+
+| log_url | string | URL from which log data |
+| | | for the feed can be |
+| | | obtained |
++--------------------------------+---------+---------------------------+
+| username | string | Username the publisher |
+| | | uses to authenticate to |
+| | | Data Router |
++--------------------------------+---------+---------------------------+
+| password | string | Password the publisher |
+| | | uses to authenticate to |
+| | | Data Router |
++--------------------------------+---------+---------------------------+
+| publisher_id | string | Publisher id in Data |
+| | | Router |
++--------------------------------+---------+---------------------------+
+
+The –dmaap-file argument (to the component ``run`` or ``dev`` command),
+must minimally contain:
+
+.. code:: json
+
+ {
+ "type": "data_router",
+ "dmaap_info": {
+ "publish_url": "https://we-are-data-router.us/feed/xyz"
+ }
+ }
+
+Subscriber
+~~~~~~~~~~
+
+Here’s an example of what the generated ``Dmaap Connection Object`` for
+a Data Router Subscriber looks like: (This is what will be passed in
+application_configuration)
+
+.. code:: json
+
+ {
+ "type": "data_router",
+ "dmaap_info": {
+ "location": "mtc00",
+ "delivery_url": "https://my-subscriber-app.dcae:8080/target-path",
+ "username": "some-user",
+ "password": "some-password",
+ "subscriber_id": "789012"
+ }
+ }
+
+At the top-level:
+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| type | string | *Required as input*. Must |
+| | | be ``data_router`` for |
+| | | data router feeds |
++--------------------------------+---------+---------------------------+
+| dmaap_info | JSON | *Required as input*. |
+| | object | Contains the feed |
+| | | connection details |
++--------------------------------+---------+---------------------------+
+
+The ``dmaap_info`` object contains:
+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| location | string | DCAE location for the |
+| | | subscriber, used to set |
+| | | up routing |
++--------------------------------+---------+---------------------------+
+| delivery_url | string | URL to which the Data |
+| | | Router should deliver |
+| | | files |
++--------------------------------+---------+---------------------------+
+| username | string | Username Data Router uses |
+| | | to authenticate to the |
+| | | subscriber when |
+| | | delivering files |
++--------------------------------+---------+---------------------------+
+| password | string | Password Data Router uses |
+| | | to authenticate to the |
+| | | subscriber when |
+| | | delivering files |
++--------------------------------+---------+---------------------------+
+| subscriber_id | string | Subscriber id in Data |
+| | | Router |
++--------------------------------+---------+---------------------------+
+
+The –dmaap-file argument (to the component ``run`` or ``dev`` command),
+must minimally contain:
+
+.. code:: json
+
+ {
+ "type": "data_router",
+ "dmaap_info": {
+ }
+ }
+
+It is the recommended security practice that ``username`` and
+``password`` are specified. You cannot provide the ``delivery_url`` in
+your dmaap-file, because it’s not constructed until deployment time.
+Therefore, after the test deployment, go back to the Data Router Feed
+and provide the delivery_url (in order to start receiving the feeds).
+
+Data Router Example
+~~~~~~~~~~~~~~~~~~~
+
+(After the Data Router feed has been manually provisioned)
+
+::
+
+ $ dcae_cli component run --dmaap-file $dmaap_file $component-name
+ DCAE.Run | WARNING | Your component is a data router subscriber. Here are the delivery urls:
+
+ some-sub-dr: http://135.205.226.128:32838/identity
+
+(Update the Data Router Feed to provide the delivery_url).
diff --git a/docs/sections/components/component-specification/docker-specification.rst b/docs/sections/components/component-specification/docker-specification.rst index 38612e17..01e11536 100755 --- a/docs/sections/components/component-specification/docker-specification.rst +++ b/docs/sections/components/component-specification/docker-specification.rst @@ -23,58 +23,38 @@ Auxiliary Details -----------------
``auxiliary`` contains Docker specific details like health check, port
-mapping, volume mapping, and policy reconfiguration script details.
-
-+-------------+----+----------+
-| Name | Ty\| Descript\|
-| | pe | ion |
-+=============+====+==========+
-| healthcheck | JS\| *Require\|
-| | ON | d*. |
-| | ob\| Health |
-| | je\| check |
-| | ct | definiti\|
-| | | on |
-| | | details |
-+-------------+----+----------+
-| ports | JS\| each |
-| | ON | array |
-| | ar\| item |
-| | ra\| maps a |
-| | y | containe\|
-| | | r |
-| | | port to |
-| | | the host |
-| | | port. |
-| | | See |
-| | | example |
-| | | below. |
-+-------------+----+----------+
-| volume | JS\| each |
-| | ON | array |
-| | ar\| item |
-| | ra\| contains |
-| | y | a host |
-| | | and |
-| | | containe\|
-| | | r |
-| | | object. |
-| | | See |
-| | | example |
-| | | below. |
-+-------------+----+----------+
-| policy | JS\| *Require\|
-| | ON | d*. |
-| | ar\| Policy |
-| | ra\| script |
-| | y | details |
-+-------------+----+----------+
+mapping, volume mapping and policy reconfiguration script details.
+(Policy reconfiguration is not yet supported).
+
++--------------------------------+---------+---------------------------+
+| Name | Type | Description |
++================================+=========+===========================+
+| healthcheck | JSON | *Required*. Health check |
+| | object | definition details |
++--------------------------------+---------+---------------------------+
+| ports | JSON | each array item maps a |
+| | array | container port to the |
+| | | host port. See example |
+| | | below. |
++--------------------------------+---------+---------------------------+
+| volume | JSON | each array item contains |
+| | array | a host and container |
+| | | object. See example |
+| | | below. |
++--------------------------------+---------+---------------------------+
+| *Planned for 1806* | | |
++--------------------------------+---------+---------------------------+
+| policy | JSON | *Required*. Policy |
+| | array | reconfiguration script |
+| | | details |
++--------------------------------+---------+---------------------------+
Health Check Definition
~~~~~~~~~~~~~~~~~~~~~~~
The platform uses Consul to perform periodic health check calls. Consul
-provides different types of `check definitions <https://www.consul.io/docs/agent/checks.html>`_. The
+provides different types of `check
+definitions <https://www.consul.io/docs/agent/checks.html>`__. The
platform currently supports http and docker health checks.
When choosing a value for interval, consider that too frequent
@@ -85,47 +65,28 @@ problematic resource, then more frequent healthchecks are warranted (eg When choosing a value for timeout, consider that too small a number will
result in increasing timeout failures, and too large a number will
-result in a delay in the notification of resource problem. A suggestion
-is to start with 5s and workd from there.
+result in a delay in the notification of the resource problem. A
+suggestion is to start with 5s and work from there.
http
^^^^
-+-------------+----+----------+
-| Property | Ty\| Descript\|
-| Name | pe | ion |
-+=============+====+==========+
-| type | st\| *Require\|
-| | ri\| d*. |
-| | ng | ``http`` |
-+-------------+----+----------+
-| interval | st\| Interval |
-| | ri\| duration |
-| | ng | in |
-| | | seconds |
-| | | i.e. |
-| | | ``60s`` |
-+-------------+----+----------+
-| timeout | st\| Timeout |
-| | ri\| in |
-| | ng | seconds |
-| | | i.e. |
-| | | ``5s`` |
-+-------------+----+----------+
-| endpoint | st\| *Require\|
-| | ri\| d*. |
-| | ng | GET |
-| | | endpoint |
-| | | provided |
-| | | by the |
-| | | componen\|
-| | | t |
-| | | for |
-| | | Consul |
-| | | to call |
-| | | to check |
-| | | health |
-+-------------+----+----------+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| type | string | *Required*. ``http`` |
++--------------------------------+---------+---------------------------+
+| interval | string | Interval duration in |
+| | | seconds i.e. ``60s`` |
++--------------------------------+---------+---------------------------+
+| timeout | string | Timeout in seconds i.e. |
+| | | ``5s`` |
++--------------------------------+---------+---------------------------+
+| endpoint | string | *Required*. GET endpoint |
+| | | provided by the component |
+| | | for Consul to call to |
+| | | check health |
++--------------------------------+---------+---------------------------+
Example:
@@ -143,43 +104,25 @@ Example: docker script example
^^^^^^^^^^^^^^^^^^^^^
-+-------------+----+------------+
-| Property | Ty\| Descript\ |
-| Name | pe | ion |
-+=============+====+============+
-| type | st\| *Require\ |
-| | ri\| d*. |
-| | ng | ``docker`` |
-+-------------+----+------------+
-| interval | st\| Interval |
-| | ri\| duration |
-| | ng | in |
-| | | seconds |
-| | | i.e. |
-| | | ``15s`` |
-+-------------+----+------------+
-| timeout | st\| Timeout |
-| | ri\| in |
-| | ng | seconds |
-| | | i.e. |
-| | | ``1s`` |
-+-------------+----+------------+
-| script | st\| *Require\ |
-| | ri\| d*. |
-| | ng | Full |
-| | | path of |
-| | | script |
-| | | that |
-| | | exists |
-| | | in the |
-| | | Docker |
-| | | containe\ |
-| | | r |
-| | | to be |
-| | | executed |
-+-------------+----+------------+
-
-Consul will use the `Docker exec API <https://docs.docker.com/engine/api/v1.29/#tag/Exec>`_ to
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| type | string | *Required*. ``docker`` |
++--------------------------------+---------+---------------------------+
+| interval | string | Interval duration in |
+| | | seconds i.e. ``15s`` |
++--------------------------------+---------+---------------------------+
+| timeout | string | Timeout in seconds i.e. |
+| | | ``1s`` |
++--------------------------------+---------+---------------------------+
+| script | string | *Required*. Full path of |
+| | | script that exists in the |
+| | | Docker container to be |
+| | | executed |
++--------------------------------+---------+---------------------------+
+
+Consul will use the `Docker exec
+API <https://docs.docker.com/engine/api/v1.29/#tag/Exec>`__ to
periodically call your script in your container. It will examine the
script result to identify whether your component is healthy. Your
component is considered healthy when the script returns ``0`` otherwise
@@ -201,6 +144,13 @@ Example: Ports
~~~~~
+This method of exposing/mapping a local port to a host port is NOT
+RECOMMENDED because of the possibility of port conflicts. If multiple
+instances of a docker container will be running, there definitely will
+be port conflicts. Use at your own risk. (The preferred way to expose a
+port is to do so in the Dockerfile as described
+:any:`here <dcae-cli-docker-ports>`).
+
.. code:: json
"auxilary": {
@@ -238,19 +188,19 @@ At the top-level: The ``container`` object contains:
-+----------------------+----------------------+----------------------+
-| Property Name | Type | Description |
-+======================+======================+======================+
-| bind | string | path to the |
-| | | container volume |
-+----------------------+----------------------+----------------------+
-| mode | string | “ro” - indicates |
-| | | read-only volume |
-| | | “” - indicates that |
-| | | the container can |
-| | | write into the bind |
-| | | mount |
-+----------------------+----------------------+----------------------+
++-----------------------+-----------------------+-----------------------+
+| Property Name | Type | Description |
++=======================+=======================+=======================+
+| bind | string | path to the container |
+| | | volume |
++-----------------------+-----------------------+-----------------------+
+| mode | string | “ro” - indicates |
+| | | read-only volume |
++-----------------------+-----------------------+-----------------------+
+| | “” - indicates that |
+| | the contain can write |
+| | into the bind mount |
++-----------------------+-----------------------+-----------------------+
The ``host`` object contains:
@@ -280,39 +230,24 @@ Here’s an example of the minimal JSON that must be provided as an input: In the example above, the container volume “/tmp/docker.sock” maps to
host volume “/var/run/docker.sock”.
-Policy
-~~~~~~
+
+Policy (not yet supported)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
Policy changes made in the Policy UI will be provided to the Docker
component by triggering a script that is defined here.
-+-------------+----+------------+
-| Property | Ty\| Descript\ |
-| Name | pe | ion |
-+=============+====+============+
-| reconfigure | st\| *Require\ |
-| _type | ri\| d*. |
-| | ng | Current |
-| | | value |
-| | | supporte |
-| | | d |
-| | | is |
-| | | ``policy`` |
-+-------------+----+------------+
-| script_path | st\| *Require\ |
-| | ri\| d*. |
-| | ng | Current |
-| | | value |
-| | | for |
-| | | ‘policy’ |
-| | | reconfig\ |
-| | | ure_type |
-| | | must be |
-| | | “/opt/ap\ |
-| | | p/reconf\ |
-| | | igure.sh |
-| | | ” |
-+-------------+----+------------+
++--------------------------------+---------+---------------------------+
+| Property Name | Type | Description |
++================================+=========+===========================+
+| reconfigure_type | string | *Required*. Current value |
+| | | supported is ``policy`` |
++--------------------------------+---------+---------------------------+
+| script_path | string | *Required*. Current value |
+| | | for ‘policy’ |
+| | | reconfigure_type must be |
+| | | “/opt/app/reconfigure.sh” |
++--------------------------------+---------+---------------------------+
Example:
@@ -328,59 +263,22 @@ Example: The docker script interface is as follows: \`/opt/app/reconfigure.sh
$reconfigure_type {“updated policies”: , “application config”: }
-+-----+----+---------------------------+
-| Na\ | Ty\| Descript\ |
-| me | pe | ion |
-+=====+====+===========================+
-| re\ | st\| “policy” |
-| co\ | ri\| |
-| nf\ | ng | |
-| ig\ | | |
-| ur\ | | |
-| e_t\| | |
-| y\ | | |
-| pe\ | | |
-+-----+----+---------------------------+
-| up\ | js\| TBD |
-| da\ | on | |
-| te\ | | |
-| d_p\| | |
-| o\ | | |
-| li\ | | |
-| ci\ | | |
-| es | | |
-+-----+----+---------------------------+
-| up\ | js\| complete |
-| da\ | on | generate\ |
-| te\ | | d |
-| d_a\| | app_conf\ |
-| p\ | | ig, |
-| pl\ | | not |
-| _c\ | | fully-re\ |
-| on\ | | solved, |
-| fi\ | | but |
-| g | | ``policy-enabled`` |
-| | | paramete\ |
-| | | rs |
-| | | have |
-| | | been |
-| | | updated. |
-| | | In order |
-| | | to get |
-| | | the |
-| | | complete |
-| | | updated |
-| | | app_conf\ |
-| | | ig, |
-| | | the |
-| | | componen\ |
-| | | t |
-| | | would |
-| | | have to |
-| | | call |
-| | | ``config-binding-service``|
-| | | . |
-+-----+----+---------------------------+
++--------------+--------------+----------------------------------------+
+| Name | Type | Description |
++==============+==============+========================================+
+| reconfigure_ | string | “policy” |
+| type | | |
++--------------+--------------+----------------------------------------+
+| updated_poli | json | TBD |
+| cies | | |
++--------------+--------------+----------------------------------------+
+| updated_appl | json | complete generated app_config, not |
+| _config | | fully-resolved, but ``policy-enabled`` |
+| | | parameters have been updated. In order |
+| | | to get the complete updated |
+| | | app_config, the component would have |
+| | | to call ``config-binding-service``. |
++--------------+--------------+----------------------------------------+
Docker Component Spec - Complete Example
----------------------------------------
@@ -390,7 +288,7 @@ Docker Component Spec - Complete Example {
"self": {
"version": "1.0.0",
- "name": "asimov.component.kpi_anomaly",
+ "name": "yourapp.component.kpi_anomaly",
"description": "Classifies VNF KPI data as anomalous",
"component_type": "docker"
},
@@ -402,7 +300,7 @@ Docker Component Spec - Complete Example "type": "http"
}],
"publishes": [{
- "format": "asimov.format.integerClassification",
+ "format": "yourapp.format.integerClassification",
"version": "1.0.0",
"config_key": "prediction",
"type": "http"
@@ -427,7 +325,7 @@ Docker Component Spec - Complete Example "version": "1.0.0"
},
"response": {
- "format": "asimov.format.integerClassification",
+ "format": "yourapp.format.integerClassification",
"version": "1.0.0"
}
}]
@@ -448,7 +346,7 @@ Docker Component Spec - Complete Example }
},
"artifacts": [{
- "uri": "fake.nexus.com/dcae/kpi_anomaly:1.0.0",
+ "uri": "fake.nexus.att.com/dcae/kpi_anomaly:1.0.0",
"type": "docker image"
}]
}
diff --git a/docs/sections/components/component-specification/generated-configuration.rst b/docs/sections/components/component-specification/generated-configuration.rst deleted file mode 100755 index ba5ae4a0..00000000 --- a/docs/sections/components/component-specification/generated-configuration.rst +++ /dev/null @@ -1,114 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License.
-.. http://creativecommons.org/licenses/by/4.0
-
-.. _generated-configuration:
-
-Generated configuration
-=======================
-
-The DCAE platform relies on the component specification to generate the
-component’s application configuration JSON at deployment time. The
-component developer should expect to use this configuration JSON in
-their application to provision themselves.
-
-Pro-tip: As you build your component specification, you can use the
-:any:`dcae-cli dev command <walkthrough-dev>` to view what
-the resulting application configuration will look like.
-
-Streams and services
---------------------
-
-For both Docker and CDAP, when your component is deployed, any
-``streams`` and ``services/calls`` you specified will be injected into
-your configuration under the following well known structure. Your
-component is required to parse this information if you have any
-connectivity to DMaaP or are calling another DCAE component.
-
-More details about the DMaaP connection objects are found
-:doc:`here <../dcae-cli/dmaap-connection-objects>`.
-
-This is best served with an example.
-
-The following component spec snippet (from String Matching):
-
-::
-
- "streams":{
- "subscribes": [{
- "format": "VES_specification",
- "version": "4.27.2",
- "type": "message_router",
- "config_key" : "mr_input"
- }],
- "publishes": [{
- "format": "VES_specification",
- "version": "4.27.2",
- "config_key": "mr_output",
- "type": "message_router"
- }]
- },
- "services":{
- "calls": [{
- "config_key" : "aai_broker_handle",
- "verb": "GET",
- "request": {
- "format": "get_with_query_params",
- "version": "1.0.0"
- },
- "response": {
- "format": "aai_broker_response",
- "version": "3.0.0"
- }
- }],
- "provides": []
- },
-
-Will turn into the following top level keys in your configuration (for
-CDAP, this will be under AppConfig)
-
-::
-
- "streams_publishes":{
- "mr_output":{ // notice the config key above
- "aaf_password":"XXX",
- "type":"message_router",
- "dmaap_info":{
- "client_role": null,
- "client_id": null,
- "location": null,
- "topic_url":"XXX"
- },
- "aaf_username":"XXX"
- }
- },
- "streams_subscribes":{
- "mr_input":{ // notice the config key above
- "aaf_password":"XXX",
- "type":"message_router",
- "dmaap_info":{
- "client_role": null,
- "client_id": null,
- "location": null,
- "topic_url":"XXX"
- },
- "aaf_username":"XXX"
- }
- },
- "services_calls":{
- "aai_broker_handle":[ // notice the config key above
- "SOME_IP:32768" // based on deployment time, just an example
- ]
- }
-
-These keys will always be populated regardless of whether they are
-empty. So the minimal you will get, in the case of a component that
-provides an HTTP service and does not call any services and has no
-streams, is:
-
-::
-
- "streams_publishes":{},
- "streams_subscribes":{},
- "services_calls":{}
-
-Thus your component should expect these well-known top level keys.
diff --git a/docs/sections/components/component-specification/start-here.rst b/docs/sections/components/component-specification/start-here.rst deleted file mode 100755 index d3f5a12f..00000000 --- a/docs/sections/components/component-specification/start-here.rst +++ /dev/null @@ -1 +0,0 @@ -
diff --git a/docs/sections/components/component-type-cdap.rst b/docs/sections/components/component-type-cdap.rst index 0f5150fd..f2dce6c5 100755 --- a/docs/sections/components/component-type-cdap.rst +++ b/docs/sections/components/component-type-cdap.rst @@ -65,10 +65,7 @@ Future DMaaP abstraction Shown below is our *vision* for how DMaaP is abstracted from component
developers:
-.. figure:: ../images/dmdvision.png
- :alt: Screenshot
-
- Screenshot
+.. figure:: ./images/dmdvision.png
Today, this is a vision; it is not in place. Today, each CDAP app is
built with built in assumptions about where they are getting their data
@@ -84,7 +81,7 @@ of by the platform. They should be spending their time on the problem at hand—the analytic.
This also allows each CDAP application to have a standard set of
-interfaces: HTTP and HDFS: |Screenshot|
+interfaces: HTTP and HDFS:
-.. |Screenshot| image:: ../images/io.png
+.. figure:: ./images/io.png
diff --git a/docs/sections/components/component-type-docker.rst b/docs/sections/components/component-type-docker.rst index 449e58a8..bb6a5a70 100755 --- a/docs/sections/components/component-type-docker.rst +++ b/docs/sections/components/component-type-docker.rst @@ -13,36 +13,41 @@ Component developers are required to provide artifacts for the platform to be able to deploy your component including:
- :any:`Component specification <docker-specification>`
-- :doc:`Data formats <data-formats>`
-- :any:`Auxiliary component specification <docker-auxiliary-details>`
-- :any:`Docker image <docker-on-the-platform>`
+- :any:`One or more Data Formats <data-formats>` \*unless they already exist
+- `Docker image <#docker-on-the-platform>`__
In addition, components will have to be enhanced to be compliant with
the DCAE platform in order to correctly be deployed and be managed. This
page will discuss the changes which are grouped into the following
categories:
-- :any:`Service registration <service-registration>`
-- :any:`Configuration management on the new platform <docker-configuration>`
-- :any:`Operational concerns <operational-concerns>`
+- `Service Registration <#service-registration>`__
+- `Configuration management on the new platform <#configuration-management>`__
+- `Docker on the Platform <#docker-on-the-platform>`__
+- `Operational concerns <#operational-concerns>`__
+
+Additional considerations are:
+
+- `Policy Reconfiguration <#policy-reconfiguration>`__
To help component developers to make and to test the changes needed to
have components run on the platform, a command-line tool called
:doc:`dcae-cli <dcae-cli/quickstart>` is provided by the platform team.
+(Testing withing the dcae_cli tool is not yet available for Policy).
-.. _service-registration:
-
-Service registration
+Service Registration
--------------------
-Every :doc:`Docker component is registered <architecture/service-discovery>` with the platform’s
+Every :doc:`Docker component is registered <../components/architecture/service-discovery>` with the platform’s
service discovery layer. Docker components are not expected to make the
explicit make registration calls because that is done by through a
platform 3rd party registration service. A couple things are needed from
component developers in order for this registration to occur
successfully:
-1. Docker images must be created from a Dockerfile that has an `EXPOSE <https://docs.docker.com/engine/reference/builder/#/expose>`_ instruction. This applies to components that listen on a port.
+1. Docker images must be created from a Dockerfile that has an
+ `EXPOSE <https://docs.docker.com/engine/reference/builder/#/expose>`__
+ instruction. This applies to components that listen on a port.
2. Component healthcheck details must be provided in the Docker
auxiliary component specification
@@ -53,46 +58,69 @@ Components that listen on a specific port must explicitly declare in their Dockerfile that port using the ``EXPOSE`` instruction before
building the image. Warning! At the current time, you can not expose
multiple ports in your Dockerfile or registration *will not work*
-correctly.
+correctly. Warning! Be sure to choose a port that is available. This may
+vary by environment.
Health check
~~~~~~~~~~~~
-Component developers are required to provide a way for the platform to periodically check the health of their running components. The platform uses Consul to perform these periodic calls. Consul provides different types of `check definitions <https://www.consul.io/docs/agent/checks.html>`_. The details of the definition used by your component is to be provided through the :any:`Docker auxiliary specification <docker-auxiliary-details>`.
-
-.. _docker-configuration:
+Component developers are required to provide a way for the platform to
+periodically check the health of their running components. The platform
+uses Consul to perform these periodic calls. Consul provides different
+types of `check
+definitions <https://www.consul.io/docs/agent/checks.html>`__. The
+details of the definition used by your component is to be provided
+through the :any:`Docker auxiliary specification <docker-auxiliary-details>`.
-Configuration
--------------
+Configuration Management
+------------------------
-The component’s application configuration is generated from the
-submitted component specification into a JSON representation. The DCAE
-platform will provide the configuration JSON by making it available on a
-remote HTTP service. Components are required to pull their configuration
-JSON at application startup. The configuration JSON is stored under the
+All configuration for a component is stored in CONSUL under the
components uniquely generated name which is provided by the environment
-variable ``HOSTNAME``.
+variable ``HOSTNAME`` as well as ``SERVICE_NAME``. It is then made
+available to the component via a remote HTTP service call to CONFIG
+BINDING SERVICE.
+
+The main entry in CONSUL for the component contains its
+``generated application configuration``. This is based on the submitted
+component specification, and consists of the ``interfaces`` (streams and
+services/calls) and ``parameters`` sections. Other entries may exist as
+well, under specific keys, such as :dmaap . Each key represents a
+specific type of information and is also available to the component by
+calling CONFIG BINDING SERVICE. More on this below.
+
+Components are required to pull their
+``generated application configuration`` at application startup. The
+component must provide an initialization script that retrieves the
+application configuration and reference that script in its Dockerfile.
+Other calls can be made to CONFIG BINDING SERVICE to retrieve DMaaP,
+or Pollicy Reconfiguration (not yet supported).
You can see more details on the generated application configuration
-:any:`here <docker-specification>`.
+:any:`here <dcae-cli-view-the-platform>`
-Config binding service
+.. _config_binding_service:
+
+Config Binding Service
~~~~~~~~~~~~~~~~~~~~~~
The config binding service is a platform HTTP service that is
-responsible for providing clients with a fully resolved configuration
-JSON at runtime. Components should make an HTTP GET on:
+responsible for providing clients with its fully resolve configuration
+JSON at startup, and also other configurations objects
+when requested.
+
+At runtime, components should make an HTTP GET on:
::
<config binding service hostname>:<port>/service_component/NAME
-For Docker components, NAME should be set to ``HOSTNAME``, which was
-provided as an ENV variable inside of your container.
+For Docker components, NAME should be set to ``HOSTNAME``, which is
+provided as an ENV variable to the container.
The binding service integrates with the streams and services section of
-your component specification. For example, if you specify that you call
-a service:
+the component specification. For example, if you specify that you call a
+service:
::
@@ -112,8 +140,8 @@ a service: }
Then the config binding service will find all available IP addresses of
-services meeting your needs, and provide them to you under your
-``config_key``:
+services meeting the containers needs, and provide them to the container
+under your ``config_key``:
::
@@ -130,15 +158,111 @@ variable, ``CONFIG_BINDING_SERVICE``, and you will need to query Consul’s service discovery to get
``<config binding service hostname>:<port>``.
-Policy Reconfiguration
-~~~~~~~~~~~~~~~~~~~~~~
+Generated Application Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-*(Draft and subject to change)*
+The DCAE platform uses the component specification to generate the
+component’s application configuration provided at deployment time. The
+component developer should expect to use this configuration JSON in the
+component.
+
+Pro-tip: As you build the component specification, use the :any:`dcae-cli dev command <dcae-cli-view-the-platform>`
+to see what the resulting application configuration will look like.
+
+For both Docker and CDAP, when the component is deployed, any
+``streams`` and ``services/calls`` specified, will be injected into the
+configuration under the following well known structure, along with all
+``parameters``. (``services/provides`` is not passed in to the
+application config). Your component is required to parse this
+information if it has any DMaaP connections or interfaces with another
+DCAE component.
+
+This is best served by an example.
+
+The following component spec snippet (from String Matching):
+
+::
+
+ "streams":{
+ "subscribes": [{
+ "format": "VES_specification",
+ "version": "4.27.2",
+ "type": "message_router",
+ "config_key" : "mr_input"
+ }],
+ "publishes": [{
+ "format": "VES_specification",
+ "version": "4.27.2",
+ "config_key": "mr_output",
+ "type": "message_router"
+ }]
+ },
+ "services":{
+ "calls": [{
+ "config_key" : "aai_broker_handle",
+ "verb": "GET",
+ "request": {
+ "format": "get_with_query_params",
+ "version": "1.0.0"
+ },
+ "response": {
+ "format": "aai_broker_response",
+ "version": "3.0.0"
+ }
+ }],
+ "provides": []
+ },
+
+Will result in the following top level keys in the configuration (for
+CDAP, this will be under AppConfig)
+
+::
+
+ "streams_publishes":{
+ "mr_output":{ // notice the config key above
+ "aaf_password":"XXX",
+ "type":"message_router",
+ "dmaap_info":{
+ "client_role": null,
+ "client_id": null,
+ "location": null,
+ "topic_url":"https://YOUR_HOST:3905/events/com.att.dcae.dmaap.FTL2.DCAE-CL-EVENT" // just an example
+ },
+ "aaf_username":"XXX"
+ }
+ },
+ "streams_subscribes":{
+ "mr_input":{ // notice the config key above
+ "aaf_password":"XXX",
+ "type":"message_router",
+ "dmaap_info":{
+ "client_role": null,
+ "client_id": null,
+ "location": null,
+ "topic_url":"https://YOUR_HOST:3905/events/com.att.dcae.dmaap.FTL2.TerrysStringMatchingTest" // just an example
+ },
+ "aaf_username":"XXX"
+ }
+ },
+ "services_calls":{
+ "aai_broker_handle":[ // notice the config key above
+ "135.205.226.128:32768" // based on deployment time, just an example
+ ]
+ }
+
+These keys will always be populated whether they are empty or not. So
+the minimum configuration you will get, (in the case of a component that
+provides an HTTP service, doesn’t call any services, and has no streams,
+is:
+
+::
+
+ "streams_publishes":{},
+ "streams_subscribes":{},
+ "services_calls":{}
+
+Thus your component should expect these well-known top level keys.
-Components must provide a way to receive policy reconfiguration, that
-is, configuration parameters that have been updated via the Policy UI.
-The component developer provides a docker script (defined in the :any:`Docker auxiliary specification <docker-specification>`)
-that will be triggered when this occurs.
DMaaP
~~~~~
@@ -147,18 +271,52 @@ Components can be publishers or subscribers to either message router topics or data router feeds. This is defined in the component
specification under the ``streams`` section where you can specify
whether your component is expected to subscribe or to publish to a
-:any:`message router topic <message-router>` or to a :any:`data router feed <data-router>`.
-Given a composition with components that use DMaaP, the platform will
-provision the topic or feed and provide the necessary :any:`connection details <dcae-cli/dmaap-connection-objects>` at runtime for each DMaaP
-dependent component. These connection details will be provided through
-your application’s :any:`generated configuration <component-specification/generated-configuration>`.
+:any:`message router <message-router>`
+topic or to a :any:`data router <data-router>`
+feed. Given a composition with components that use DMaaP, the platform
+will provision the topic or feed and provide the necessary :doc:`connection
+details <./component-specification/dmaap-connection-objects>`
+at runtime for each DMaaP dependent component. These connection details
+are provided through your application’s generated configuration.
In order to test DMaaP connections in onboarding, the developer
(currently) must provision all test topics and feeds manually and
-provide the :any:`dcae-cli with the connection details <dmaap-testing>` when deploying your
+provide the :any:`dcae-cli with the connection details <dcae-cli-walkthrough-dmaap-testing>` when deploying your
application.
-.. _docker-on-the-platform:
+Even thought the DMaaP connection information is included in the
+generated application configuration, it may be obtained by doing a call
+as in this example:
+
+::
+
+ curl http://<config binding service>:<port>/dmaap/jm416b.d345ada1-cc31-4121-a741-9007b9f64808.1-0-1.dcae-collectors-cli-pm
+
+This would return the following:
+
+::
+
+ {"cli_gamma_cisco_pm_config_stat":
+ {
+ "publish_url": "https://YOUR_HOST/publish/1362",
+ "username": "mtl5-0",
+ "log_url": null,
+ "location": "mtl5-0",
+ "password": "i5qji048hdm2e38f0bg872tnqd",
+ "publisher_id": "1234"
+ }
+ }
+
+Policy Reconfiguration
+~~~~~~~~~~~~~~~~~~~~~~
+
+*(not yet supported)*
+
+Components must provide a way to receive policy reconfiguration, that
+is, configuration parameters that have been updated via the Policy UI.
+The component developer provides a docker script (defined in the :any:`Docker
+auxiliary specification <docker-auxiliary-details>`)
+that will be triggered when this occurs.
Docker on the platform
----------------------
@@ -170,54 +328,73 @@ Docker images must be pushed to the environment specific Nexus repository. This requires tagging your build with the full name of you
image which includes the Nexus repository name.
-Use the Docker command-line to `tag <https://docs.docker.com/engine/reference/commandline/tag/>`_ your Docker image where the *target image* must contain the registry host name and port.
+Use the Docker command-line to
+`tag <https://docs.docker.com/engine/reference/commandline/tag/>`__ your
+Docker image where the *target image* must contain the registry host
+name and port.
-For example, an application called laika has been tagged for an example
+For example, an application called yourapp has been tagged for an example
Nexus registry:
::
- $ docker images
- REPOSITORY TAG IMAGE ID CREATED SIZE
- YOUR_NEXUS_DOCKER_REGISTRY/laika 0.4.0 154cc382df61 7 weeks ago 710.5 MB
- laika 0.4.0 154cc382df61 7 weeks ago 710.5 MB
+ YOUR_NEXUS_DOCKER_REGISTRY/yourapp 0.4.0 154cc382df61 7 weeks ago 710.5 MB
+ yourapp 0.4.0 154cc382df61 7 weeks ago 710.5 MB
-Note, the Docker registry that is used may require a login to
-authenticate.
+The solutioning evironment’s Nexus host for the Docker registry is
+``YOUR_NEXSUS_HOST:18443``. You must run
+``docker login YOUR_NEXSUS_HOST:18443`` to access the registry.
+Please contact the DCAE platform team to provide you with the
+credentials.
::
- docker login YOUR_NEXUS_DOCKER_REGISTRY
+ docker login YOUR_NEXSUS_HOST:18443
Tag your image:
::
- docker tag laika:0.4.0 YOUR_NEXUS_DOCKER_REGISTRY/laika:0.4.0
+ docker tag yourapp:0.4.0 YOUR_NEXSUS_HOST:18443/dcae-platform/yourapp:0.4.0
Or build and tag:
::
- docker build -t YOUR_NEXUS_DOCKER_REGISTRY/laika:0.4.0 .
+ docker build -t YOUR_NEXSUS_HOST:18443/dcae-platform/yourapp:0.4.0 .
-After tagging, upload your image to the remote registry using the Docker `push command <https://docs.docker.com/engine/reference/commandline/push/>`_ .
-Note that the registry may require a login. Use the Docker `login command <https://docs.docker.com/engine/reference/commandline/login/>`_
+After tagging, upload your image to the remote registry using the Docker
+`push
+command <https://docs.docker.com/engine/reference/commandline/push/>`__.
+Note that the registry may require a login. Use the Docker `login
+command <https://docs.docker.com/engine/reference/commandline/login/>`__
before pushing in that case.
::
- docker push YOUR_NEXUS_DOCKER_REGISTRY/laika:0.4.0
+ docker push YOUR_NEXSUS_HOST:18443/dcae-platform/yourapp:0.4.0
+
+*NOTE* Replace ``dcae-platform`` with the group directory that is
+applicable to your image. Replace ``yourapp`` with your application’s
+name. Replace the ``0.4.0`` version with your application’s version.
-*NOTE* Replace ``laika`` with your application’s name. Replace the
-``0.4.0`` version with your application’s version.
+Dockerfile
+~~~~~~~~~~
+
+The Dockerfile must contain the name of the container’s initialization
+script. This will be called when the container is deployed, and must
+call Config Binding Service as described in `Config Binding
+Service <#config-binding-service>`__
+
+.. _dcae-cli-docker-ports:
Ports
~~~~~
-On the DCAE platform, Docker components are run with the ``--publish-all`` or ``-P`` argument. This means the Docker container
+On the DCAE platform, Docker components are run with the
+``--publish-all`` or ``-P`` argument. This means the Docker container
for your component will be listening on a random port and that random
-port will be mapped to the port :any:`you exposed <service-registration>`.
+port will be mapped to the port `you exposed <#service-registration>`__.
Envs
~~~~
@@ -225,58 +402,24 @@ Envs The platform provides a set of environment variables into each Docker
container:
-+----------------------------+----+----------+
-| Na\ | Ty\| Descript\|
-| me | pe | ion |
-+============================+====+==========+
-| ``HOSTNAME`` | st\| Unique |
-| | ri\| name of |
-| | ng | the |
-| | | componen\|
-| | | t |
-| | | instance |
-| | | that is |
-| | | generate\|
-| | | d |
-+----------------------------+----+----------+
-| ``CONSOL_HOST`` | st\| Hostname |
-| | ri\| of the |
-| | ng | platform\|
-| | | ’s |
-| | | Consul |
-| | | instance |
-| | | |
-| | | |
-+----------------------------+----+----------+
-| ``CONFIG_BINDING_SERVICE`` | st\| Hostname |
-| | ri\| of the |
-| | ng | platform\|
-| | | ’s |
-| | | config |
-| | | binding |
-| | | service |
-| | | instance |
-| | | |
-| | | |
-| | | |
-| | | |
-| | | |
-+----------------------------+----+----------+
-| ``DOCKER_HOSTS`` | st\| Host of |
-| | ri\| the |
-| | ng | target |
-| | | platform |
-| | | Docker |
-| | | host to |
-| | | run the |
-| | | containe\|
-| | | r |
-| | | on |
-+----------------------------+----+----------+
-
-.. _operational-concerns:
-
-Operational concerns
++--------------+--------------+----------------------------------------+
+| Name | Type | Description |
++==============+==============+========================================+
+| ``HOSTNAME`` | string | Unique name of the component instance |
+| | | that is generated |
++--------------+--------------+----------------------------------------+
+| ``CONSUL_HOS | string | Hostname of the platform’s Consul |
+| T`` | | instance |
++--------------+--------------+----------------------------------------+
+| ``CONFIG_BIN | string | Hostname of the platform’s config |
+| DING_SERVICE | | binding service instance |
+| `` | | |
++--------------+--------------+----------------------------------------+
+| ``DOCKER_HOS | string | Host of the target platform Docker |
+| T`` | | host to run the container on |
++--------------+--------------+----------------------------------------+
+
+Operational Concerns
--------------------
Logging
@@ -291,29 +434,32 @@ The requirement is that applications must write to stdout and/or stderr. To use the ``docker logs`` command for your deployed running Docker
container,
-1. You must have Docker installed on your local machine
-2. Have the generated name of your component. This is generated for you
+- You must have Docker installed on your local machine
+- Have the generated name of your component. This is generated for you
when you execute ``dcae_cli component dev`` or
``dcae_cli component run``.
-3. Find the target Docker host using the ``dcae_cli profiles show``
+- Find the target Docker host using the ``dcae_cli profiles show``
command:
::
- $ dcae_cli profiles show YOUR_PROFILE_NAME
+ $ dcae_cli profiles show solutioning
{
- ...
+ "cdap_broker": "cdap_broker",
+ "config_binding_service": "config_binding_service",
+ "consul_host": "YOUR_CONSUL_HOST",
"docker_host": "YOUR_DOCKER_HOST:2376"
}
-4. Set your Docker client to point to the target Docker host:
+- Set your Docker client to point to the target Docker host:
::
$ export DOCKER_HOST="tcp://YOUR_DOCKER_HOST:2376"
-5. Use the ``docker logs`` command:
+- Use the ``docker logs`` command:
::
$ docker logs <generated component name>
+
diff --git a/docs/sections/components/data-formats.rst b/docs/sections/components/data-formats.rst index ecb019c2..bf2ffbf2 100755 --- a/docs/sections/components/data-formats.rst +++ b/docs/sections/components/data-formats.rst @@ -3,258 +3,65 @@ .. _data-formats:
+
Data Formats
============
-| Because the DCAE designer composes your component with others at
- service design time, in most cases you do not know what specific
- component(s) your component will send data to during runtime. Thus, it
- is vital that DCAE has a language of describing the data passed
- between components, so that it is known which components are
- composable with others. Data formats are descriptions of data—they are
- the data contract between your component and other components. You
- need to describe the available outputs and assumed inputs of your
- components as data formats. These data descriptions are onboarded into
- ASDC, and each receives a UUID. If component X outputs data format
- DF-Y, and another component Z specifies DF-Y as their input data
- format, then X is said to be *composable* with that component. The
- data formats are referenced in the component specifications by the
- data format’s id and version.
-| The vision is to have a repository of shared data formats that
- developers and teams can re-use and also provide them the means to
- extend and create new custom data formats.
-
-.. _dataformat_metadata:
+Data formats are descriptions of data; they are the data contract
+between your component and other components. When the components are
+‘composed’ into services in the SDC tool, they can only be matched with
+components that have compatible data formats. Data formats will be
+onboarded to SDC and assigned a UUID at that time. This UUID is then
+used to ensure compatibility amoung components. (If component X outputs
+data format ‘DF-Y’, and another component Z specifies ‘DF-Y’ as its
+input data format, then X is said to be ``composable`` with component
+Z).
+
+Since data formats will be shared across components, the onboarding
+catalog should be checked first to see if the desired data format is
+available before creating one. The vision is to have a repository of
+shared data formats that developers and teams can re-use and also
+provide them the means to extend and create new custom data formats. A
+data format is referenced by its data format id and version number.
+
+JSON schema
+-----------
+
+ The data format specification is represented (and validated) against
+ this `Data Format json schema <https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;h=66aa2ab77449e3cafc6afb5c959c5eb793ad86c1;hb=HEAD>`__
+ and described below:
Meta Schema Definition
-----------------------
+~~~~~~~~~~~~~~~~~~~~~~
The “Meta Schema” implementation defines how data format JSON schemas
can be written to define user input. It is itself a JSON schema (thus it
is a “meta schema”). It requires the name of the data format entry, the
data format entry version and allows a description under “self” object.
The meta schema version must be specified as the value of the
-“dataformatversion” key. Then the input schema itself is described.
-There are four types of schema descriptions objects - jsonschema for
-inline standard JSON Schema definitions of JSON inputs, delimitedschema
-for delimited data input using a defined JSON description, unstructured
-for unstructured text, and reference that allows a pointer to another
-artifact for a schema. The reference allows for XML schema, but can be
-used as a pointer to JSON, Delimited Format, and Unstructured schemas as
-well.
-
-The current Meta Schema implementation is defined below:
-
-::
-
- {
- "$schema": "http://json-schema.org/draft-04/schema#",
- "title": "Data format specification schema Version 1.0",
- "type": "object",
- "oneOf": [{
- "properties": {
- "self": {
- "$ref": "#/definitions/self"
- },
- "dataformatversion": {
- "$ref": "#/definitions/dataformatversion"
- },
- "reference": {
-
- "type": "object",
- "description": "A reference to an external schema - name/version is used to access the artifact",
- "properties": {
- "name": {
- "$ref": "#/definitions/name"
- },
- "version": {
- "$ref": "#/definitions/version"
- },
- "format": {
- "$ref": "#/definitions/format"
- }
- },
- "required": [
- "name",
- "version",
- "format"
- ],
- "additionalProperties": false
- }
- },
- "required": ["self", "dataformatversion", "reference"],
- "additionalProperties": false
- }, {
- "properties": {
- "self": {
- "$ref": "#/definitions/self"
- },
- "dataformatversion": {
- "$ref": "#/definitions/dataformatversion"
- },
- "jsonschema": {
- "$schema": "http://json-schema.org/draft-04/schema#",
- "description": "The actual JSON schema for this data format"
- }
-
- },
- "required": ["self", "dataformatversion", "jsonschema"],
- "additionalProperties": false
- }, {
- "properties": {
- "self": {
- "$ref": "#/definitions/self"
- },
- "dataformatversion": {
- "$ref": "#/definitions/dataformatversion"
- },
- "delimitedschema": {
- "type": "object",
- "description": "A JSON schema for delimited files",
- "properties": {
- "delimiter": {
- "enum": [",", "|", "\t", ";"]
- },
- "fields": {
- "type": "array",
- "description": "Array of field descriptions",
- "items": {
- "$ref": "#/definitions/field"
- }
- }
- },
- "additionalProperties": false
- }
- },
- "required": ["self", "dataformatversion", "delimitedschema"],
- "additionalProperties": false
- }, {
- "properties": {
- "self": {
- "$ref": "#/definitions/self"
- },
- "dataformatversion": {
- "$ref": "#/definitions/dataformatversion"
- },
- "unstructured": {
- "type": "object",
- "description": "A JSON schema for unstructured text",
- "properties": {
- "encoding": {
- "type": "string",
- "enum": ["ASCII", "UTF-8", "UTF-16", "UTF-32"]
- }
- },
- "additionalProperties": false
-
- }
- },
- "required": ["self", "dataformatversion", "unstructured"],
- "additionalProperties": false
- }],
- "definitions": {
- "name": {
- "type": "string"
- },
- "version": {
- "type": "string",
- "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$"
- },
- "self": {
- "description": "Identifying Information for the Data Format - name/version can be used to access the artifact",
- "type": "object",
- "properties": {
- "name": {
- "$ref": "#/definitions/name"
- },
- "version": {
- "$ref": "#/definitions/version"
- },
- "description": {
- "type": "string"
- }
- },
- "required": [
- "name",
- "version"
- ],
- "additionalProperties": false
- },
- "format": {
- "description": "Reference schema type",
- "type": "string",
- "enum": [
- "JSON",
- "Delimited Format",
- "XML",
- "Unstructured"
- ]
- },
- "field": {
- "description": "A field definition for the delimited schema",
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "fieldtype": {
- "description": "the field type - from the XML schema types",
- "type": "string",
- "enum": ["string", "boolean",
- "decimal", "float", "double",
- "duration", "dateTime", "time",
- "date", "gYearMonth", "gYear",
- "gMonthDay", "gDay", "gMonth",
- "hexBinary", "base64Binary",
- "anyURI", "QName", "NOTATION",
- "normalizedString", "token",
- "language", "IDREFS", "ENTITIES",
- "NMTOKEN", "NMTOKENS", "Name",
- "NCName", "ID", "IDREF", "ENTITY",
- "integer", "nonPositiveInteger",
- "negativeInteger", "long", "int",
- "short", "byte",
- "nonNegativeInteger", "unsignedLong",
- "unsignedInt", "unsignedShort",
- "unsignedByte", "positiveInteger"
-
- ]
- },
- "fieldPattern": {
- "description": "Regular expression that defines the field format",
- "type": "integer"
- },
- "fieldMaxLength": {
- "description": "The maximum length of the field",
- "type": "integer"
- },
- "fieldMinLength": {
- "description": "The minimum length of the field",
- "type": "integer"
- },
- "fieldMinimum": {
- "description": "The minimum numeric value of the field",
- "type": "integer"
- },
- "fieldMaximum": {
- "description": "The maximum numeric value of the field",
- "type": "integer"
- }
- },
- "additionalProperties": false
- },
- "dataformatversion": {
- "type": "string",
- "enum": ["1.0.0"]
- }
- }
- }
-
-Examples
------------
+“dataformatversion” key. Then the input schema itself is described as
+one of the four types listed below:
+
++------------------+---------------------------------------------------+
+| Type | Description |
++==================+===================================================+
+| jsonschema | inline standard JSON Schema definitions of JSON |
+| | inputs |
++------------------+---------------------------------------------------+
+| delimitedschema | delimited data input using a JSON description and |
+| | defined delimiter |
++------------------+---------------------------------------------------+
+| unstructured | unstructured text, and reference that allows a |
+| | pointer to another artifact for a schema. |
++------------------+---------------------------------------------------+
+| reference | allows for XML and Protocol Buffers schema, |
+| | but can be used to reference other JSON, |
+| | delimitedschema and unstructured schemas as well. |
++------------------+---------------------------------------------------+
+
+
+Example Schemas
+---------------
By reference example - Common Event Format
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -416,3 +223,16 @@ An example of a delimited schema }]
}
}
+
+Note: The referenced data format (in this case, a schema named “Common
+Event Format” with version of “25.0.0”) must already exist in the
+onboarding catalog.
+
+Working with Data Formats
+-------------------------
+
+Data Formats can be added to the onboarding catalog (which first
+validates them) by using the :doc:`dcae_cli Tool <dcae-cli/quickstart/>`.
+Here you can also list all of your data formats, show the contents of a
+data format, publish your data format, and even generate a data format
+from a input sample file. For a list of these capabilities, see :any:`Data Format Commands <dcae_cli_data_format>`.
diff --git a/docs/sections/components/dcae-cli/commands.rst b/docs/sections/components/dcae-cli/commands.rst new file mode 100644 index 00000000..d34ba43d --- /dev/null +++ b/docs/sections/components/dcae-cli/commands.rst @@ -0,0 +1,590 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+
+.. _dcae_cli_commands:
+
+dcae_cli Commands
+=================
+
+The dcae_cli tool has four command groups. Each has several
+sub-commands.
+
+``catalog``
+-----------
+
+The ``catalog`` command lists and shows resources (components and
+data_formats) in the ‘onboarding’ catalog (regardless of the owner). A
+resource can have a status of ``staged`` or ``published``. By default,
+only ``published`` resources are displayed. To see ``staged`` resources,
+add the –expanded argument.
+
++-----------------------------------------------+-----------------------+
+| Catalog Status | Meaning |
++===============================================+=======================+
+| staged | resource has be added |
+| | (and validated), but |
+| | is under development |
++-----------------------------------------------+-----------------------+
+| staged data_formats can only be referenced in |
+| their owners component specs |
++-----------------------------------------------+-----------------------+
+| staged components can only be deployed by |
+| their owners |
++-----------------------------------------------+-----------------------+
+| published | resource has been |
+| | tested and can be |
+| | shared |
++-----------------------------------------------+-----------------------+
+| published data_formats can be used in |
+| anyone’s component spec |
++-----------------------------------------------+-----------------------+
+| published components and be deployed by |
+| anyone |
++-----------------------------------------------+-----------------------+
+
+::
+
+ $ dcae_cli catalog --help
+ Usage: dcae_cli catalog [OPTIONS] COMMAND [ARGS]...
+
+ Options:
+ --help Show this message and exit.
+
+ Commands:
+ list Lists resources in the onboarding catalog
+ show Provides more information about resource
+
+List onboarding catalog contents
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ $ dcae_cli catalog list
+ Components:
+ +--------------------------------+---------+--------+---------------------------------------------------------------------+--------+-----------+------------+
+ | Name | Version | Type | Description | Owner | Status | Published |
+ +--------------------------------+---------+--------+---------------------------------------------------------------------+--------+-----------+------------+
+ | DcaeSyslogCollector | 2.0.0 | docker | DCAE Control Plane Syslog Collector | sh1986 | published | 2017-08-04 |
+ | cdap.dmaap.spec.example | 0.2.0 | cdap | dmaap spec example. Not a functioning application, only for showing | tc677g | published | 2017-07-24 |
+ | | | | how to pub/sub dmaap. Pretend this is like MAP with VES in and ou.. | | | |
+ | cdap.event.proc.enrich.app | 1.0.3 | cdap | CDAP Event Processing Enrich application | an4828 | published | 2017-09-20 |
+ | cdap.event.proc.map.app | 1.0.3 | cdap | CDAP Event Processing Map application | an4828 | published | 2017-09-20 |
+
+ ...
+
+ Data formats:
+ +--------------------------------------------+---------+-----------------------------------------------------------------------+--------+-----------+------------+
+ | Name | Version | Description | Owner | Status | Published |
+ +--------------------------------------------+---------+-----------------------------------------------------------------------+--------+-----------+------------+
+ | FOI_PM_VHSS_data_format | 1.0.0 | CSV pipe delimited data format for VHSS PM files | sr229c | published | 2017-09-05 |
+ | Map_input | 1.0.0 | The input format for Mapper, that in 1707 is the UCSNMP Collector | an4828 | published | 2017-07-18 |
+ | | | output format, but will support more formats later | | | |
+ | Syslog Collector Parsed Json Message | 1.0.0 | Post processed/parsed collected syslog message | sh1986 | published | 2017-08-04 |
+ | Syslog Collector Syslog Message Input | 1.0.0 | The input message for the DCAE syslog collector is free/unstructured | sh1986 | published | 2017-08-04 |
+ | | | text | | | |
+ | myapp Alert Definition | 1.0.0 | The format of the output event from myapp | an4828 | published | 2017-08-10 |
+ | VES_specification | 5.28.4 | VES spec for 5.4 | vv770d | published | 2017-09-19 |
+
+ ...
+
+Show the contents of an item in the catalog
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ $ dcae_cli catalog show FOI_PM_VHSS_data_format
+
+ Data format
+ -----------
+ {
+ "dataformatversion": "1.0.0",
+ "delimitedschema": {
+ "delimiter": "|",
+ "fields": [
+ {
+ "description": "System ID",
+ "fieldtype": "string",
+ "name": "SYSTEM"
+ },
+ {
+ "description": "Date",
+ "fieldtype": "string",
+ "name": "DATE"
+ },
+ {
+ "description": "Time",
+ "fieldtype": "string",
+ "name": "TIME"
+ },
+
+ ...
+
+.. _dcae_cli_component_commands:
+
+--------------
+
+``component``
+-------------
+
+The ``component`` command is for validating (adding), listing, showing,
+verifying generated configuration, running, undeploying, and publishing
+components that YOU own.
+
+::
+
+ $ dcae_cli component --help
+ Usage: dcae_cli component [OPTIONS] COMMAND [ARGS]...
+
+ Options:
+ --help Show this message and exit.
+
+ Commands:
+ add
+ dev Set up component in development for...
+ list Lists components in the onboarding catalog.
+ publish Pushes COMPONENT to the onboarding catalog
+ run Runs the latest version of COMPONENT.
+ show Provides more information about COMPONENT
+ undeploy Undeploys the latest version of COMPONENT.
+
+--------------
+
+.. _dcae_cli_add_a_component:
+
+Add a Component
+~~~~~~~~~~~~~~~
+
+A component must be added to the onboarding catalog in order to be
+tested by the dcae_cli tool. The process of adding a component also
+validates it’s component specification. In order to add a component, the
+component docker image (or CDAP jar) must exist locally.
+
+Components in the onboarding catalog can be run by others, once they are
+``published.`` ``Published`` components cannot be modified or deleted.
+Rather a new version can be created instead.
+
+Validated component specs are used later to generate Tosca models and
+Cloudify Blueprints for the component, which makes them available for
+use in the SDC Tool for creating services.
+
+::
+
+ $ dcae_cli component add --help
+ Usage: dcae_cli component add [OPTIONS] COMPONENT-SPECIFICATION
+
+ Options:
+ --update Updates a previously added component if it has not been already
+ published
+ --help Show this message and exit.
+
+::
+
+ $ dcae_cli component add component-spec.json
+
+--------------
+
+List Components
+~~~~~~~~~~~~~~~
+
+List components in the onboarding catalog that owned by YOUR userid..
+
+::
+
+ $ dcae_cli component list
+ Active profile: solutioning
+ +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+ | Name | Version | Type | Description | Status | Modified | #Deployed |
+ +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+ | cdap.helloworld.endnode | 0.8.0 | cdap | cdap test component | staged | 2017-05-23 04:14:35.588075 | 0 |
+ | sandbox.platform.yourapp| 0.5.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-05-23 04:07:44.065610 | 0 |
+ +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+
+The fields ``Name``, ``Version``, ``Type``, ``Description`` are
+referenced from the component specification’s ``self`` JSON. Use the
+“–deployed” argument to see more details on deploymed components
+
+--------------
+
+.. _dcae_cli_run_a_component:
+
+Run a Component
+~~~~~~~~~~~~~~~
+
+The ``run`` operation is to be used for running your application in its
+container remotely on the activated environment. Docker containers have
+the additional option to run locally on your development machine. If the
+component uses Dmaap, you can specify the Dmaap Connection Object as
+well. Refer to :doc:`Dmaap Connection Object <../component-specification/dmaap-connection-objects>`.
+
+When you run a component via the dcae_cli Tool, remember the blueprint
+has not been created and is not used for deployment.
+
+In order to run the component, the data formats and component must have
+been added to the onboarding catalog.
+
+**DOCKER NOTE:** Make sure the Docker image has been uploaded to the
+shared registry.
+
+A docker component can be run in either ``attached`` or ``unattached``
+mode. (Default is unattached).
+
++------------------+---------------------------------------------------+
+| Mode | Description |
++==================+===================================================+
+| attached | component is run in the ‘foreground’, container |
+| | logs are streamed to stdout. Ctrl-C is used to |
+| | terminate the dcae_cli session. |
++------------------+---------------------------------------------------+
+| unattached | component is run in the ‘background’, container |
+| | logs are viewed via ``docker logs`` command, |
+| | container runs until undeployed with dcae_cli |
+| | ``undeploy`` command. |
++------------------+---------------------------------------------------+
+
+Run a component in attached mode:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+ $ dcae_cli -v component run --attached sandbox.platform.yourapp:0.5.0
+
+ DCAE.Docker | INFO | Running image 'registry.proto.server.com/dcae-rework/yourapp:0.4.0' as 'user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp'
+ DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | Consul host: yourconsulhost.com
+
+ DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | service name: mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp
+
+ DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | Generated config: {'multiplier': 3}
+
+ DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | * Running on http://0.0.0.0:8080/ (Press CTRL+C to quit)
+
+ DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | 135.205.226.140 - - [24/May/2017 03:37:57] "GET /health HTTP/1.1" 200 -
+
+ DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | 135.205.226.140 - - [24/May/2017 03:38:12] "GET /health HTTP/1.1" 200 -
+
+Hit Ctrl-C to terminate session.
+
+::
+
+ ^CDCAE.Docker | INFO | Stopping container 'user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp' and cleaning up...
+
+Run a component in unattached mode:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+ $ dcae_cli -v component run sandbox.platform.yourapp:0.5.0
+ DCAE.Docker | INFO | Running image 'registry.proto.server.com/dcae-rework/yourapp:0.4.0' as 'user1.4811da0e-08d5-429f-93bf-bf6814924577.0-5-0.sandbox-platform-yourapp'
+ DCAE.Run | INFO | Deployed /user1.4811da0e-08d5-429f-93bf-bf6814924577.0-5-0.sandbox-platform-yourapp
+
+**NOTE** You must undeploy this component when finished testing. This is
+important to conserve resources in the environment.
+
+Run a component that subscribes to Dmaap MR or DR
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+ $ dcae_cli -v component run --attached --dmaap-file $dmaap-connection-file sandbox.platform.yourapp:0.5.0
+
+--------------
+
+Undeploy a Component
+~~~~~~~~~~~~~~~~~~~~
+
+The ``undeploy`` command is used to undeploy any instance of a specified
+component/version that you have deployed. This includes cleaning up the
+configuration.
+
+::
+
+ $ dcae_cli component undeploy sandbox.platform.yourapp:0.5.0
+ DCAE.Undeploy | WARNING | Undeploying components: 1
+ DCAE.Undeploy | WARNING | Undeployed components: 1
+
+--------------
+
+Publish a component
+~~~~~~~~~~~~~~~~~~~
+
+| Once a component has been tested, it should be published in the
+ onboarding catalog using the ``publish`` sub-command .
+| Publishing will change the status of the component (from ``staged`` to
+ ``published``), indicating that it has been tested, and making it
+ accessible for other developers to use.
+
+**Note** Before a component can be published, all data_formats that it
+references must be published.
+
+::
+
+ dcae_cli component publish sandbox.platform.yourapp:0.5.0
+
+--------------
+
+Show a Component
+~~~~~~~~~~~~~~~~
+
+This will print out the contents of a component and is useful to copy a
+component spec.
+
+::
+
+ $ dcae_cli component show
+
+--------------
+
+.. _dcae-cli-run-the-dev-command:
+
+Run the ``dev`` command
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``dev`` command is used as part of a process to see the platform
+generated configuration. It established the environment variables and is
+best explained
+:any:`here <dcae-cli-view-the-platform>`.
+
+::
+
+ $ dcae_cli component dev component-spec.json
+ Ready for component development
+
+--------------
+
+.. _dcae_cli_data_format:
+
+``data_format``
+---------------
+
+The ``data_format`` command is for validating (adding), listing,
+showing, publishing data_formats that YOU own. data_formats can also be
+generated with this command.
+
+::
+
+ $ dcae_cli data_format --help
+ Usage: dcae_cli data_format [OPTIONS] COMMAND [ARGS]...
+
+ Options:
+ --help Show this message and exit.
+
+ Commands:
+ add Tracks a data format file DATA_FORMAT-SPECIFICATION...
+ generate Create schema from a file or directory...
+ list Lists all your data formats
+ publish Publishes data format to make available to others...
+ show Provides more information about FORMAT
+
+--------------
+
+.. _dcae_cli_add_a_data_format:
+
+Add a Data Format
+~~~~~~~~~~~~~~~~~
+
+A data_format must be in the onboarding catalog in order to be
+referenced in the component specification. The process of adding a
+data_format also validates it. Data_formats in the onboarding catalog
+can be run by others, once they are ``published.`` ``Published``
+data_formats cannot be modified or deleted. Rather a new version can be
+created instead.
+
+::
+
+ $ dcae_cli data_format add --help
+ Usage: dcae_cli data_format add [OPTIONS] DATA_FORMAT-SPECIFICATION
+
+ Options:
+ --update Updates a previously added data_format if it has not been already
+ published
+ --help Show this message and exit.
+
+::
+
+ dcae_cli data_format add health.json
+
+--------------
+
+List Data Formats
+~~~~~~~~~~~~~~~~~
+
+Only data_formats owned by YOUR userid will be shown.
+
+::
+
+ $ dcae_cli data_format list
+
+ Data formats for user1
+ +---------------------------------+---------+-------------------------------------------+--------+----------------------------+
+ | Name | Version | Description | Status | Modified |
+ +---------------------------------+---------+-------------------------------------------+--------+----------------------------+
+ | sandbox.platform.yourapp.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-05-23 04:02:38.952799 |
+ +---------------------------------+---------+-------------------------------------------+--------+----------------------------+
+
+The fields ``name``, ``version``, ``description`` are referenced from
+the data format specification’s ``self`` JSON. ``Status`` represents the
+status of the data format in the catalog. See `Publish a Data
+Format <#publish-a-data-format>`__ for more info.
+
+--------------
+
+Show a Data Format
+~~~~~~~~~~~~~~~~~~
+
+This will print out the contents of a data_format and is useful for
+copying a data_format.
+
+::
+
+ $ dcae_cli data_format show
+
+--------------
+
+Publish a Data Format
+~~~~~~~~~~~~~~~~~~~~~
+
+| Once a data_format has been tested (by referencing it in a component
+ spec that has been tested), it should be published in the onboarding
+ catalog using the ``publish`` sub-command .
+| Publishing will change the status of the data_format (from ``staged``
+ to ``published``), indicating that it has been tested, and making it
+ accessible for other developers to use.
+
+::
+
+ $ dcae_cli data_format publish data_format.json
+
+--------------
+
+Generate a Data Format
+~~~~~~~~~~~~~~~~~~~~~~
+
+If you already have a valid input or output file, you can use the
+generate command to create the it’s data format specification.
+
+::
+
+ $ dcae_cli data_format generate name:version file-or-dir-path
+
+--------------
+
+``profiles``
+------------
+
+The\ ``profiles`` command is for creating, deleting, listing, showing,
+activating, and updating (set) profiles. The profile contains
+environment variables used to connect to different environments. This is
+used in the running and deployment of a component using the
+``dcae_cli component run`` or ``dev`` command.
+
+::
+
+ $ dcae_cli profiles --help
+ Usage: dcae_cli profiles [OPTIONS] COMMAND [ARGS]...
+
+ Options:
+ --help Show this message and exit.
+
+ Commands:
+ activate Sets profile NAME as the active profile
+ create Creates a new profile NAME initialized with...
+ delete Deletes profile NAME
+ list Lists available profiles
+ set Updates profile NAME such that KEY=VALUE
+ show Prints the profile dictionary
+
+--------------
+
+List the available profiles
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ $ dcae_cli profiles list
+ * solutioning
+ 1710
+ 1802
+
+The \* identifies the active profile. ``dcae-cli`` is currently
+installed with profiles for the ``solutioning``, ``1710``, and ``1802``
+environments. They are intended for the following:
+
++-----------------------------------+-----------------------------------+
+| Environment | Description |
++===================================+===================================+
+| solutioning | default environment; used for |
+| | initial component developer |
+| | testing with the dcae_cli tool. |
++-----------------------------------+-----------------------------------+
+| 1710 | FTL3 (Functional Testing Lab 3) |
+| | environment, which represents the |
+| | 1710 release. |
++-----------------------------------+-----------------------------------+
+| 1802 | FTL3a (Functional Testing Lab 3a) |
+| | environment, which represents the |
+| | 1802 release. |
++-----------------------------------+-----------------------------------+
+
+--------------
+
+Show the details of a profile
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ $ dcae_cli profiles show solutioning
+ {
+ "cdap_broker": "cdap_broker",
+ "config_binding_service": "config_binding_service",
+ "consul_host": "yourconsulhost.com",
+ "docker_host": "yourdockerhost.com:2376"
+ }
+
+--------------
+
+.. _dcae_cli_activate_profile:
+
+Activate a profile
+~~~~~~~~~~~~~~~~~~
+
+To switch among profiles, use the activate sub-command. A subsequent
+``list`` will reveal the change made.
+
+::
+
+ $ dcae_cli profiles activate test
+
+--------------
+
+Create a new profile
+~~~~~~~~~~~~~~~~~~~~
+
+If you want to work in a different environment using the dcae_cli tool,
+you can make your own profile. (The environment must be a working DCAE
+Platform environment).
+
+::
+
+ $ dcae_cli profiles create new-profile
+
+After creating you would assign the variables with the ``set``
+sub-command. Then activate it to use.
+
+--------------
+
+Set variables in a profile
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ $ dcae_cli profiles set $profile $key $value
+
+--------------
+
+Delete a profile
+~~~~~~~~~~~~~~~~
+
+::
+
+ $ dcae_cli profiles delete new-profile
+
+
diff --git a/docs/sections/components/dcae-cli/dcae-cli-tool.rst b/docs/sections/components/dcae-cli/dcae-cli-tool.rst index 6f564cee..477fc341 100644 --- a/docs/sections/components/dcae-cli/dcae-cli-tool.rst +++ b/docs/sections/components/dcae-cli/dcae-cli-tool.rst @@ -8,6 +8,6 @@ dcae-cli Tool :maxdepth: 1 ./quickstart.rst + ./commands.rst ./walkthrough.rst - ./dmaap-connection-objects.rst diff --git a/docs/sections/components/dcae-cli/dmaap-connection-objects.rst b/docs/sections/components/dcae-cli/dmaap-connection-objects.rst deleted file mode 100755 index cad3b0c9..00000000 --- a/docs/sections/components/dcae-cli/dmaap-connection-objects.rst +++ /dev/null @@ -1,415 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License.
-.. http://creativecommons.org/licenses/by/4.0
-
-DMaaP connection objects
-========================
-
-DMaaP connection objects are JSON objects that:
-
-1. Components should expect at runtime in their application
- configuration and is to be used to connect to the appropriate DMaaP
- feed or topic.
-2. Developers must provide through the command-line argument
- ``--dmaap-file`` to test their component with manually provisioned
- feeds and topics.
-
-This page is a reference to the specific structure that each type of
-DMaaP stream requires.
-
-Note for #1 that components should expect the entire object with all
-properties at runtime where the default will be ``null`` unless
-specified otherwise.
-
-Note for #2 that developers are not required to provide the entire
-object. The required properties will be labeled with “*required as
-input*”.
-
-.. _dmaap-message-router:
-
-Message router
---------------
-
-Publishers and subscribers both have the same JSON object structure.
-Here’s an example:
-
-.. code:: json
-
- {
- "type": "message_router",
- "aaf_username": "some-user",
- "aaf_password": "some-password",
- "dmaap_info": {
- "client_role": "com.dcae.member",
- "client_id": "1500462518108",
- "location": "mtc00",
- "topic_url": "https://we-are-message-router.us:3905/events/some-topic"
- }
- }
-
-
-At the top-level:
-
-+-------------+----+--------------------+
-| Property | Ty\| Descript\ |
-| Name | pe | ion |
-+=============+====+====================+
-| type | st\| *Require\ |
-| | ri\| d \ |
-| | ng | as \ |
-| | | input*. |
-| | | Must be |
-| | | ``message_router`` |
-| | | for |
-| | | message |
-| | | router |
-| | | topics |
-+-------------+----+--------------------+
-| aaf_usernam\| st\| AAF |
-| e | ri\| username |
-| | ng | message |
-| | | router |
-| | | clients |
-| | | use to |
-| | | authenti\ |
-| | | cate |
-| | | with |
-| | | secure |
-| | | topics |
-+-------------+----+--------------------+
-| aaf_passwor\| st\| AAF |
-| d | ri\| password |
-| | ng | message |
-| | | router |
-| | | clients |
-| | | use to |
-| | | authenti\ |
-| | | cate |
-| | | with |
-| | | secure |
-| | | topics |
-+-------------+----+--------------------+
-| dmaap_info | JS\| *Require\ |
-| | ON | d \ |
-| | ob\| as \ |
-| | je\| input*. |
-| | ct | Contains |
-| | | the |
-| | | topic |
-| | | connecti\ |
-| | | on |
-| | | details |
-+-------------+----+--------------------+
-
-The ``dmaap_info`` object contains:
-
-+-------------+----+----------+
-| Property | Ty\| Descript\|
-| Name | pe | ion |
-+=============+====+==========+
-| client_role | st\| AAF |
-| | ri\| client |
-| | ng | role |
-| | | that’s |
-| | | requesti\|
-| | | ng |
-| | | publish |
-| | | or |
-| | | subscrib\|
-| | | e |
-| | | access |
-| | | to the |
-| | | topic |
-+-------------+----+----------+
-| client_id | st\| Client |
-| | ri\| id for |
-| | ng | given |
-| | | AAF |
-| | | client |
-+-------------+----+----------+
-| location | st\| DCAE |
-| | ri\| location |
-| | ng | for the |
-| | | publishe\|
-| | | r |
-| | | or |
-| | | subscrib\|
-| | | er, |
-| | | used to |
-| | | set up |
-| | | routing |
-+-------------+----+----------+
-| topic_url | st\| *Require\|
-| | ri\| d \ |
-| | ng | as \ |
-| | | input*. |
-| | | URL for |
-| | | accessin\|
-| | | g |
-| | | the |
-| | | topic to |
-| | | publish |
-| | | or |
-| | | receive |
-| | | events |
-+-------------+----+----------+
-
-Here’s an example of the minimal JSON that must be provided as an input:
-
-.. code:: json
-
- {
- "type": "message_router",
- "dmaap_info": {
- "topic_url": "https://we-are-message-router.us:3905/events/some-topic"
- }
- }
-
-.. _dmaap-data-router:
-
-Data router
------------
-
-Publisher
-~~~~~~~~~
-
-Here’s an example of what the JSON object connection for data router
-publisher looks like:
-
-.. code:: json
-
- {
- "type": "data_router",
- "dmaap_info": {
- "location": "mtc00",
- "publish_url": "https://we-are-data-router.us/feed/xyz",
- "log_url": "https://we-are-data-router.us/feed/xyz/logs",
- "username": "some-user",
- "password": "some-password",
- "publisher_id": "123456"
- }
- }
-
-At the top-level:
-
-+-------------+----+----------------+
-| Property | Ty\| Descript\ |
-| Name | pe | ion |
-+=============+====+================+
-| type | st\| *Require\ |
-| | ri\| d \ |
-| | ng | as \ |
-| | | input*. |
-| | | Must be |
-| | | ``data_router``|
-| | | for data |
-| | | router |
-| | | feeds |
-+-------------+----+----------------+
-| dmaap_info | JS\| *Require\ |
-| | ON | d \ |
-| | ob\| as \ |
-| | je\| input*. |
-| | ct | Contains |
-| | | the |
-| | | topic |
-| | | connecti\ |
-| | | on |
-| | | details |
-+-------------+----+----------------+
-
-The ``dmaap_info`` object contains:
-
-+-------------+----+----------+
-| Property | Ty\| Descript\|
-| Name | pe | ion |
-+=============+====+==========+
-| location | st\| DCAE |
-| | ri\| location |
-| | ng | for the |
-| | | publishe\|
-| | | r, |
-| | | used to |
-| | | set up |
-| | | routing |
-+-------------+----+----------+
-| publish_url | st\| *Require\|
-| | ri\| d \ |
-| | ng | as \ |
-| | | input*. |
-| | | URL to |
-| | | which |
-| | | the |
-| | | publishe\|
-| | | r |
-| | | makes |
-| | | Data |
-| | | Router |
-| | | publish |
-| | | requests |
-+-------------+----+----------+
-| log_url | st\| URL from |
-| | ri\| which |
-| | ng | log data |
-| | | for the |
-| | | feed can |
-| | | be |
-| | | obtained |
-+-------------+----+----------+
-| username | st\| Username |
-| | ri\| the |
-| | ng | publishe\|
-| | | r |
-| | | uses to |
-| | | authenti\|
-| | | cate |
-| | | to Data |
-| | | Router |
-+-------------+----+----------+
-| password | st\| Password |
-| | ri\| the |
-| | ng | publishe\|
-| | | r |
-| | | uses to |
-| | | authenti\|
-| | | cate |
-| | | to Data |
-| | | Router |
-+-------------+----+----------+
-| publisher_i | st\| Publishe\|
-| d | ri\| r |
-| | ng | id in |
-| | | Data |
-| | | Router |
-+-------------+----+----------+
-
-Here’s an example of the minimal JSON that must be provided as an input:
-
-.. code:: json
-
- {
- "type": "data_router",
- "dmaap_info": {
- "publish_url": "https://we-are-data-router.us/feed/xyz"
- }
- }
-
-Subscriber
-~~~~~~~~~~
-
-Here’s an example of what the JSON object connection for data router
-subscriber looks like:
-
-.. code:: json
-
- {
- "type": "data_router",
- "dmaap_info": {
- "location": "mtc00",
- "delivery_url": "https://my-subscriber-app.dcae:8080/target-path",
- "username": "some-user",
- "password": "some-password",
- "subscriber_id": "789012"
- }
- }
-
-At the top-level:
-
-+-------------+----+----------------+
-| Property | Ty\| Descript\ |
-| Name | pe | ion |
-+=============+====+================+
-| type | st\| *Require\ |
-| | ri\| d |
-| | ng | as \ |
-| | | input*. |
-| | | Must be |
-| | | ``data_router``|
-| | | for data |
-| | | router |
-| | | feeds |
-+-------------+----+----------------+
-| dmaap_info | JS\| *Require\ |
-| | ON | d \ |
-| | ob\| as \ |
-| | je\| input*. |
-| | ct | Contains |
-| | | the |
-| | | topic |
-| | | connecti\ |
-| | | on |
-| | | details |
-+-------------+----+----------------+
-
-The ``dmaap_info`` object contains:
-
-+--------------+----+----------+
-| Property | Ty\| Descript\|
-| Name | pe | ion |
-+==============+====+==========+
-| location | st\| DCAE |
-| | ri\| location |
-| | ng | for the |
-| | | publishe\|
-| | | r, |
-| | | used to |
-| | | set up |
-| | | routing |
-+--------------+----+----------+
-| delivery_ur\ | st\| URL to |
-| l | ri\| which |
-| | ng | the Data |
-| | | Router |
-| | | should |
-| | | deliver |
-| | | files |
-+--------------+----+----------+
-| username | st\| Username |
-| | ri\| Data |
-| | ng | Router |
-| | | uses to |
-| | | authenti\|
-| | | cate |
-| | | to the |
-| | | subscrib\|
-| | | er |
-| | | when |
-| | | deliveri\|
-| | | ng |
-| | | files |
-+--------------+----+----------+
-| password | st\| Password |
-| | ri\| Data |
-| | ng | Router |
-| | | uses to |
-| | | authenti\|
-| | | cate |
-| | | to the |
-| | | subscrib\|
-| | | er |
-| | | when |
-| | | deliveri\|
-| | | ng |
-| | | files |
-+--------------+----+----------+
-| subscriber_i\| st | Subscrib\|
-| d | ri | er |
-| | ng | id in |
-| | | Data |
-| | | Router |
-+--------------+----+----------+
-
-Here’s an example of the minimal JSON that must be provided as an input:
-
-.. code:: json
-
- {
- "type": "data_router",
- "dmaap_info": {
- }
- }
-
-Developers are recommended to use ``username`` and ``password`` since
-this is the recommended security practice.
-
-Note that the dcae-cli will construct the ``delivery_url`` when
-deploying the component since this can only be known at deployment time.
diff --git a/docs/sections/components/dcae-cli/quickstart.rst b/docs/sections/components/dcae-cli/quickstart.rst index 60370330..47b9ac1b 100755 --- a/docs/sections/components/dcae-cli/quickstart.rst +++ b/docs/sections/components/dcae-cli/quickstart.rst @@ -3,86 +3,125 @@ .. _quickstart:
-Quickstart
-==========
+Overview
+========
-The ``dcae-cli`` is a Python command-line tool built to aide component
-developers with the development and testing of their micro-service
-component for the DCAE platform. It will help developers do functional
-and integration testing of their components locally and on remote
-environments as simple as possible.
+The ``dcae-cli`` is a Python command-line tool for component developers.
+With it, the developer can :
-The tool requires the component developers to create a valid component
-specification for their component which is used by the tool. This same
-component specification will be published in the :any:`onboarding catalog <glossary-onboarding-catalog>` at the end of development
-and testing.
+- validate the data formats and component specifications
+- publish the validated data formats and component specifications into
+ the ``onboarding catalog``
+- access the ``onboarding catalog`` to search for existing data formats
+ (for possible reuse) and component specs
+- deploy a component onto a local or remote DCAE platform for
+ functional and pair-wise testing (This is done without Cloudify)
-.. The git repository can be found `here <ONAP%20LINK%20TBD>`__
+The git repository for the dcae_cli tool can be found
+`here <https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git>`__
-Pre-requisite
--------------
+Pre-requisites
+--------------
For Docker
~~~~~~~~~~
-There are two options for development with Docker: developing locally on
-your machine which requires Docker to be installed and developing
-remotely by deploying onto remote infrastructure.
+There are two options for development with Docker:
For local development
^^^^^^^^^^^^^^^^^^^^^
-- You must install `Docker
- engine <https://docs.docker.com/engine/installation/>`__ locally on
+- Install `Docker engine <https://docs.docker.com/engine/installation/>`__ locally on
your machine.
-- You must know the *external ip* of where the Docker engine is
- running. The external ip is needed so that service discovery will
- wire up correctly.
+- Know the *external ip* of where the Docker engine is running. The
+ external ip is needed so that service discovery will connect to it.
- - For OSX users, this means making sure the VirtualBox VM that is
+ - *(For OSX users, this means making sure the VirtualBox VM that is
running your Docker engine has a bridged adapter and getting the
- ip of that adapter.
+ ip of that adapter).*
For remote development
^^^^^^^^^^^^^^^^^^^^^^
-You need access to a remote host with Docker engine installed with
-remote API access. You must have the associated connection information:
-domain name or IP and port (should be either 2375 or 2376). This
-information should be set in :any:`an active profile <setting-profile>`.
+- Have access to a remote host with Docker engine installed and with
+ remote API access.
+- Have the associated connection information:
+
+ - domain name or IP and port (port should be either 2375 or 2376).
+ Use this information to establish an active
+ :any:`profile <dcae_cli_activate_profile>`.
For CDAP
~~~~~~~~
-TBD
+None at this time.
+
+Python, Pip, Virtualenv
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Install python, pip (9.0.1 or higher), and virtualenv if they are not
+installed. Do these when not in a VPN to avoid possible network issues.
+
+::
+
+ sudo apt-get -f install python
+ sudo apt-get -f install python-pip
+ sudo pip install virtualenv
+
+Set up a virtual environment and activate
+
+::
+
+ virtualenv cli_tool
+ source cli_tool/biin/activate
+
+Install dcae_cli
+----------------
+
+::
+
+ pip install onap-dcae-cli
-Install
--------
+Check dcae_cli version
+----------------------
+
+You can verify the version of the dcae-cli with the following command.
+To get the latest version of the dcae_cli tool,
::
- pip install --extra-index-url https://YOUR_NEXUS_PYPI_SERVER/simple dcae-cli
+ $ dcae_cli --version
-To do an upgrade, use the ``--upgrade`` flag.
+Upgrade dcae_cli
+----------------
+
+Periodically, upgrade the dcae_cli to get the latest version
+
+::
+
+ pip install --upgrade onap-dcae-cli
Configuration
-~~~~~~~~~~~~~
+-------------
-When you run the tool for the first time, the tool will create a
-`configuration
+When running the tool for the first time, a `configuration
directory <http://click.pocoo.org/5/api/#click.get_app_dir>`__ and
-generate a configuration file.
+configuration file will be created.
-Configuration is first sourced from an remote server that the platform
-team manages. This is overlaid with configuration details that you will
-be prompted to input particularly your user id.
+The configuration is first sourced from a remote server that is managed
+by the platform team. You will be prompted to enter your ATTUID to
+complete this process.
-``--reinit``
-^^^^^^^^^^^^
+Re-initializing Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Configuration can be re-initialized or reset. There is a ``--reinit``
flag that is to be used to re-initialize your configuration and your
-environment profiles.
+environment profiles. You may be instructed to re-initialize after
+certain updates are made to the dcae_cli tool. When you re-initialize
+the configuration, your configuration will be added to or updated from
+the platform configuration and profiles. No profiles will be deleted via
+the reinit process.
To re-initialize:
@@ -90,10 +129,11 @@ To re-initialize: $ dcae_cli --reinit
-Verify
-~~~~~~
+Verify Installation
+-------------------
-Verify that its installed:
+To Verify that the dcae_cli tool is installed, run the following command
+and look for the output below.
::
@@ -112,11 +152,5 @@ Verify that its installed: data_format
profiles
-Version
--------
-
-You can verify the version of the dcae-cli with the following command:
+Refer to :doc:`dcae_cli Commands <./commands>`.
-::
-
- $ dcae_cli --version
diff --git a/docs/sections/components/dcae-cli/walkthrough.rst b/docs/sections/components/dcae-cli/walkthrough.rst index d33c35fb..559ba3ab 100755 --- a/docs/sections/components/dcae-cli/walkthrough.rst +++ b/docs/sections/components/dcae-cli/walkthrough.rst @@ -6,560 +6,389 @@ Walk-through
============
-The goal of this quickstart is to provide an overview of the
-functionalities of the ``dcae-cli`` and walk you through the
-capabilities:
+This section demonstrates the flow and usage of the dcae_cli tool to
+onboard a typical component to the DCAE platform. The commands are
+explained in more detail in :doc:`dcae_cli Commands <commands>`.
+
+- `Add (and validate) a data format <#add-a-data-format>`__
+- `Add (and validate) the component <#add-the-component>`__
+- `View the platform generated
+ configuration <#view-the-platform-generated-configuration>`__
+- `If needed, Create the dmaap file for Dmaap Testing <#create-the-input-file-for-dmaap-testing>`__
+- `If needed, Create the input file for *Sourced at Deployment* Testing <#create-the-input-file-for-sourced-at-deployment-testing>`__
+- `Run the component <#run-the-component>`__
+- :any:`Undeploy the component <dcae_cli_undeploy_the_component>`
+- :any:`Publish the component and data_format <dcae_cli_publish_the_component_and_data_format>` to let others
+ know its ready for reuse
+- `List the Catalog Contents <#list-the-catalog-contents>`__ to see
+ your published resources
-- `Adding data formats <#adding-data-formats>`__
-- `Adding component <#adding-component>`__
-- `Setting profile <#setting-profile>`__
-- `Development and testing <#development-and-testing>`__
-- `Publishing component <#publishing-component>`__
-- `Shared catalog <#shared-catalog>`__
-.. This walk-through uses example projects: COMMENTED OUT FOR NOW TBD
-..
-.. - `laika <ONAP%20URL%20TBD>`__
-.. - `CDAP examples <ONAP%20URL%20TBD>`__
-
-.. _adding-data-formats:
-
-Adding data formats
--------------------
+--------------
-``data_format`` is the sub-command that is used to execute operations
-that manage `data formats <../data-formats.md>`__.
+Add a Data Format
+-----------------
::
- $ dcae_cli data_format --help
- Usage: dcae_cli data_format [OPTIONS] COMMAND [ARGS]...
-
- Options:
- --help Show this message and exit.
-
- Commands:
- add Tracks a data format file SPECIFICATION...
- generate Generates a data format file from JSON input examples
- list Lists all your data formats
- publish Publishes data format to make publicly...
- show Provides more information about FORMAT
+ $ dcae_cli data_format add $HOME/yourapp/data-formats/health.json
-Your data format must be in the catalog in order to use in the component
-specification. Check the catalog using the ``data_format list``
-sub-command:
+Verify that the data_format was added
::
- $ dcae_cli data_format list
+ $ dcae_cli data_format list | grep yourapp
+ | sandbox.platform.yourapp.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-11-07 21:48:47.736518 |
- Data formats for mh677g
- +------+---------+-------------+--------+----------+
- | Name | Version | Description | Status | Modified |
- +------+---------+-------------+--------+----------+
- | | | | | |
- +------+---------+-------------+--------+----------+
+(Note: Each of the data formats for your component need to be added,
+unless already existing in the onboarding catalog )
-The fields ``name``, ``version``, ``description`` are referenced from
-the data format JSON from the ``self`` JSON.
-
-There are no data formats so you must add the data formats that your
-component specification references. Use the ``data_format add``
-sub-command:
+--------------
-Here’s an example command:
+Add the Component
+-----------------
::
- dcae_cli data_format add health.json
+ $ dcae_cli component add $HOME/yourapp/component-spec.json
-Verify that it was added:
+Verify that the component was added
::
- $ dcae_cli data_format list
-
- Data formats for mh677g
- +-------------------------------+---------+-------------------------------------------+--------+----------------------------+
- | Name | Version | Description | Status | Modified |
- +-------------------------------+---------+-------------------------------------------+--------+----------------------------+
- | sandbox.platform.laika.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-05-23 04:02:38.952799 |
- +-------------------------------+---------+-------------------------------------------+--------+----------------------------+
+ $ dcae_cli component list
+ Active profile: solutioning
-Go ahead and add other referenced data formats.
+ +-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+ | Name | Version | Type | Description | Status | Modified | #Deployed |
+ +-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+ | sandbox.platform.yourapp | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-11-08 20:27:34.168854 | 0 |
+ +-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
-If you have JSON input you can generate a data format like this:
+--------------
-::
+.. _dcae-cli-view-the-platform:
- $ dcae_cli data_format --keywords generate myname:1.0.0 myjsoninputfile
+View the platform generated configuration
+-----------------------------------------
-where ``myname`` is the name of your data format, ``1.0.0`` is an
-example version, and ``myjsoninputfile`` is an example JSON input file
-(a directory of input JSON files can also be provided). The
-``--keywords`` option adds additional data attributes that can be
-completed to provide a more detailed data characterization. In any event
-the output should be reviewed for accuracy. The data format is written
-to stdout.
+The ``component dev`` command is useful during onboarding. Running this
+command is part of a multi-step process that sets up a temporary test
+environment, generates your application configuration, makes it
+available in that environment, and allows you to view that configuration
+to help with debugging.
-.. _adding-component:
+Here is a step-by-step example based on a component specification called
+``component-spec.json``.
-Adding component
-----------------
+Step 1 - Run the component dev command
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``component`` is the sub-command that is used to work with operations
-for components:
+(This creates a file called env_$ENV (in the current directory)- where
+$ENV is the name of the active profile. Note: SERVICE_NAME and HOSTNAME
+always resolve to the same value).
::
- $ dcae_cli component --help
- Usage: dcae_cli component [OPTIONS] COMMAND [ARGS]...
-
- Options:
- --help Show this message and exit.
-
- Commands:
- add
- dev Set up component in development for...
- list Lists components in the public catalog.
- publish Pushes COMPONENT to the public catalog
- run Runs the latest version of COMPONENT.
- show Provides more information about COMPONENT
- undeploy Undeploys the latest version of COMPONENT.
-
-Your component must be accessible from the catalog in order for it to be
-used. Check the catalog using the ``component list`` sub-command:
-
-::
+ $ dcae_cli component dev component-spec.json
+ Ready for component development
- $ dcae_cli component list
- Active profile: solutioning
+ Setup these environment variables. Run "source env_solutioning":
- +------+---------+------+-------------+--------+----------+-----------+
- | Name | Version | Type | Description | Status | Modified | #Deployed |
- +------+---------+------+-------------+--------+----------+-----------+
- | | | | | | | |
- +------+---------+------+-------------+--------+----------+-----------+
+ export DOCKER_HOST=yourdockerhost.com:2376
+ export SERVICE_CHECK_INTERVAL=15s
+ export CONFIG_BINDING_SERVICE=config_binding_service
+ export HOSTNAME=user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp
+ export CONSUL_HOST=yourconsulhost.com
+ export CDAP_BROKER=cdap_broker
+ export SERVICE_NAME=user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp
+ export SERVICE_CHECK_TIMEOUT=1s
+ export SERVICE_CHECK_HTTP=/health
- Use the "--deployed" option to see more details on deployments
+ Press any key to stop and to clean up
-The fields ``name``, ``version``, ``type``, ``description`` are
-referenced from the component specification’s ``self`` JSON.
+Step 2 - Setup the environment
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-There are no components so you must add your component. Use the
-``component add`` sub-command. The command is the same for docker and
-cdap components:
+In another window, setup the temporary testing environment, by executing
+the environment file created above.
::
- $ dcae_cli component add --help
- Usage: dcae_cli component add [OPTIONS] SPECIFICATION
-
- Options:
- --update Updates a locally added component if it has not been already
- pushed
- --help Show this message and exit.
+ $ source env_solutioning
-*Note* use the ``--update`` flag to replace existing staged instances.
+(The application configuration is now available under the SERVICE_NAME
+shown above -
+``user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp``).
-The ``component dev`` sub-command can be useful in validating and
-experimenting when crafting your component specification. See details
-about ``dev`` under `Development and
-testing <#development-and-testing>`__.
+Step 3 - Query CONSUL
+~~~~~~~~~~~~~~~~~~~~~
-Once we add the components laika and helloworld, let’s verify that they
-got added ok:
+Query CONSUL to get the IP/PORT of CONFIG BINDING SERVICE
::
- $ dcae_cli component list
- Active profile: solutioning
+ $ curl http://$CONSUL_HOST:8500/v1/catalog/service/$CONFIG_BINDING_SERVICE
+ [
+ {
+ "ID": "bfbc220d-4603-7f90-ec2e-611d3c330f20",
+ "Node":"docker00",
+ "Address": "10.226.1.15",
+ "Datacenter":"solutioning-central",
+ "TaggedAddresses": {
+ "lan":"10.226.1.15",
+ "wan":"10.226.1.15"
+ },
+ "NodeMeta": {},
+ "ServiceID": "472b116f9035:config_binding_service:10000",
+ "ServiceName": "config_binding_service",
+ "ServiceTags": [],
+ "ServiceAddress":"135.205.226.126",
+ "ServicePort":10000,
+ "ServiceEnableTagOverride": false,
+ "CreateIndex":1078990,
+ "ModifyIndex":1078990
+ }
+ ]
- +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
- | Name | Version | Type | Description | Status | Modified | #Deployed |
- +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
- | cdap.helloworld.endnode | 0.8.0 | cdap | cdap test component | staged | 2017-05-23 04:14:35.588075 | 0 |
- | sandbox.platform.laika | 0.5.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-05-23 04:07:44.065610 | 0 |
- +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+Fetch the generated configuration from CONFIG BINDING SERVICE using the
+‘serviceaddress’ and ‘serviceport’ from above along with $SERVICE_NAME
+from earlier.
- Use the "--deployed" option to see more details on deployments
+::
-.. _setting-profile:
+ $ curl http://135.205.226.126:10000/service_component/user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp
-Setting profile
----------------
+ {"streams_subscribes": {}, "services_calls": {}, "multiplier": 3, "streams_publishes": {}}
-``profile`` is the sub-command that is used to manage profiles. These
-profiles contain environment variables used to connect to different
-environments. This is used in the running and deployment of your
-component using the ``dcae_cli component run`` command. The ``dcae-cli``
-ships with profiles for ``solutioning`` and ``rework``.
+--------------
-::
+.. _dcae-cli-walkthrough-dmaap-testing:
- $ dcae_cli profiles --help
- Usage: dcae_cli profiles [OPTIONS] COMMAND [ARGS]...
+Create the input file for Dmaap Testing
+---------------------------------------
- Options:
- --help Show this message and exit.
+Currently, the dcae-cli tool does not have the capability to provision
+topics or feeds. Therefore, in order to test with ``message router`` or
+``data router`` feeds, the developer must manually provision the topic
+or feed and then provide the connection details in the form of a DMaap
+JSON file for testing. This file is then passed in on the
+``component run`` or ``component dev`` commands by using the argument
+``--dmaap-file``.
- Commands:
- activate Sets profile NAME as the active profile
- create Creates a new profile NAME initialized with...
- delete Deletes profile NAME
- list Lists available profiles
- set Updates profile NAME such that KEY=VALUE
- show Prints the profile dictionary
+The structure of the DMaaP JSON is an object of config keys with the
+topic or feed connection details. The config keys are the ``config_key``
+values specified in the component specification streams section where
+the streams must be type ``message router`` or ``data router``. This
+file corresponds to the ``Dmaap Connection Object`` which is generated
+by the platform and provided to the component at runtime. The exception
+is that ``delivery_url`` cannot be provided in the dmaap-file because it
+is not created until the component is deployed. Refer to :any:`Dmaap Connection Object <dmaap-connection-objects>`, for details on creating the dmaap-file for testing.
-To see what variables a profile contains, you can use the ``show``
-command, as in ``dcae_cli profiles show PROFILE_NAME``
+--------------
-Use the ``create`` sub-command to create your own profile and assign new
-values using the ``set`` command. Afterwards you will need to
-``activate`` the profile you wish to use. First take a look at which
-profile is active:
+Create the input file for *Sourced at Deployment* Testing
+---------------------------------------------------------
-::
+Components may have configuration parameters whose values are to be
+sourced at deployment time. This is established in the
+:any:`component specification <common-specification-parameters>`
+by setting the property ``sourced_at_deployment`` to ``true`` for each
+applicable parameter.
- $ dcae_cli profiles list
- rework
- * solutioning
+Then, use the ``--inputs-file`` command-line argument when running the
+component ``dev`` or ``run`` command for your component. This is to
+simulate providing the dynamic, deployment time values for those
+parameters marked as ``sourced_at_deployment``.
-The active profile is ``solutioning`` so to activate *rework* to use
-``rework``:
+For example, if your component specification has the following
+configuration parameters:
::
- $ dcae_cli profiles activate rework
+ "parameters": [{
+ "name": "vnf-ip",
+ "value": "",
+ "sourced_at_deployment": true
+ },
+ {
+ "name": "static-param",
+ "value": 5
+ }]
-Check
+Pass in an input file that looks like:
::
- $ dcae_cli profiles list
- * rework
- solutioning
+ {
+ "vnf-ip": "10.100.1.100"
+ }
-.. _development-and-testing:
+The application configuration would look like:
-Development and testing
------------------------
+::
-The following operations under the sub-command ``component`` are aimed
-to help developers with testing:
+ {
+ "vnf-ip": "10.100.1.100",
+ "static-param": 5
+ }
-- ``run``
-- ``undeploy``
-- ``dev``
+--------------
-``run``
-~~~~~~~
+Run the component
+-----------------
The ``run`` operation is to be used for running your application in its
container remotely on the activated environment. Docker containers have
-the additional option to run locally on your development machine.
+the additional option to run locally on your development machine. If the
+component uses Dmaap, you can specify the Dmaap Connection Object as
+well. Refer to :any:`Dmaap Connection Object <dmaap-connection-objects>`.
-In order to run your application, you must have added your data formats
-and your component to your catalog.
+In order to run the component, the data formats and component must have
+been added to the onboarding catalog.
-Let’s verify that your component is in the catalog:
+To verify what’s in the catalog:
::
- $ dcae_cli component list
+ $ dcae_cli catalog list --expanded
Active profile: solutioning
+ +---------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+ | Name | Version | Type | Description | Status | Modified | #Deployed |
+ +---------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
+ | sandbox.platform.yourapp | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-11-08 20:27:34.168854 | 0 |
+ +---------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
- +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
- | Name | Version | Type | Description | Status | Modified | #Deployed |
- +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
- | cdap.helloworld.endnode | 0.8.0 | cdap | cdap test component | staged | 2017-05-23 04:14:35.588075 | 0 |
- | sandbox.platform.laika | 0.5.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-05-23 04:07:44.065610 | 0 |
- +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+
-
- Use the "--deployed" option to see more details on deployments
-
-Docker
-^^^^^^
+For Docker
-**NOTE** Make sure your Docker image has been uploaded to the shared
+**NOTE** Make sure the Docker image has been uploaded to the shared
registry.
-For Docker containers, you can run either attached or unattached.
-Attached means that the dcae-cli tool will launch the container and not
-terminate. The dcae-cli while attached will stream in the logs of the
-Docker container. Doing a Ctrl-C will terminate the run session which
-means undeploy your container and force a clean up automatically.
-
-Running unattached means simply deploy your container. You will need to
-execute ``undeploy`` when you are done testing. #### CDAP
-
-**NOTE** Make sure your CDAP jar has been uploaded to Nexus.
-
-TODO
-
-``undeploy``
-~~~~~~~~~~~~
-
-The ``undeploy`` operation is to be used to undeploy any instances of a
-specified component, version that you have deployed. This includes
-cleaning up of configuration.
-
-Let’s undeploy ``sandbox.platform.laika`` that was deployed from the
-previous section:
+A docker component can be run in either ``attached`` or ``unattached``
+mode. (Default is unattached).
+
++------------------+---------------------------------------------------+
+| Mode | Description |
++==================+===================================================+
+| attached | component is run in the ‘foreground’, container |
+| | logs are streamed to stdout. Ctrl-C is used to |
+| | terminate the dcae_cli session. |
++------------------+---------------------------------------------------+
+| unattached | component is run in the ‘background’, container |
+| | logs are viewed via ``docker logs`` command, |
+| | container runs until undeployed with dcae_cli |
+| | ``undeploy`` command. |
++------------------+---------------------------------------------------+
+
+Run a component in attached mode:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
::
- $ dcae_cli component undeploy sandbox.platform.laika:0.5.0
- DCAE.Undeploy | WARNING | Undeploying components: 1
- DCAE.Undeploy | WARNING | Undeployed components: 1
-
-.. _walkthrough-dev:
-
-``dev``
-~~~~~~~
-
-The ``dev`` operation is a convenient operation that can be useful for
-the development and testing of your component. It can be used to:
+ $ dcae_cli -v component run --attached sandbox.platform.yourapp:0.7.0
+ DCAE.Docker | INFO | Running image 'nexus01.server.com:18443/repository/solutioning01-mte2-docker/dcae-platform/yourapp:0.7.0' as 'user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp'
+ DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | Consul host: yourconsulhost.com
-- Help validate your experimental component specification before
- uploading to the catalog
-- Generate the application configuration from the component
- specification and make it available in a test environment. This
- allows you to view your resulting configuration for local development
- and to help debug potential related issues.
+ DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | service name: user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp
-Let’s say you have a component specification called
-``component-spec.json``:
-
-::
-
- $ dcae_cli component dev component-spec.json
- Ready for component development
+ DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | get_config returned the following configuration: {"streams_subscribes": {}, "multiplier": 3, "services_calls": {}, "streams_publishes": {}}
- Setup these environment varibles. Run "source env_solutioning":
+ DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | * Running on http://0.0.0.0:8080/ (Press CTRL+C to quit)
- export DOCKER_HOST=SOME_DOCKER_HOST:2376
- export SERVICE_CHECK_INTERVAL=15s
- export CONFIG_BINDING_SERVICE=config_binding_service
- export HOSTNAME=mh677g.95740959-63d2-492a-b964-62a6dce2591d.0-6-0.sandbox-platform-laika
- export CONSUL_HOST=SOME_CONSUL_HOST
- export CDAP_BROKER=cdap_broker
- export SERVICE_NAME=mh677g.95740959-63d2-492a-b964-62a6dce2591d.0-6-0.sandbox-platform-laika
- export SERVICE_CHECK_TIMEOUT=1s
- export SERVICE_CHECK_HTTP=/health
+ DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | 135.205.226.156 - - [08/Nov/2017 23:27:30] "GET /health HTTP/1.1" 200 -
- Press any key to stop and to clean up
-Your application configuration is now available under the name
-``mh677g.95740959-63d2-492a-b964-62a6dce2591d.0-6-0.sandbox-platform-laika``.
+ Hit Ctrl-C to terminate session.
-To view the resulting configuration, you can ``curl`` a request to the
-config binding service or programmatically fetch your configuration
-within your application.
+ ^C
+ DCAE.Docker | INFO | Stopping container 'user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp' and cleaning up...
-You need to first query Consul to get the ip and port of config binding
-service:
+Run a component in unattached mode:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
::
- curl http://$CONSUL_HOST:8500/v1/catalog/service/$CONFIG_BINDING_SERVICE
- [
- {
- "ID": "983d5c94-c508-4a8a-9be3-5912bd09786b",
- "Node": "realsolcnsl00",
- "Address": "10.226.1.22",
- "TaggedAddresses": {
- "lan": "10.226.1.22",
- "wan": "10.226.1.22"
- },
- "NodeMeta": {},
- "ServiceID": "5f371f295c90:config_binding_service:10000",
- "ServiceName": "config_binding_service",
- "ServiceTags": [],
- "ServiceAddress": "XXXX",
- "ServicePort": 32770,
- "ServiceEnableTagOverride": false,
- "CreateIndex": 487,
- "ModifyIndex": 487
- }
- ]
-
-.. _dmaap-testing:
-
-DMaaP testing
-~~~~~~~~~~~~~
-
-Currently, the dcae-cli does not have the capability of provisioning
-topics. In order to do testing with message router topics or with data
-router feeds, the developer must provision the topic or the feed
-manually and provide the connection details in the form of a JSON in a
-file to the dcae-cli. This file is to be passed in when using the
-``run`` and ``dev`` commands with the option ``--dmaap-file``.
-
-The structure of the DMaaP JSON is an object of config keys to matching
-topic or feed connection details. Config keys are the ``config_key``
-values specified in your component specification streams section where
-the streams must be type message router or data router. Information
-about the associated connection details can be found on `this
-page <dmaap-connection-objects.md>`__. Please check it out.
-
-For example, if you have a component specification that has the
-following streams entry:
-
-.. code:: json
-
- "streams": {
- "publishes": [{
- "format": "ves",
- "version": "1.0.0",
- "type": "message router",
- "config_key": "ves_connection"
- }]
- }
+ $ dcae_cli -v component run sandbox.platform.yourapp:0.7.0
+ DCAE.Docker | INFO | Running image 'nexus01.server.com:18443/repository/solutioning01-mte2-docker/dcae-platform/yourapp:0.7.0' as 'user12.22629ebd-417e-4e61-a9a0-f0cb16d4cef2.0-7-0.sandbox-platform-yourapp'
+ DCAE.Run | INFO | Deployed user12.22629ebd-417e-4e61-a9a0-f0cb16d4cef2.0-7-0.sandbox-platform-yourapp. Verifying..
+ DCAE.Run | INFO | Container is up and healthy
-Then to deploy and to run your component, you must use the
-``--dmaap-file`` command and pass in a JSON that looks like:
+**NOTE** You must undeploy this component when finished testing. This is
+important to conserve resources in the environment.
-.. code:: json
+Run a component that subscribes to Dmaap Message Router or Data Router
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- {
- "ves_connection": {
- "type": "message_router",
- "dmaap_info": {
- "topic_url": "https://we-are-message-router.us:3905/events/some-topic"
- }
- }
- }
-
-The provided DMaaP JSON is used to simulate the output of provisioning
-and will be used to merge with the generated application configuration
-at runtime.
-
-Your final application config will look like:
-
-.. code:: json
-
- {
- "streams_publishes": {
- "ves_connection": {
- "type": "message_router",
- "dmaap_info": {
- "topic_url": "https://we-are-message-router.us:3905/events/some-topic"
- }
- }
- }
- }
+::
-Data router subscribers
-^^^^^^^^^^^^^^^^^^^^^^^
+ $ dcae_cli -v component run $component-that-uses-dmamp --dmaap-file $dmaap-connection-object
-Note for data router subscriber testing, you will need the delivery url
-in order to provision the subscriber to the feed. This is constructed at
-deployment time and will be provided by the dcae-cli after you deploy
-your component. The delivery urls will be displayed to the screen:
+Run a component that expects input that is ``sourced at deployment``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
::
- DCAE.Run | WARNING | Your component is a data router subscriber. Here are the delivery urls:
+ $ dcae_cli -v component run $component-that-expects-dti --inputs-file $input-file-to-simulate-dti
- some-sub-dr: http://SOME_IP:32838/identity
+--------------
-*Sourced at deployment* testing
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Components may have configuration parameters whose values are to be
-sourced at deployment time. For example, there are components whose
-configuration parameters are to come from DTI events which are only
-available when the component is getting deployed. These configuration
-parameters must be setup correctly in the `component
-specification <http://localhost:8000/components/component-specification/docker-specification/#configuration-parameters>`__
-by setting the property ``sourced_at_deployment`` to ``true`` for each
-and every parameter that is expected to come in at deployment time.
+.. _dcae_cli_undeploy_the_component:
-Once your component specification has been updated correctly, you must
-use the ``--inputs-file`` command-line argument when running the
-commands ``dev`` or ``run`` with your component. This is to simulate
-providing the dynamic, deployment time values for those parameters
-marked as ``sourced_at_deployment``.
+Undeploy the component
+----------------------
-For example, if your component specification has the following
-configuration parameters:
-
-::
++-----------------------------------------------------------------+
+| The ``undeploy`` command is used to undeploy any instance of a |
+| specified component/version that you have deployed. This |
+| includes cleaning up the configuration. |
++-----------------------------------------------------------------+
+| Undeploy ``sandbox.platform.yourapp:0.7.0`` that was deployed |
+| above: |
++-----------------------------------------------------------------+
+| ``$ dcae_cli -v component undeploy sandbox.platform.yourapp:0.7.0 |
+| DCAE.Undeploy | WARNING | Undeploying components: 1 DCAE.Undep |
+| loy | WARNING | Undeployed components: 1`` |
++-----------------------------------------------------------------+
- "parameters": [{
- "name": "vnf-ip",
- "value": "",
- "sourced_at_deployment": true
- },
- {
- "name": "static-param",
- "value": 5
- }]
+.. _dcae_cli_publish_the_component_and_data_format:
-You would have to pass in an inputs file that looks like:
+Publish the component and data_format
+-------------------------------------
-::
+Once a component has been tested, it (and the data_format(s)) should be
+published in the onboarding catalog using the ``publish`` sub-command
+for both the ``data_format`` and ``component`` command.
- {
- "vnf-ip": "10.100.1.100"
- }
+**Note** Before a component can be published, all data_formats that it
+references must be published.
-Your application configuration would look like:
+Publishing will change the status of a component or data_format,
+indicating that it has been tested, make accessible for other developers
+to use.
::
- {
- "vnf-ip": "10.100.1.100",
- "static-param": 5
- }
-
-Publishing component
---------------------
+ $ dcae_cli data_format publish sandbox.platform.yourapp:0.7.0
+ Data format has been published
-Once components have their component specifications crafted and
-validated and have been tested, components should be published in the
-shared onboarding catalog using the ``publish`` sub-command for both
-data formats and components. You must publish all data formats of a
-component before publishing a component.
+ $dcae_cli component publish sandbox.platform.yourapp:0.7.0
+ Component has been published
-Publishing will change the status of a component, be made accessible for
-other developers to use, and will generate the associated TOSCA models
-for use in designing of compositions.
-
-::
-
- dcae_cli component publish sandbox.platform.laika:0.5.0
-
-Shared catalog
--------------
-``catalog`` is the sub-command used to access and to browse the shared
-onboarding catalog to view components and data formats that have been
-published and that are being worked on. Components and data formats have
-two statuses ``staged`` and ``published``.
-
-Staged means that the resource has been simply added and is under
-development. It is to be used only by the owner. Published means that
-the resource has been fully developed and tested and is ready to be
-shared.
-
-Published components can be deployed by non-owners and published data
-formats can be used in component specifications of non-owners.
-
-There are two available operations:
+List the catalog contents
+-------------------------
::
- $ dcae_cli catalog --help
- Usage: dcae_cli catalog [OPTIONS] COMMAND [ARGS]...
-
- Options:
- --help Show this message and exit.
+ $dcae_cli catalog list
- Commands:
- list
- show
+ $ dcae_cli data_format list | grep sandbox
+ | sandbox.platform.yourapp | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | user12 | published | 2017-11-13 |
+ | sandbox.platform.yourapp.health | 0.1.0 | Data format used for the /health endpoint | published | 2017-11-13 17:48:10.121588 |
+ | sandbox.platform.any | 0.1.0 | Data format used when no data format is required. | published | 2017-11-13 17:47:51.622607 |
+ | sandbox.platform.yourapp.identity.response | 0.1.0 | Data format used for the /identity endpoint response which should | published | 2017-11-13 17:47:43.234715 |
+ | sandbox.platform.yourapp.identity.request | 0.1.0 | Data format used for the /identity endpoint request. This is | published | 2017-11-13 17:47:36.693643 |
+ | sandbox.platform.yourapp.rollcall.response | 0.1.0 | Data format used for the /rollcall endpoint respon.. | published | 2017-11-13 17:46:30.026846 |
-Staged components can be viewed under the ``list`` operation using the
-``--expanded`` flag.
diff --git a/docs/sections/components/glossary.rst b/docs/sections/components/glossary.rst index 6c048cf3..1ad4fcd0 100644 --- a/docs/sections/components/glossary.rst +++ b/docs/sections/components/glossary.rst @@ -3,35 +3,208 @@ .. _glossary: + Glossary ======== -.. _glossary-asdc: +A&AI - Active and Available Inventory +------------------------------------- + +Inventory DB for all network components + +CLAMP +----- + +Non DCAE Platform Component - Controls the input and processing for +Closed Loop services. -ASDC +Closed Loop +----------- + +Services designed to monitor and report back to a controlling function +that automatically deals with the event reported without human +interaction. + +CDAP ---- -The ECOMP resource catalog. We assume the existince and usage of ASDC, though for near term testing purposes, we use a mock version of the catalog. -We assume all DCAE artifacts: components like collectors, CDAP applications, data formats (see below), etc, are onboarded and searchable in the catalog. -Further, we assume that every catalog artifact has a *UUID*, a globally unique identifier that identifies that artifact. +Opensource Platform for development of Big Data platforms using Hadoop. +Some DCAE service components are written utilizing CDAP. + +Cloudify +-------- -.. _glossary-component: +Open Source application and network orchestration framework, based on +TOSCA used in DCAE to deploy platform and service components from +Cloudify Blueprints. Refer to :doc:`Architecture <./architecture/pieces>` +for more information. + +Cloudify Blueprints +------------------- + +YAML formatted file used by Cloudify to deploy platform and service +components. Contains all the information needed for installation. + +Consul +------ + +Opensource Platform Component that supports Service Discovery, +Configuration, and Healthcheck. Refer to +:doc:`Architecture <./architecture/pieces>` for more information. Component --------- -Refers to a DCAE service component which is a single micro-service that is written to be run by the DCAE platform and to be composeable to form a DCAE service. +Refers to a DCAE service component which is a single micro-service that +is written to be run by the DCAE platform and to be composeable to form +a DCAE service. That composition occurs in the SDC. + +Config Binding Service +---------------------- + +DCAE Platform Component - Service Components use Config Binding Service +to access Consul and retrieve configuration variables. + +Component Specification +----------------------- -.. _glossary-data-format: +JSON formatted file that fully describes a component and its interfaces -Data format +Data Format / Data Format Specification +--------------------------------------- + +JSON formatted file that fully describes a components input or output + +dcae_cli Tool +------------- + +Tool used for development and testing. It validates the component and +data format specifications against their respective schemas and provides +the capability to view platform generated configuration for the +component. + +Deployment Handler +------------------ + +DCAE Platform Component - talks to +Cloudify to deploy components. + +Design-Time ----------- -Artifact that describes a data structure and schema. +Refers to when the System Designer uses the SDC Tool to compose services +from components in the SDC catalog. The Designer can provide input to +assign/override defaults for configuration for any parameter with the +property ‘designer_editable’ set to ‘true’. -.. _glossary-onboarding-catalog: +Deploy-Time +----------- + +Refers to when a service is being deployed. This can be done +automatically via the SDC Tool, or manually via the DCAE Dashboard or +CLAMP UI. When manually deployed, DevOps can provide input to +assign/override defaults for configuration for any parameter with the +property ‘sourced_at_deployment’ set to ‘true’. + +Docker +------ + +Opensource Platform for development of containerized applications in the +cloud. Many DCAE service components and all DCAE collectors are written +utilizing Docker. + +Dmaap +----- + +A data transportation service platform that supports message-based +topics and file-based feeds. Runs locally at the Edge and Centrally. + +Inventory +--------- + +DCAE Platform Component - Postgres DB containing Cloudify Blueprints for +platform and service components. Onboarding catalog ------------------ -The onboarding catalog is the intermediary store for project details, component specifications and data formats. This is to be used by component developers through the `dcae-cli` to browse and to launch existing components and push their own component artifacts in preparation to move their component to the ASDC resource catalog. +Catalog used exclusively by the dcae_cli tool during development and +testing. Contains validated components and data_formats to be used among +developers during development and testing. + +Policy (not yet implemented) +---------------------------- + +Refers to the setting of configuration parameters for a component, by +Operations via the Policy UI. + +Policy Handler (not yet implemented) +------------------------------------ + +DCAE Platform Component that received Policy updates from Policy UI + +Policy UI (not yet implemented) +------------------------------- + +Non DCAE Component - Policy User Interace where Operations assigns +values to configuraton specified for this. + +Run-Time +-------- + +Refers to the when a service is running on the platform. + +SCH - Service Change Handler +---------------------------- + +DCAE Platform Component - Receives updates from SDC and updates +Inventory + +SDC - Service Design and Creation - (formerly ASDC) +--------------------------------------------------- + +Tool used by Service Designers to compose services from SDC catalog +artifacts. Once services are created, Cloudify Blueprints can be +generated to deployment and installation. + +SDC Catalog +----------- + +Catalog of composable Components and Data Formats to be used in the SDC +Tool to create services. Currently, there is no access to the SDC +Catalog from the dcae_cli tool. Artifacts are manually placed there +after testing. Every catalog artifact has a ``UUID``, a globally unique +identifier that identifies that artifact. + +Self-Service +------------ + +Refers to services that are supported by SDC, and that are automatically +installed as a result of a Service Designer’s composition and submission +of a service. Only a handful of services are ‘self-service’ currently. +Most require manual effort to generate the Tosca Model files and +Cloudify Blueprints. + +Service Component +----------------- + +Microservice that provides network monitoring or analytic function on +the DCAE platform. + +Service +------- + +Generally composed of multiple service components, which is deployed to +the DCAE platform. + +Tosca Model +----------- + +Model generated from validately component specification, (stored in SDC +catalog for Self-Service components), and used as input to generate +Cloudify Blueprints + +VNF - Virtualized Network Function +---------------------------------- + +A network function that runs on one or more virtualized machines. diff --git a/docs/sections/components/images/IO.graffle b/docs/sections/components/images/IO.graffle Binary files differnew file mode 100644 index 00000000..47c18b2a --- /dev/null +++ b/docs/sections/components/images/IO.graffle diff --git a/docs/sections/components/images/dmd vision.graffle b/docs/sections/components/images/dmd vision.graffle Binary files differnew file mode 100644 index 00000000..c6ad0d51 --- /dev/null +++ b/docs/sections/components/images/dmd vision.graffle diff --git a/docs/sections/components/images/dmdvision.png b/docs/sections/components/images/dmdvision.png Binary files differnew file mode 100644 index 00000000..cc6f195f --- /dev/null +++ b/docs/sections/components/images/dmdvision.png diff --git a/docs/sections/components/images/io.png b/docs/sections/components/images/io.png Binary files differnew file mode 100644 index 00000000..26c5eba9 --- /dev/null +++ b/docs/sections/components/images/io.png diff --git a/docs/sections/components/intro.rst b/docs/sections/components/intro.rst index 8a31e844..db7175c1 100755 --- a/docs/sections/components/intro.rst +++ b/docs/sections/components/intro.rst @@ -4,29 +4,30 @@ .. _intro:
-Component Developer Overview
-============================
+Overview
+========
DCAE components are services that provide a specific functionality and
-are written to be composable with other DCAE service components. The
-DCAE platform is responsible for running and managing DCAE service
-components reliably.
+are generally written to be composable with other DCAE components,
+although a component can run independently as well. The DCAE platform is
+responsible for running and managing DCAE service components reliably.
Currently, the DCAE platform supports two types of components, CDAP
applications and Docker containers. For each, there are requirements
that must be met for the component to integrate into the DCAE platform
-(see :doc:`CDAP <component-type-cdap>` and :doc:`Docker <component-type-docker>`.
+(see :doc:`CDAP <component-type-cdap>` and
+:doc:`Docker <component-type-docker>`).
-Components requires one or more data formats.
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A Component requires one or more data formats.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Components are software applications that do some function. Components
-don’t run independently, they depend upon other components. A
+A component is a software application that performs a function. It
+doesn’t run independently; it depends upon other components. A
component’s function could require connecting to other components to
fulfill that function. A component could also be providing its function
as a service through an interface for other components to use.
-Components cannot connect to or be connected with any other component.
+A component cannot connect to or be connected with any other component.
The upstream and downstream components must *speak* the same vocabulary
or *data format*. The output of an one component must match another
component’s input. This is necessary for components to function
@@ -42,8 +43,7 @@ Each Component requires a component specification. The component specification is a JSON artifact that fully specifies the
component, it’s interfaces, and configuration. It’s standardized for
-CDAP and Docker applications and is validated using a :any:`JSON
-schema <dcae-component-schema>`.
+CDAP and Docker applications and is validated using a :doc:`JSON schema <./component-json-schema>`.
The component specification fully specifies all the configuration
parameters of the component. This is used by the designer and by policy
@@ -80,18 +80,26 @@ Onboarding ----------
Onboarding is a process that ensures that the component is compliant
-with the DCAE platform rules. A command-line tool called :doc:`dcae-cli <dcae-cli/quickstart>` is provided to help with onboarding. The high level summary of the onboarding process is:
+with the DCAE platform rules. A command-line tool called
+:doc:`dcae-cli <./dcae-cli/quickstart>` is provided to
+help with onboarding. The high level summary of the onboarding process
+is:
1. Defining the :doc:`data formats <data-formats>` if they don’t already
exist.
-2. Define the :doc:`component specification <component-specification/common-specification>`. See :doc:`Docker <component-specification/docker-specification>` and :doc:`CDAP <component-specification/cdap-specification>`.
-3. Use the dcae_cli tool to :any:`add the data formats <adding-data-formats>`
- and :any:`add the component <adding-component>` to
- the onboarding catalog. This process will validate them as well.
-4. Use the dcae_cli tool to :any:`deploy <development-and-testing>`
- the component. (The component is deployed to the environment
- indicated in :any:`profile <setting-profile>`).
+2. Defining the :doc:`component specification <./component-specification/common-specification>`.
+ See :doc:`docker <./component-specification/docker-specification>` and
+ :doc:`CDAP <./component-specification/cdap-specification>`.
+3. Use the dcae_cli tool to :any:`add the data formats <dcae_cli_add_a_data_format>` and
+ :any:`add the component <dcae_cli_add_a_component>` to the
+ onboarding catalog. This process will validate them as well.
+4. Use the dcae_cli tool to
+ :any:`deploy <dcae_cli_run_a_component>` the
+ component. (The component is deployed to the environment indicated in
+ the :any:`profile <dcae_cli_activate_profile>`
+ section).
5. Test the component. Also do pairwise-test the component with any
other components it connects with.
6. Publish the component and data formats into the Service Design and
- Creation (SDC) ‘catalog’. (Currently, this is a manual step).
+ Creation (SDC) ‘catalog’. (Currently, this is a manual step, not done
+ via the dcae_cli tool).
diff --git a/platformdoc/DCAE-DEPLOYMENT-GUIDE-1707.md b/platformdoc/DCAE-DEPLOYMENT-GUIDE-1707.md new file mode 100644 index 00000000..415fb2b6 --- /dev/null +++ b/platformdoc/DCAE-DEPLOYMENT-GUIDE-1707.md @@ -0,0 +1,73 @@ +### DCAE Controller - Deployment Guide 1707
+
+#### 1) Download latest deployer tar file to your cloudify bootstrap/cli server.
+
+Say, your working directory is ${WORKING-DIR}
+````
+cd ${WORKING-DIR}
+$ wget http://dockercentral.it.att.com:8093/nexus/repository/rawcentral/com.att.ecompcntr/opstools/dcae-deployer-5.0.tar
+$ tar -xvf dcae-deployer-5.0.tar
+$ cd ${WORKING-DIR}/dcae-deployer
+````
+#### 2) Clone the auto generated inputs files from code cloud and save it in <inputDirectory>
+````
+$ git clone ssh://<userid>@codecloud.web.att.com:7999/st_dcae/com.att.ecomp.dcae.operational.staging.ftl.git <inputsDirectory>
+
+````
+#### 3) Clone the latest consulAgent.sh
+````
+$ git clone ssh://git@codecloud.web.att.com:7999/st_ecompcntr/platform_installation.git
+NOTE: consulAgent.sh will be inside ec-deployer-v2.5.1/ec-deployer/bin
+````
+#### 4) Update app-inputs.conf with user defined values
+````
+$ cd ${WORKING-DIR}/dcae-deployer/conf
+$ vi app-inputs.conf
+
+ -- Update the following values -
+ * CLOUDIFY_INPUT=<complete path to the auto generated inputs.json for cloudify ( downloaded in inputsDirectory from Step 2 )>
+ e.g. /home/attcloud/dcae/dcae-deployer/testing/1710-FTL3/cloudify-gen/inputs/bootstrap/bootstrap/rdm5adcc1/inputs.json
+ * CONSUL_INPUT=<complete path to the auto generated inputs.json for consul ( downloaded in inputsDirectory from Step 2 )>
+ e.g. /home/attcloud/dcae/dcae-deployer/testing/1710-FTL3/cloudify-gen/inputs/consul/consul/rdm5adcc1/inputs.json
+ * CONSUL_BLUEPRINT=<complete path to the zip for consul on dockercentral>
+ - git clone ssh://git@codecloud.web.att.com:7999/st_ecompcntr/blueprints.git
+ e.g. /home/attcloud/blueprints/serviceConfigRegistry/blueprint/consul.yaml
+ Note: consul.yaml will be inside blueprints/serviceConfigRegistry/blueprint
+ * OPENSTACK_KEYFILE=<complete path the pem file used for bootstraping>
+ e.g. '/home/attcloud/dcae-m82823u.pem'
+ * OPENSTACK_KEY=<name of the keypair used in openstack>
+ e.g. 'dcae-m82823'
+ * CLOUDIFY_USERNAME=<Username for Cloudify>
+ * CLOUDIFY_PASSWORD=<Password for Cloudify>
+ * CONSUL_AGENT=<complete path to the extracted consulAgent.sh from Step 3>
+ e.g. /home/attcloud/dcae/dcae-deployer/bin/install/bin/consulAgent.sh
+ * LOCATION=<put the value of location_id from consul inputs.json file( downloaded in inputsDirectory from Step 2 )>
+
+Save the file and exit!!
+````
+#### 3) Run the ./deploy.sh script for building entire environment. (This includes, spinning up cloudify manager VM, bootstrapping cloudify manager, and deploying consul blueprint)
+````
+##### Option-1: Just bootstrapping cloudify manager on new VM
+ Note: This will instantiate a new VM and will bootstrap it with cloudify 3.4.0
+ $ ./deploy.sh --action bootstrap
+````
+````
+##### Option-2: Bootstrap cloudify manager on existing centOS VM
+ Note : This will bootstrap the provide IP-ADDRESS with the cloudify 3.4.0
+ $ ./deploy.sh --action bootstrap --cloudify-ip <IP-ADDRESS>
+````
+````
+##### Option-3: Teardown cloudify manager
+ Note: This will tear down the cloudify from the provided IP-ADDRESS but will not remove the VM
+ $ ./deploy.sh --action teardown --cloudify-ip <IP-ADDRESS>
+````
+````
+##### Option-4: Deploy consul
+ Note: This will deploy the consul on provided IP-ADDRESS and will instantiate 3 new VMs
+ $ ./deploy.sh --action deploy --cloudify-ip <IP-ADDRESS> --component consul
+````
+````
+##### Option-5: Undeploy consul
+ Note : This will undeploy the consul from the provide IP-ADDRESS and will remove the 3 VMs
+ $ ./deploy.sh --action undeploy --cloudify-ip <IP-ADDRESS> --component consul
+````
\ No newline at end of file diff --git a/platformdoc/README.md b/platformdoc/README.md index ea5739ab..8c4c5996 100644 --- a/platformdoc/README.md +++ b/platformdoc/README.md @@ -4,7 +4,7 @@ Contains the public facing technical documentation for the dcae platform whose a * Architects * Component developers -* Operations +* Operations (TBD) ## Usage @@ -51,3 +51,4 @@ This is a [Mkdocs](http://www.mkdocs.org/) project. To serve a local version of docker run -d --name dpd -p 80:80 dcae-platform-documentation:YYYYMMDD ``` +Or email Patty Heffner (ph8547@att.com) diff --git a/platformdoc/docs/architecture/service-discovery.md b/platformdoc/docs/architecture/service-discovery.md index 25d69fcf..9831efcf 100644 --- a/platformdoc/docs/architecture/service-discovery.md +++ b/platformdoc/docs/architecture/service-discovery.md @@ -2,7 +2,7 @@ Service discovery is an architecture pattern used for components (micro-services) to locate each other. The DCAE platform uses [server-side discovery](http://microservices.io/patterns/server-side-discovery.html) and is using [Consul](https://www.consul.io/) as the service registry solution. -## Service registration +## Service Registration All components are required to register with Consul in order to be discovered. There are two methods of registration: self and 3rd party. The DCAE platform uses 3rd party registration which means components don't actually make the registration calls but defers that responsibility to a platform service. diff --git a/platformdoc/docs/architecture/services.md b/platformdoc/docs/architecture/services.md index 985953c3..f17f563b 100644 --- a/platformdoc/docs/architecture/services.md +++ b/platformdoc/docs/architecture/services.md @@ -1,7 +1,11 @@ # 'Services' Overview -DCAE Services run on the DCAE platform. A service performs either monitors the virtualized network services or does analytics. A service is composed of one or more components, and performs a business need. +DCAE Services run on the DCAE platform. A service either monitors the virtualized network services or does analytics. A service is composed of one or more components, and performs a business need. Services are created in a 'Service Design and Creation' tool called 'Service Assurance Flow Designer' by a Service Designer. This is done by -* 1. 'compose'ing a service from the available SDC catalog components (actually from the TOSCA models representing the components), then -* 2. 'submit'ing the service, which generates a Cloudify blueprint, is then automatically uploaded to INVENTORY, and then deployed by DEPLOYMENT HANDLER (once a DTI Event is triggered for the service). It should be noted that some service flows, specifally 'closed-loop' flows, are not initiated by DTI, but are done by CLAMP. + +* 'compose'ing a service from the available SDC catalog components (actually from the TOSCA models representing the components), then +* 'submit'ing the service, which generates a Cloudify blueprint, which is then automatically uploaded to INVENTORY, and then deployed by DEPLOYMENT HANDLER (and CLOUDIFY) (once a DTI Event is triggered for the service). It should be noted that some service flows, specifally 'closed-loop' flows, are not initiated by DTI, but are done by CLAMP. + +Only a few services are supported in SDC so far, and therefore steps above are done manually by the Onboarding Team for most services. + diff --git a/platformdoc/docs/changelog.md b/platformdoc/docs/changelog.md new file mode 100644 index 00000000..fbd7524a --- /dev/null +++ b/platformdoc/docs/changelog.md @@ -0,0 +1,19 @@ +# DCAE Platform Documentation Change Log + +## Versions + +Version | Date Built | +---- | ----- | +Current | 12/20/2017 +Previous | 11/30/2017 + +## Change Log + +Date | Changes | +---- | ----- | +12/20/2017 | updated for DTI, made corrections, add example component spec, added to concept, walkthrough section +11/30/2017 | updated for comments, updated configuration grid, added information about DTI development and configuration, updated Auxilary section, new component_properties section added to component spec, updated component jsonschema for DTI, this creates a new version - v4. (In order to utilize the new schema version, a dcae_cli --reinit needs to be done) +11/14/2017 | added ChangeLog, revised site outline, updated glossary, added dcae_cli commands section detailing each command, added various notes, added specific environment profiles for 1710 and 1802, made major revisions on many pages, added several concepts, added more detailed steps to walk-thru section, added more details about Dmaap Connection Object +10/03/2017 | added configuration grid and examples grid, updated common, cdap and docker specifications, and cdap and docker requirements pages. +09/28/2017 | added streams grid and services page, updated index, overview, and common spec pages + diff --git a/platformdoc/docs/components/component-specification/cdap-specification.md b/platformdoc/docs/components/component-specification/cdap-specification.md index d4d01406..ce17d811 100644 --- a/platformdoc/docs/components/component-specification/cdap-specification.md +++ b/platformdoc/docs/components/component-specification/cdap-specification.md @@ -11,11 +11,11 @@ The CDAP component specification contains the following groups of information. M ## Current Limitations and TODOs -* The integration of DMD is likely to significantly change the [Interfaces](#interfaces) section in this specification, see [DMaaP abstraction](/components/component-type-cdap.md#dmaap-abstraction). +* The integration of DMD is likely to significantly change the [Interfaces](common-specification.md#interfaces) section in this specification.. ## Parameters -There is a `parameters` section in your component specification. This section contains three optional keys: [app_config](#appconfig), [app_preferences](#apppreferences), and [propram_preferences](#program_preferences): +There is a `parameters` section in your component specification. This section contains three optional keys: [app_config](#appconfig), [app_preferences](#apppreferences), and [program_preferences](#programpreferences): ``` "parameters" : { "app_config" : [ ...], @@ -89,7 +89,7 @@ Each `program_pref` JSON is passed into the CDAP API as the preference for `prog NOTE: for CDAP, this section is very likely to change when DMD is available. -The _future_ vision, as per [DMaaP intentionally abstracted](/components/component-type-cdap.md#dmaap-abstraction) is that you would publish your data as a series of files on HDFS, and DMD will pick them up and send them to the appropriate DMaaP feeds or directly when needed. +The _future_ vision is that you would publish your data as a series of files on HDFS, and DMD will pick them up and send them to the appropriate DMaaP feeds or directly when needed. ## Auxiliary Details diff --git a/platformdoc/docs/components/component-specification/common-specification.md b/platformdoc/docs/components/component-specification/common-specification.md index 06f7f76c..14c1297c 100644 --- a/platformdoc/docs/components/component-specification/common-specification.md +++ b/platformdoc/docs/components/component-specification/common-specification.md @@ -2,6 +2,16 @@ This page describes the component specification (JSON) sections that are common to both Docker and CDAP components. Differences for each are pointed out below. Elements that are very different, and described in the CDAP or Docker specific pages. +## Meta Schema Definition + +The component specification is represented (and validated) against this +[Component Spec json schema](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/component-json-schemas/browse/component-spec-schema.json) and described below: + +The "Meta Schema" implementation defines how component specification JSON schemas can be written to define user input. It is itself a JSON schema (thus it is a "meta schema"). It requires the name of the component entry, component type (either 'cdap' or 'docker') and a description under "self" object. The meta schema version must be specified as the value of the "version" key. Then the input schema itself is described. + +The table below shows the four types of schema descriptions supported +There are four types of schema descriptions objects - jsonschema for inline standard JSON Schema definitions of JSON inputs, delimitedschema for delimited data input using a JSON description defined by AT&T, unstructured for unstructured text, and reference that allows a pointer to another artifact for a schema. The reference allows for XML schema, but can be used as a pointer to JSON, Delimited Format, and Unstructured schemas as well. + ## Component Metadata Metadata refers to the properties found under the `self` JSON. This group of properties is used to uniquely identify this component specification and identify the component that this specification is used to capture. @@ -32,6 +42,9 @@ Interfaces are the JSON objects found under the `streams` key and the `services` * The `streams` JSON is for specifying data produced for consumption by other components, and the streams expected to subscribe to that is produced by other components. These are "fire and forget" type interfaces where the publisher of a stream does not expect or parse a response from the subscriber. * The term `stream` here is abstract and neither refers to "CDAP streams" or "DMaaP feeds". While a stream is very likely a DMaaP feed, it could be a direct stream of data being routed via HTTP too. It abstractly refers to a sequence of data leaving a publisher. * Streams have anonymous publish/subscribe semantics, which decouples the production of information from its consumption. +Like the component specification, the data format specification is represented/validated against this +[Data Format json schema](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/component-json-schemas/browse/data-format-schema.json) + * In general, components are not aware of who they are communicating with. * Instead, components that are interested in data, subscribe to the relevant stream; components that generate data publish to the relevant stream. * There can be multiple publishers and subscribers to a stream. Streams are intended for unidirectional, streaming communication. @@ -93,7 +106,7 @@ Message router subscribers are http clients rather than http services and perfor ##### Data router -Data router subscribers are http or https services that handle `PUT` requests from data router. Developers must provide the `route` or url path/endpoint that is expected to handle data router requests. This will be used to construct the delivery url needed to register the subscriber to the provisioned feed. Developers must also provide a `config_key` because there is dynamic configuration information associated with the feed that the application will need e.g. username and password. See the page on [DMaaP connection objects](../dcae-cli/dmaap-connection-objects) for more details on the configuration information. +Data router subscribers are http or https services that handle `PUT` requests from data router. Developers must provide the `route` or url path/endpoint that is expected to handle data router requests. This will be used to construct the delivery url needed to register the subscriber to the provisioned feed. Developers must also provide a `config_key` because there is dynamic configuration information associated with the feed that the application will need e.g. username and password. See the page on [DMaaP connection objects](/components/component-specification/dmaap-connection-objects/#data-router) for more details on the configuration information. Example (not tied to the larger example): @@ -140,7 +153,7 @@ type | string | *Required*. Type of stream: `http`, `message router`, `data rout ##### Message router -Message router publishers are http clients of DMaap message_router. Developers must provide a `config_key` because there is dynamic configuration information associated with the feed that the application will need to receive e.g. topic url, username, password. See the page on [DMaaP connection objects](../dcae-cli/dmaap-connection-objects/#message_router) for more details on the configuration information. +Message router publishers are http clients of DMaap message_router. Developers must provide a `config_key` because there is dynamic configuration information associated with the feed that the application needs to receive e.g. topic url, username, password. See the page on [DMaaP connection objects](/components/component-specification/dmaap-connection-objects/#message-router) for more details on the configuration information. Example (not tied to the larger example): @@ -158,7 +171,7 @@ Example (not tied to the larger example): ##### Data router -Data router publishers are http clients that make `PUT` requests to data router. Developers must also provide a `config_key` because there is dynamic configuration information associated with the feed that the application will need to receive e.g. publish url, username, password. See the page on [DMaaP connection objects](../dcae-cli/dmaap-connection-objects) for more details on the configuration information. +Data router publishers are http clients that make `PUT` requests to data router. Developers must also provide a `config_key` because there is dynamic configuration information associated with the feed that the application will need to receive e.g. publish url, username, password. See the page on [DMaaP connection objects](/components/component-specification/dmaap-connection-objects/#data-router) for more details on the configuration information. Example (not tied to the larger example): @@ -313,9 +326,9 @@ type | string | The required data type for the parameter | required | boolean | An optional key that declares a parameter as required (true) or not (false) | true constraints | array | The optional list of sequenced constraint clauses for the parameter. See below | entry_schema | string | The optional key that is used to declare the name of the Datatype definition for entries of set types such as the TOSCA 'list' or 'map'. Only 1 level is supported at this time | -designer_editable | boolean | An optional key that declares a parameter to be editable by designer (true) or not (false) | true -sourced_at_deployment | boolean | An optional key that declares a parameter's value to be assigned at deployment time (true) | false -policy_editable | boolean | An optional key that declares a parameter to be editable by policy (true) or not (false) | true +designer_editable | boolean | An optional key that declares a parameter to be editable by designer (true) or not (false) | +sourced_at_deployment | boolean | An optional key that declares a parameter's value to be assigned at deployment time (true) | +policy_editable | boolean | An optional key that declares a parameter to be editable by policy (true) or not (false) | policy_schema | array | The optional list of schema definitions used for policy. See below | Example: @@ -365,21 +378,8 @@ entry_schema | string | The optional key that is used to declare the name of the ## Generated Application Configuration -The above example for component `asimov.component.kpi_anomaly` will get transformed into the following application configuration JSON that is fully resolved and provided at runtime by calling the `config binding service`: - -```json -{ - "streams_publishes": { - "prediction": ["10.100.1.100:32567"] - }, - "streams_subscribes": {}, - "threshold": 0.75, - "services_calls": { - "vnf-db": ["10.100.1.101:32890"] - } -} -``` - +The Platform generates configuration for the component (based on the component spec) at deployment time. The generated application configuration will be made up of the Parameters, Streams, and Services sections, after any provisioning for streams and services. The component developer can see what this configuration will look like by reviewing the [component dev command](/components/dcae_cli/commands/#run-the-dev-command). + ## Artifacts `artifacts` contains a list of artifacts associated with this component. For Docker, this is the full path (including the registry) to the Docker image. For CDAP, this is the full path to the CDAP jar. @@ -395,3 +395,8 @@ Property Name | Type | Description uri | string | *Required*. Uri to the artifact, full path type | string | *Required*. `docker image` or `jar` +# Working with Component Specs + +Components can be added to the onboarding catalog (which first validates the component spec) by using the [dcae_cli Tool](http://dcae-platform.research.att.com/components/dcae-cli/quickstart/). Here you can also list the components, show the contents of a component, publish the component, validate the generated configuration for the component, deploy (run) and undeploy the component. For a list of these capabilities, see [Component Commands](/components/dcae-cli/commands/#component). + + diff --git a/platformdoc/docs/components/component-specification/configuration-grid.md b/platformdoc/docs/components/component-specification/configuration-grid.md index e9054593..e1154218 100644 --- a/platformdoc/docs/components/component-specification/configuration-grid.md +++ b/platformdoc/docs/components/component-specification/configuration-grid.md @@ -1,13 +1,19 @@ # Configuration Quick Reference -The following types of configuration are supported by the DCAE Platform. - -|Input Sources|Default Values|Designer Input |Clamp Input|Policy Input|Runtime Input| -|---------|---------|----------|---------|----------|---| -|Notes||This applies only to components that are self-service (supported by SDC) |This applies only to components that are part of a closed-loop interface || | -|Who provides?|Component Developer|Service Designer|CLAMP|Operations|Runtime Platform| -| When/Where it is provided |During onboarding – in the component specification | At design time – in the SDC UI | At installation – in the CLAMP UI | Anytime – in the POLICY GUI | When the component is deployed | -|Component Specification Details|For CDAP:<br> ‘value’ Name and KV pairs in AppConfig or AppPreferences For Docker:<br> ‘value’ is provided for variable in ‘parameter’ section|‘designer-editable’ must be set to ‘true’ for variable in ‘parameter’ section. || ‘policy-editable’ must be set to ‘true’ and ‘policy_schema’ must be provided for variable in ‘parameter’ section |'sourced_at_<br>deployment' must be set to 'true' for variable in 'parameter' section | -| How it is used | This is passed to the component in the generated configuration if not overridden.|This overrides any values previously set, but can be overridden by CLAMP or POLICY.|This overrides any values previously set, but can be overridden by POLICY.|This overrides any values previously set, but can be overridden at any point thereafter.|This overrides any values previously set, but can be overridden at any point thereafter by Policy. | -| Additional Info for Component Developer|For CDAP:<br> ‘value’ is provided for variable in the ‘AppConfig’ or ‘AppPreferences’ sections<br><br> For Docker:<br> ‘value’ is provided for variable in ‘parameter’ section|||For Docker: In the auxiliary section:<br> {"policy": {"trigger_type": "policy","script_path": “/opt/app/reconfigure.sh”} }<br> Script interface must be "opt/app/reconfigure.sh” $trigger_type $updated_policies $updated_appl_config" <br> where $updated_policies is a json provided by the Policy Handler and <br> $update_appl_config is the post-merged appl config which may contain unresolved configuration that didn’t come from policy.<br> Suggestion is for script to call CONFIG BINDING SERVICE to resolve any configuration. | +#### Default Values + +The component developer can provide default values for any `parameter` in the component specification. These defaults will be passed to the component in its generated configuration. + +#### Overridden/Entered Values + +Depending on the other properties set for the parameter, the default value can be overridden at 'design-time', 'deploy-time' or once the microservice is running ('run-time'). +(*In the future, when Policy is supported, configuration will also be able to be provided/changed in the Policy UI at any time).* + +||Design-Time Input |CLAMP Input|Policy Input (future) |Deploy-Time Input|Run-Time Input (DTI)| +|-----|-----|-----|-----|----------|----------| +|Description|Applies to SDC self-service components|Applies to components deployed by CLAMP |(not yet supported)|Applies to manually deployed services| Applies to components supporting DTI reconfiguration| +|Input provided by|Service Designer|CLAMP|Operations|DevOps|Runtime Platform(DTI)| +|How it is provided |In the SDC UI |In the CLAMP UI |In the POLICY GUI |In the DCAE Dashboard (or Jenkins job)|In the DTI Event +|Component Specification Details|‘designer-editable’ set to ‘true’| None. Developer provides CLAMP an email with parameters to be supported|‘policy_editable’ must be set to ‘true’ and ‘policy_schema’ must be provided|'sourced_at_<br>deployment' must be set to 'true'|parameter 'dcae_target_type' defined with default value set to supported vnfType-vnfFuncId, with properties 'designer_editable' and 'sourced_at_deployment' set appropriately| +|Additional Info for Component Developer|||For Docker only: In the auxiliary section:<br> {"policy": {"trigger_type": "policy","script_path": “/opt/app/reconfigure.sh”} }<br> Script interface would then be "/opt/app/reconfigure.sh” $trigger_type $updated_policy" <br> where $updated_policy is json provided by the Policy Handler.||For Docker only: In the auxiliary section:<br> {"dti": “/opt/app/reconfigure.sh”} <br> Script interface would then be "/opt/app/reconfigure.sh” $trigger_type $updated_dti" <br> where $updated_dti is json provided by the DTI Plugin.| diff --git a/platformdoc/docs/components/dcae-cli/dmaap-connection-objects.md b/platformdoc/docs/components/component-specification/dmaap-connection-objects.md index cf03ccec..b1b650fd 100644 --- a/platformdoc/docs/components/dcae-cli/dmaap-connection-objects.md +++ b/platformdoc/docs/components/component-specification/dmaap-connection-objects.md @@ -1,19 +1,14 @@ # DMaaP connection objects -DMaaP connection objects are JSON objects that: +DMaaP Connection Objects are JSON objects that: -1. Components should expect at runtime in their application configuration and is to be used to connect to the appropriate DMaaP feed or topic. -2. Developers must provide through the command-line argument `--dmaap-file` to test their component with manually provisioned feeds and topics. +1. At Runtime - this is generated by the DCAE Platform and passed to the component in its application_configuration to be used to connect to the DMaaP feed or topic. Components will receive the entire object with all properties populated (default will be `null) unless specified otherwise. +2. During dcae_cli testing - this is provided through the command-line argument `--dmaap-file` to test the component with manually provisioned feeds and topics. Developers are not required to provide the entire object. The required properties are labeled below with "*Required as input*". -This page is a reference to the specific structure that each type of DMaaP stream requires. +## Message Router -Note for #1 that components should expect the entire object with all properties at runtime where the default will be `null` unless specified otherwise. - -Note for #2 that developers are not required to provide the entire object. The required properties will be labeled with "*required as input*". - -## Message router - -Publishers and subscribers both have the same JSON object structure. Here's an example: +Publishers and subscribers have the same generated `Dmaap Connection Object` structure. Here's an example for any given config-key: +(This is what will be in application_configuration) ```json { @@ -47,7 +42,7 @@ client_id | string | Client id for given AAF client location | string | DCAE location for the publisher or subscriber, used to set up routing topic_url | string | *Required as input*. URL for accessing the topic to publish or receive events -Here's an example of the minimal JSON that must be provided as an input: +The --dmaap-file argument (to the component `run` or `dev` command), must minimally contain: ```json { @@ -58,11 +53,12 @@ Here's an example of the minimal JSON that must be provided as an input: } ``` -## Data router +## Data Router ### Publisher -Here's an example of what the JSON object connection for data router publisher looks like: +Here's an example of what the generated `Dmaap Connection Object` for Data Router Publisher looks like: +(This is what will be in application_configuration) ```json { @@ -83,7 +79,7 @@ At the top-level: Property Name | Type | Description ------------- | ---- | ----------- type | string | *Required as input*. Must be `data_router` for data router feeds -dmaap_info | JSON object | *Required as input*. Contains the topic connection details +dmaap_info | JSON object | *Required as input*. Contains the feed connection details The `dmaap_info` object contains: @@ -96,7 +92,7 @@ username | string | Username the publisher uses to authenticate to Data Router password | string | Password the publisher uses to authenticate to Data Router publisher_id | string | Publisher id in Data Router -Here's an example of the minimal JSON that must be provided as an input: +The --dmaap-file argument (to the component `run` or `dev` command), must minimally contain: ```json { @@ -109,7 +105,8 @@ Here's an example of the minimal JSON that must be provided as an input: ### Subscriber -Here's an example of what the JSON object connection for data router subscriber looks like: +Here's an example of what the generated `Dmaap Connection Object` for a Data Router Subscriber looks like: +(This is what will be passed in application_configuration) ```json { @@ -129,19 +126,19 @@ At the top-level: Property Name | Type | Description ------------- | ---- | ----------- type | string | *Required as input*. Must be `data_router` for data router feeds -dmaap_info | JSON object | *Required as input*. Contains the topic connection details +dmaap_info | JSON object | *Required as input*. Contains the feed connection details The `dmaap_info` object contains: Property Name | Type | Description ------------- | ---- | ----------- -location | string | DCAE location for the publisher, used to set up routing +location | string | DCAE location for the subscriber, used to set up routing delivery_url | string | URL to which the Data Router should deliver files username | string | Username Data Router uses to authenticate to the subscriber when delivering files password | string | Password Data Router uses to authenticate to the subscriber when delivering files subscriber_id | string | Subscriber id in Data Router -Here's an example of the minimal JSON that must be provided as an input: +The --dmaap-file argument (to the component `run` or `dev` command), must minimally contain: ```json { @@ -151,6 +148,22 @@ Here's an example of the minimal JSON that must be provided as an input: } ``` -Developers are recommended to use `username` and `password` since this is the recommended security practice. +It is the recommended security practice that `username` and `password` are specified. +You cannot provide the `delivery_url` in your dmaap-file, because it's not constructed until deployment time. Therefore, after the test deployment, go back to the Data Router Feed and provide the delivery_url (in order to start receiving the feeds). + +### Data Router Example + +(After the Data Router feed has been manually provisioned) + +``` +$ dcae_cli component run --dmaap-file $dmaap_file $component-name +DCAE.Run | WARNING | Your component is a data router subscriber. Here are the delivery urls: + + some-sub-dr: http://135.205.226.128:32838/identity +``` + +(Update the Data Router Feed to provide the delivery_url). + + + -Note that the dcae-cli will construct the `delivery_url` when deploying the component since this can only be known at deployment time. diff --git a/platformdoc/docs/components/component-specification/docker-specification.md b/platformdoc/docs/components/component-specification/docker-specification.md index 79d3a88b..60905046 100644 --- a/platformdoc/docs/components/component-specification/docker-specification.md +++ b/platformdoc/docs/components/component-specification/docker-specification.md @@ -10,14 +10,16 @@ The Docker component specification contains the following groups of information. ## Auxiliary Details -`auxiliary` contains Docker specific details like health check, port mapping, volume mapping, and policy reconfiguration script details. +`auxiliary` contains Docker specific details like health check, port mapping, volume mapping, dti and policy reconfiguration script details. (Policy reconfiguration is not yet supported). Name | Type | Description ------------- | ---- | ----------- healthcheck | JSON object | *Required*. Health check definition details ports | JSON array | each array item maps a container port to the host port. See example below. -volume | JSON array | each array item contains a host and container object. See example below. | | -policy | JSON array | *Required*. Policy script details +volume | JSON array | each array item contains a host and container object. See example below. +reconfigs | string | DTI reconfiguration script details +*Planned for 1806* | | +policy | JSON array | *Required*. Policy reconfiguration script details ### Health Check Definition @@ -25,7 +27,7 @@ The platform uses Consul to perform periodic health check calls. Consul provide When choosing a value for interval, consider that too frequent healthchecks will put unnecessary load on Consul and DCAE. If there is a problematic resource, then more frequent healthchecks are warranted (eg 15s or 60s), but as stablility increases, so can these values, (eg 300s). -When choosing a value for timeout, consider that too small a number will result in increasing timeout failures, and too large a number will result in a delay in the notification of resource problem. A suggestion is to start with 5s and workd from there. +When choosing a value for timeout, consider that too small a number will result in increasing timeout failures, and too large a number will result in a delay in the notification of the resource problem. A suggestion is to start with 5s and work from there. #### http @@ -75,6 +77,8 @@ Example: ### Ports +This method of exposing/mapping a local port to a host port is NOT RECOMMENDED because of the possibility of port conflicts. If multiple instances of a docker container will be running, there definitely will be port conflicts. Use at your own risk. (The preferred way to expose a port is to do so in the Dockerfile as described [here](/components/component-type-docker/#ports)). + ```json "auxilary": { "ports": ["8080:8000"] @@ -140,7 +144,35 @@ Here's an example of the minimal JSON that must be provided as an input: In the example above, the container volume "/tmp/docker.sock" maps to host volume "/var/run/docker.sock". -### Policy +### DTI Reconfiguration + +DTI changes will be provided to the Docker component by triggering a script that is defined here. + +Property Name | Type | Description +------------- | ---- | ----------- +dti | string | *Required*. Suggested value is "/opt/app/reconfigure.sh" + +Example: + +```json +"auxilary": { + "dti": "/opt/app/reconfigure.sh" +} +``` + +The docker script interface is as follows: <br><br> `/opt/app/reconfigure.sh $reconfigure_type {<updated_dti object>} + +Name | Type | Description +---- | ---- | ----------- +reconfigure_type | string | "dti" +updated_dti | json | dti_event object + +The dti_event object can be seen [here](https://codecloud.web.att.com/projects/ST_DCAE/repos/com.att.dcae.orch.dti-handler/browse/dti_inputs.yaml). + +An example of a DTI reconfiguration script can be found [here](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/docker-cloudify/browse/examples/reconfigure.py?at=refs%2Fheads%2Frelease%2F1802). + + +### Policy (not yet supported) Policy changes made in the Policy UI will be provided to the Docker component by triggering a script that is defined here. @@ -233,7 +265,7 @@ updated_appl_config | json | complete generated app_config, not fully-resolved, } }, "artifacts": [{ - "uri": "fake.nexus.com/dcae/kpi_anomaly:1.0.0", + "uri": "fake.nexus.att.com/dcae/kpi_anomaly:1.0.0", "type": "docker image" }] } diff --git a/platformdoc/docs/components/component-specification/examples-grid.md b/platformdoc/docs/components/component-specification/examples-grid.md new file mode 100644 index 00000000..9597e537 --- /dev/null +++ b/platformdoc/docs/components/component-specification/examples-grid.md @@ -0,0 +1,18 @@ +# Component Spec Examples + +The following table shows CDAP and Docker component spec examples for some of the commonly used platform features. Please note that these examples were chosen for the specific features that they contain, and should not necessarily be used for examples of other features, as they may have been captured before they were complete or modified to best utilize those features. + +| Feature | CDAP Example | Docker Example | +|---------|--------------|----------------| +| Dmaap MR subscriber | [String Matching](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/sm/sm_spec.json) | TBD | +| Dmaap MR publisher | [String Matching](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/sm/sm_spec.json) | [Syslog spec](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/syslog/syslog.json) | +| Dmaap DR subscriber/DMD | TBD | [Sessionization](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/sessionization/sessionization-componentspec.json) | +| Dmaap DR publisher/DMD | TBD | [Sessionization](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/sessionization/sessionization-componentspec.json) | +| Policy (simple) | [tca_policy2](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/tca_with_policy2/tca_with_policy2.json) | TBD | +| Policy (complex) | [tca_policy](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/tca_with_policy/tca_with_policy.json) | TBD | +| Healthcheck | N/A (the platform does this) | [FOI spec](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/foi/foisftpcollector_componentspec.json) +| Service Calls | [String Matching](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/sm/sm_spec.json) | TBD | +| Service Provides | [cuda](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/cuda/spec.json) | [VES](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/ves/vescollector-componentspec.json) | +| DTI Reconfig | TBD | [FOI-sftp](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/foi-sftp/component_spec.json) | + + diff --git a/platformdoc/docs/components/component-specification/generated-configuration.md b/platformdoc/docs/components/component-specification/generated-configuration.md deleted file mode 100644 index 258358de..00000000 --- a/platformdoc/docs/components/component-specification/generated-configuration.md +++ /dev/null @@ -1,91 +0,0 @@ -# Generated configuration - -The DCAE platform relies on the component specification to generate the component's application configuration JSON at deployment time. The component developer should expect to use this configuration JSON in their application to provision themselves. - -Pro-tip: As you build your component specification, you can use the [dcae-cli `dev` command](../dcae-cli/walkthrough/#dev) to view what the resulting application configuration will look like. - -## Streams and services - -For both Docker and CDAP, when your component is deployed, any `streams` and `services/calls` you specified will be injected into your configuration under the following well known structure. -Your component is required to parse this information if you have any connectivity to DMaaP or are calling another DCAE component. - -More details about the DMaaP connection objects are found [here](../dcae-cli/dmaap-connection-objects/). - -This is best served with an example. - -The following component spec snippet (from String Matching): -``` -"streams":{ - "subscribes": [{ - "format": "VES_specification", - "version": "4.27.2", - "type": "message_router", - "config_key" : "mr_input" - }], - "publishes": [{ - "format": "VES_specification", - "version": "4.27.2", - "config_key": "mr_output", - "type": "message_router" - }] - }, - "services":{ - "calls": [{ - "config_key" : "aai_broker_handle", - "verb": "GET", - "request": { - "format": "get_with_query_params", - "version": "1.0.0" - }, - "response": { - "format": "aai_broker_response", - "version": "3.0.0" - } - }], - "provides": [] - }, -``` - -Will turn into the following top level keys in your configuration (for CDAP, this will be under AppConfig) - -``` - "streams_publishes":{ - "mr_output":{ // notice the config key above - "aaf_password":"XXX", - "type":"message_router", - "dmaap_info":{ - "client_role": null, - "client_id": null, - "location": null, - "topic_url":"XXX" - }, - "aaf_username":"XXX" - } - }, - "streams_subscribes":{ - "mr_input":{ // notice the config key above - "aaf_password":"XXX", - "type":"message_router", - "dmaap_info":{ - "client_role": null, - "client_id": null, - "location": null, - "topic_url":"XXX" - }, - "aaf_username":"XXX" - } - }, - "services_calls":{ - "aai_broker_handle":[ // notice the config key above - "SOME_IP:32768" // based on deployment time, just an example - ] - } -``` -These keys will always be populated regardless of whether they are empty. So the minimal you will get, in the case of a component that provides an HTTP service and does not call any services and has no streams, is: -``` - "streams_publishes":{}, - "streams_subscribes":{}, - "services_calls":{} -``` - -Thus your component should expect these well-known top level keys. diff --git a/platformdoc/docs/components/component-specification/start-here.md b/platformdoc/docs/components/component-specification/start-here.md deleted file mode 100644 index e69de29b..00000000 --- a/platformdoc/docs/components/component-specification/start-here.md +++ /dev/null diff --git a/platformdoc/docs/components/component-type-cdap.md b/platformdoc/docs/components/component-type-cdap.md index 253dd2f8..8658b2da 100644 --- a/platformdoc/docs/components/component-type-cdap.md +++ b/platformdoc/docs/components/component-type-cdap.md @@ -6,9 +6,9 @@ This page contains information about CDAP app development in DCAE. The DCAE component specification has you input your `jar_url`, the URL on Nexus to your Jar file. This DCAE controller deploys out of Nexus. You can upload your jar(s) using the following command, replacing NAME: ``` -curl -v --user 'user:password' http://YOUR_NEXUS_RAW_REPO/NAME.jar --upload-file NAME.jar +curl -v --user 'dcae-dev:dev123' http://nexus01.research.att.com:8081/repository/dcae-dev-raw/jar_files/NAME.jar --upload-file NAME.jar ``` -During the CLI Tool Usage, in your spec, supply `http://YOUR_NEXUS_RAW_REPO/NAME.jar` as the JAR artifact URL. +During the CLI Tool Usage, in your spec, supply `http://nexus01.research.att.com:8081/repository/dcae-dev-raw/jar_files/NAME.jar` as the JAR artifact URL. ## Policy Reconfiguration We support reconfiguration of both AppConfig and AppPreferences. diff --git a/platformdoc/docs/components/component-type-docker.md b/platformdoc/docs/components/component-type-docker.md index fd3358dc..6e717a5e 100644 --- a/platformdoc/docs/components/component-type-docker.md +++ b/platformdoc/docs/components/component-type-docker.md @@ -4,20 +4,25 @@ Component developers are required to provide artifacts for the platform to be able to deploy your component including: -* [Component specification](component-specification/docker-specification) -* [Data formats](data-formats) -* [Auxiliary component specification](component-specification/docker-specification#auxiliary) +* [Component Specification](component-specification/docker-specification) +* [One or more Data Formats](data-formats) *unless they already exist * [Docker image](#docker-on-the-platform) In addition, components will have to be enhanced to be compliant with the DCAE platform in order to correctly be deployed and be managed. This page will discuss the changes which are grouped into the following categories: -* [Service registration](#service-registration) -* [Configuration management on the new platform](#configuration) -* [Operational concerns](#operational-concerns) +* [Service Registration](#service-registration) +* [Configuration Management](#configuration-management) +* [Docker on the Platform](#docker-on-the-platform) +* [Operational Concerns](#operational-concerns) -To help component developers to make and to test the changes needed to have components run on the platform, a command-line tool called [`dcae-cli`](dcae-cli/quickstart) is provided by the platform team. +Additional considerations are: -## Service registration +* [DTI Reconfiguration](#dti-reconfiguration) +* [Policy Reconfiguration](#policy-reconfiguration) + +To help component developers to make and to test the changes needed to have components run on the platform, a command-line tool called [`dcae-cli`](dcae-cli/quickstart) is provided by the platform team. (Testing withing the dcae_cli tool is not yet available for DTI Reconfiguration or Policy). + +## Service Registration Every [Docker component is registered](../architecture/service-discovery) with the platform's service discovery layer. Docker components are not expected to make the explicit make registration calls because that is done by through a platform 3rd party registration service. A couple things are needed from component developers in order for this registration to occur successfully: @@ -28,26 +33,33 @@ Every [Docker component is registered](../architecture/service-discovery) with t Components that listen on a specific port must explicitly declare in their Dockerfile that port using the `EXPOSE` instruction before building the image. Warning! At the current time, you can not expose multiple ports in your Dockerfile or registration *will not work* correctly. +Warning! Be sure to choose a port that is available. This may vary by environment. ### Health check Component developers are required to provide a way for the platform to periodically check the health of their running components. The platform uses Consul to perform these periodic calls. Consul provides different types of [check definitions](https://www.consul.io/docs/agent/checks.html). The details of the definition used by your component is to be provided through the [Docker auxiliary specification](component-specification/docker-specification#auxiliary). -## Configuration +## Configuration Management + +All configuration for a component is stored in CONSUL under the components uniquely generated name which is provided by the environment variable `HOSTNAME` as well as `SERVICE_NAME`. It is then made available to the component via a remote HTTP service call to CONFIG BINDING SERVICE. + +The main entry in CONSUL for the component contains its `generated application configuration`. This is based on the submitted component specification, and consists of the `interfaces` (streams and services/calls) and `parameters` sections. Other entries may exist as well, under specific keys, such as :dmaap or :dti. Each key represents a specific type of information and is also available to the component by calling CONFIG BINDING SERVICE. More on this below. + +Components are required to pull their `generated application configuration` at application startup. The component must provide an initialization script that retrieves the application configuration and reference that script in its Dockerfile. Other calls can be made to CONFIG BINDING SERVICE to retrieve DMaaP, DTI Reconfiguration, or Pollicy Reconfiguration (not yet supported). -The component's application configuration is generated from the submitted component specification into a JSON representation. The DCAE platform will provide the configuration JSON by making it available on a remote HTTP service. Components are required to pull their configuration JSON at application startup. The configuration JSON is stored under the components uniquely generated name which is provided by the environment variable `HOSTNAME`. +You can see more details on the generated application configuration [here](/components/dcae-cli/walkthrough/#view-the-platform-generated-configuration) -You can see more details on the generated application configuration [here](component-specification/docker-specification#generate-application-configuration). +### Config Binding Service +The config binding service is a platform HTTP service that is responsible for providing clients with its fully resolve configuration JSON at startup, and also other configurations objects (such as :dti) when requested. -### Config binding service -The config binding service is a platform HTTP service that is responsible for providing clients with a fully resolved configuration JSON at runtime. Components should make an HTTP GET on: +At runtime, components should make an HTTP GET on: ``` <config binding service hostname>:<port>/service_component/NAME ``` -For Docker components, NAME should be set to `HOSTNAME`, which was provided as an ENV variable inside of your container. +For Docker components, NAME should be set to `HOSTNAME`, which is provided as an ENV variable to the container. -The binding service integrates with the streams and services section of your component specification. For example, if you specify that you call a service: +The binding service integrates with the streams and services section of the component specification. For example, if you specify that you call a service: ``` "services": { "calls": [{ @@ -64,7 +76,7 @@ The binding service integrates with the streams and services section of your com ... } ``` -Then the config binding service will find all available IP addresses of services meeting your needs, and provide them to you under your `config_key`: +Then the config binding service will find all available IP addresses of services meeting the containers needs, and provide them to the container under your `config_key`: ``` // your configuration { @@ -76,17 +88,209 @@ Regarding `<config binding service hostname>:<port>`, there is DNS work going on However, currently you will be given a name as an ENV variable, `CONFIG_BINDING_SERVICE`, and you will need to query Consul's service discovery to get `<config binding service hostname>:<port>`. -### Policy Reconfiguration -*(Draft and subject to change)* -Components must provide a way to receive policy reconfiguration, that is, configuration parameters that have been updated via the Policy UI. The component developer provides a docker script (defined in the [Docker auxiliary specification](component-specification/docker-specification#policy-example)) that will be triggered when this occurs. +### Generated Application Configuration + +The DCAE platform uses the component specification to generate the component's application configuration provided at deployment time. The component developer should expect to use this configuration JSON in the component. + +Pro-tip: As you build the component specification, use the [dcae-cli `dev` command](/components/dcae-cli/walkthrough/#view-the-platform-generated-configuration) to see what the resulting application configuration will look like. + +For both Docker and CDAP, when the component is deployed, any `streams` and `services/calls` specified, will be injected into the configuration under the following well known structure, along with all `parameters`. (`services/provides` is not passed in to the application config). Your component is required to parse this information if it has any DMaaP connections or interfaces with another DCAE component. + +This is best served by an example. + +The following component spec snippet (from String Matching): +``` +"streams":{ + "subscribes": [{ + "format": "VES_specification", + "version": "4.27.2", + "type": "message_router", + "config_key" : "mr_input" + }], + "publishes": [{ + "format": "VES_specification", + "version": "4.27.2", + "config_key": "mr_output", + "type": "message_router" + }] + }, + "services":{ + "calls": [{ + "config_key" : "aai_broker_handle", + "verb": "GET", + "request": { + "format": "get_with_query_params", + "version": "1.0.0" + }, + "response": { + "format": "aai_broker_response", + "version": "3.0.0" + } + }], + "provides": [] + }, +``` + +Will result in the following top level keys in the configuration (for CDAP, this will be under AppConfig) + +``` + "streams_publishes":{ + "mr_output":{ // notice the config key above + "aaf_password":"XXX", + "type":"message_router", + "dmaap_info":{ + "client_role": null, + "client_id": null, + "location": null, + "topic_url":"https://dcae-msrt-mtl5-ftl2.homer.att.com:3905/events/com.att.dcae.dmaap.FTL2.DCAE-CL-EVENT" // just an example + }, + "aaf_username":"XXX" + } + }, + "streams_subscribes":{ + "mr_input":{ // notice the config key above + "aaf_password":"XXX", + "type":"message_router", + "dmaap_info":{ + "client_role": null, + "client_id": null, + "location": null, + "topic_url":"https://dcae-msrt-ftl2.homer.att.com:3905/events/com.att.dcae.dmaap.FTL2.TerrysStringMatchingTest" // just an example + }, + "aaf_username":"XXX" + } + }, + "services_calls":{ + "aai_broker_handle":[ // notice the config key above + "135.205.226.128:32768" // based on deployment time, just an example + ] + } +``` +These keys will always be populated whether they are empty or not. So the minimum configuration you will get, (in the case of a component that provides an HTTP service, doesn't call any services, and has no streams, is: +``` + "streams_publishes":{}, + "streams_subscribes":{}, + "services_calls":{} +``` + +Thus your component should expect these well-known top level keys. + +### DTI Reconfiguration + +Most Collector components will support DTI reconfiguration. That is, they must be designed to process multiple instances of a particular `vnfType-vnfFuncId`. When instances of that vnfType-vnfFuncId` are brought up or down, the collectors `reconfiguration script` will be executed. The components reconfiguration script must be defined with the following interfact: + +``` +`/opt/app/reconfigure.sh” dti $updated_dti` +``` +where $updated_dti is a json for one vnfType-vnfFuncId instance that looks like this (for example). +Note: The reconfigure script does not have to be named 'reconfigure.sh'. + +For a deployment of VNF Instance +``` +{ + "deploy": { + "vhss-ems": { + "zrdm3avhss01ems001": { + "dcae_target_collection_ip": "107.239.223.191", + ...the remaining dti_input parameters... + } + } + } +} +``` +For an undeployment of VNF instance +``` +{ + "undeploy": { + "vhss-ems": [ + "zrdm3avhss01ems002" + ] + } +} +``` + +The component spec must contain the following: + +* In the auxilary section, add the definition for the above reconfiguration script for the 'reconfigs' property. This is the script that the platform will call with DTI input when a DTI event is received for the collector supporting the specific dcae_target_type. +* In the parameter section, define a parameter 'dcae_target_type' defined with properties 'designer_editable' and property 'sourced_at_deployment'. Set 'designer_editable' to true if this is an SDC Self-Service microservice. Otherwise, set it to false. Set 'sourced_at_deployment' to true if input can be provided at deployment time by Operations. Otherwise set it to false. + +The component spec can retrieve information about ALL the instances it supports by doing a curl command to CONFIG BINDING SERVICE like this: + +``` +curl http://<config binding service>:<port>/dti/$SERVICE_NAME +``` + +This would return the following: +``` +{ + "vhss-ems": { + "zrdm3avhss01ems001": { + "dcae_target_collection_ip": "107.239.223.191", + the rest of the DTI_input fields… + }, + "zrdm3avhss01ems002": { + "dcae_target_collection_ip": "107.239.223.192", + the rest of the DTI_input fields… + } + } +} +``` +The full list of DTI parameters can be found [here](https://codecloud.web.att.com/projects/ST_DCAE/repos/com.att.dcae.orch.dti-handler/browse/dti_inputs.yaml). + +(The API for the CONFIG BINDING SERVICE is): +``` + /dti/{service_component_name} + parameters: + name: "service_component_name" + in: "path" + description: "Service Component Name. service_component_name:dti must be a key in consul." (see +below for example output) + required: true + type: "string" + get: + description: "Returns as JSON the value for service_component_name:dti" + operationId: "config_binding_service.controller.dti" + responses: + 200: + description: OK; the KV value is returned as an object + schema: + type: object + 404: + description: there is no configuration in Consul for this component's DTI events +``` ### DMaaP -Components can be publishers or subscribers to either message router topics or data router feeds. This is defined in the component specification under the `streams` section where you can specify whether your component is expected to subscribe or to publish to a [message router topic](component-specification/common-specification/#message-router) or to a [data router feed](component-specification/common-specification/#data-router). Given a composition with components that use DMaaP, the platform will provision the topic or feed and provide the necessary [connection details](dcae-cli/dmaap-connection-objects) at runtime for each DMaaP dependent component. These connection details will be provided through your application's [generated configuration](component-specification/generated-configuration). +Components can be publishers or subscribers to either message router topics or data router feeds. This is defined in the component specification under the `streams` section where you can specify whether your component is expected to subscribe or to publish to a [message router](component-specification/common-specification/#message-router) topic or to a [data router](component-specification/common-specification/#data-router) feed. Given a composition with components that use DMaaP, the platform will provision the topic or feed and provide the necessary [connection details](/components/component-specification/dmaap-connection-objects) at runtime for each DMaaP dependent component. These connection details are provided through your application's generated configuration. In order to test DMaaP connections in onboarding, the developer (currently) must provision all test topics and feeds manually and provide the [dcae-cli with the connection details](dcae-cli/walkthrough/#dmaap-testing) when deploying your application. +Even thought the DMaaP connection information is included in the generated application configuration, it may be obtained by doing a call as in this example: + +``` +curl http://<config binding service>:<port>/dmaap/jm416b.d345ada1-cc31-4121-a741-9007b9f64808.1-0-1.dcae-collectors-cli-pm +``` + +This would return the following: +``` +{"cli_gamma_cisco_pm_config_stat": + { + "publish_url": "https://dcae-drps-ftl2.homer.att.com/publish/1362", + "username": "mtl5-0", + "log_url": null, + "location": "mtl5-0", + "password": "i5qji048hdm2e38f0bg872tnqd", + "publisher_id": "1234" + } +} +``` + +### Policy Reconfiguration +*(not yet supported)* + +Components must provide a way to receive policy reconfiguration, that is, configuration parameters that have been updated via the Policy UI. The component developer provides a docker script (defined in the [Docker auxiliary specification](component-specification/docker-specification#policy-example)) that will be triggered when this occurs. + ## Docker on the platform ### Images @@ -100,36 +304,39 @@ For example, an application called laika has been tagged for an example Nexus re ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -YOUR_NEXUS_DOCKER_REGISTRY/laika 0.4.0 154cc382df61 7 weeks ago 710.5 MB +nexus01.research.att.com:18443/dcae-platform/laika 0.4.0 154cc382df61 7 weeks ago 710.5 MB laika 0.4.0 154cc382df61 7 weeks ago 710.5 MB ``` -Note, the Docker registry that is used may require a login to authenticate. - +The solutioning evironment's Nexus host for the Docker registry is `nexus01.research.att.com:18443`. You must run `docker login nexus01.research.att.com:18443` to access the registry. Please contact the DCAE platform team to provide you with the credentials. ``` -docker login YOUR_NEXUS_DOCKER_REGISTRY +docker login nexus01.research.att.com:18443 ``` Tag your image: ``` -docker tag laika:0.4.0 YOUR_NEXUS_DOCKER_REGISTRY/laika:0.4.0 +docker tag laika:0.4.0 nexus01.research.att.com:18443/dcae-platform/laika:0.4.0 ``` Or build and tag: ``` -docker build -t YOUR_NEXUS_DOCKER_REGISTRY/laika:0.4.0 . +docker build -t nexus01.research.att.com:18443/dcae-platform/laika:0.4.0 . ``` After tagging, upload your image to the remote registry using the Docker [push command](https://docs.docker.com/engine/reference/commandline/push/). Note that the registry may require a login. Use the Docker [login command](https://docs.docker.com/engine/reference/commandline/login/) before pushing in that case. ``` -docker push YOUR_NEXUS_DOCKER_REGISTRY/laika:0.4.0 +docker push nexus01.research.att.com:18443/dcae-platform/laika:0.4.0 ``` -*NOTE* Replace `laika` with your application's name. Replace the `0.4.0` version with your application's version. +*NOTE* Replace `dcae-platform` with the group directory that is applicable to your image. Replace `laika` with your application's name. Replace the `0.4.0` version with your application's version. + +### Dockerfile + +The Dockerfile must contain the name of the container's initialization script. This will be called when the container is deployed, and must call Config Binding Service as described in [Config Binding Service](#config-binding-service) ### Ports @@ -146,7 +353,7 @@ Name | Type | Description `CONFIG_BINDING_SERVICE` | string | Hostname of the platform's config binding service instance `DOCKER_HOST` | string | Host of the target platform Docker host to run the container on -## Operational concerns +## Operational Concerns ### Logging @@ -154,25 +361,27 @@ Currently the platform uses the default `json-file` logging driver for Docker. To use the `docker logs` command for your deployed running Docker container, -1. You must have Docker installed on your local machine -2. Have the generated name of your component. This is generated for you when you execute `dcae_cli component dev` or `dcae_cli component run`. -3. Find the target Docker host using the `dcae_cli profiles show` command: +* You must have Docker installed on your local machine +* Have the generated name of your component. This is generated for you when you execute `dcae_cli component dev` or `dcae_cli component run`. +* Find the target Docker host using the `dcae_cli profiles show` command: ``` -$ dcae_cli profiles show YOUR_PROFILE_NAME +$ dcae_cli profiles show solutioning { - ... - "docker_host": "YOUR_DOCKER_HOST:2376" + "cdap_broker": "cdap_broker", + "config_binding_service": "config_binding_service", + "consul_host": "realsolcnsl00.dcae.solutioning.homer.att.com", + "docker_host": "realsolcpdokr00.dcae.solutioning.homer.att.com:2376" } ``` -4. Set your Docker client to point to the target Docker host: +* Set your Docker client to point to the target Docker host: ``` -$ export DOCKER_HOST="tcp://YOUR_DOCKER_HOST:2376" +$ export DOCKER_HOST="tcp://realsolcpdokr00.dcae.solutioning.homer.att.com:2376" ``` -5. Use the `docker logs` command: +* Use the `docker logs` command: ``` $ docker logs <generated component name> diff --git a/platformdoc/docs/components/data-formats.md b/platformdoc/docs/components/data-formats.md index 7d17e984..9dd0a65f 100644 --- a/platformdoc/docs/components/data-formats.md +++ b/platformdoc/docs/components/data-formats.md @@ -1,51 +1,38 @@ # Data Formats -Because the DCAE designer composes your component with others at service design time, in most cases you do not know what specific component(s) your component will send data to during runtime. -Thus, it is vital that DCAE has a language of describing the data passed between components, so that it is known which components are composable with others. -Data formats are descriptions of data---they are the data contract between your component and other components. -You need to describe the available outputs and assumed inputs of your components as data formats. -These data descriptions are onboarded into ASDC, and each receives a UUID. -If component X outputs data format DF-Y, and another component Z specifies DF-Y as their input data format, then X is said to be _composable_ with that component. -The data formats are referenced in the component specifications by the data format's id and version. -The vision is to have a repository of shared data formats that developers and teams can re-use and also provide them the means to extend and create new custom data formats. +Data formats are descriptions of data; they are the data contract between your component and other components. When the components are 'composed' into services in the SDC tool, they can only be matched with components that have compatible data formats. Data formats will be onboarded to SDC and assigned a UUID at that time. This UUID is then used to ensure compatibility amoung components. (If component X outputs data format 'DF-Y', and another component Z specifies 'DF-Y' as its input data format, then X is said to be `composable` with component Z). -# Meta Schema Definition +Since data formats will be shared across components, the onboarding catalog should be checked first to see if the desired data format is available before creating one. +The vision is to have a repository of shared data formats that developers and teams can re-use and also provide them the means to extend and create new custom data formats. +A data format is referenced by its data format id and version number. -The "Meta Schema" implementation defines how data format JSON schemas can be written to define user input. It is itself a JSON schema (thus it is a "meta schema"). It requires the name of the data format entry, the data format entry version and allows a description under "self" object. The meta schema version must be specified as the value of the "dataformatversion" key. Then the input schema itself is described. There are four types of schema descriptions objects - jsonschema for inline standard JSON Schema definitions of JSON inputs, delimitedschema for delimited data input using a defined JSON description, unstructured for unstructured text, and reference that allows a pointer to another artifact for a schema. The reference allows for XML schema, but can be used as a pointer to JSON, Delimited Format, and Unstructured schemas as well. +## JSON schema -The current Meta Schema implementation is defined at the link below. +The data format specification is represented (and validated) against this +[Data Format json schema](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/component-json-schemas/browse/data-format-schema.json) and described below: -[schema](ONAP URL TBD) +### Meta Schema Definition -#TCA Example +The "Meta Schema" implementation defines how data format JSON schemas can be written to define user input. It is itself a JSON schema (thus it is a "meta schema"). It requires the name of the data format entry, the data format entry version and allows a description under "self" object. The meta schema version must be specified as the value of the "dataformatversion" key. Then the input schema itself is described as one of the four types listed below: -TCA Input - Common Event Format by reference +Type | Description +---- | ----------- +jsonschema | inline standard JSON Schema definitions of JSON inputs +delimitedschema | delimited data input using a JSON description and defined delimiter +unstructured | unstructured text, and reference that allows a pointer to another artifact for a schema. +reference | allows for XML schema, but can be used to reference other JSON, delimitedschema, and unstructured schemas as well. -First the full JSON schema description of the Common Event Format would be loaded with a name of "Common Event Format" and the current version of "25.0.0". +## Example Schemas: -Then the data format description is loaded by the example at this link: +### `jsonschema` -[tcainput](ONAP URL TBD) +* [TCA output](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/tca_hi_lo/tcaoutput.json) -TCA Output JSON inline example: +* [CUDA simple example](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/cuda/simplejson.json) -[tcaoutput](ONAP URL TBD) +* [CUDA nested example](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/cuda/nestedjson.json) -#CUDA Example - -CUDA Simple JSON Example: - -[simplejson](ONAP URL TBD) - -CUDA Nested JSON Example: - -[nestedjson](ONAP URL TBD) - -CUDA Unstructured Example: - -[unstructuredtext](ONAP URL TBD) - -# An example of a delimited schema +### `delimitedschema` ``` { @@ -70,3 +57,18 @@ CUDA Unstructured Example: } } ``` + +### `unstructured` + +* [CUDA example](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/cuda/unstructuredtext.json) + +### `reference` + +* [TCA Hi Lo input](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse/tca_hi_lo/tcainput.json) + +Note: The referenced data format (in this case, a schema named "Common Event Format" with version of "25.0.0") must already exist in the onboarding catalog. + +## Working with Data Formats + +Data Formats can be added to the onboarding catalog (which first validates them) by using the [dcae_cli Tool](http://dcae-platform.research.att.com/components/dcae-cli/quickstart/). Here you can also list all of your data formats, show the contents of a data format, publish your data format, and even generate a data format from a input sample file. For a list of these capabilities, see [Data Format Commands](/components/dcae-cli/commands/#data_format). + diff --git a/platformdoc/docs/components/dcae-cli/commands.md b/platformdoc/docs/components/dcae-cli/commands.md new file mode 100644 index 00000000..d198f947 --- /dev/null +++ b/platformdoc/docs/components/dcae-cli/commands.md @@ -0,0 +1,453 @@ +# dcae_cli Commands + +The dcae_cli tool has four command groups. Each has several sub-commands. + +## `catalog` + +The `catalog` command lists and shows resources (components and data_formats) in the 'onboarding' catalog (regardless of the owner). A resource can have a status of `staged` or `published`. By default, only `published` resources are displayed. To see `staged` resources, add the --expanded argument. + +Catalog Status | Meaning +-------------- | ------- +staged | resource has be added (and validated), but is under development +| staged data_formats can only be referenced in their owners component specs +| staged components can only be deployed by their owners +published | resource has been tested and can be shared +| published data_formats can be used in anyone's component spec +| published components and be deployed by anyone + +``` +$ dcae_cli catalog --help +Usage: dcae_cli catalog [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + list Lists resources in the onboarding catalog + show Provides more information about resource +``` + +### List onboarding catalog contents + +``` +$ dcae_cli catalog list +Components: ++--------------------------------+---------+--------+---------------------------------------------------------------------+--------+-----------+------------+ +| Name | Version | Type | Description | Owner | Status | Published | ++--------------------------------+---------+--------+---------------------------------------------------------------------+--------+-----------+------------+ +| AAI_Broker | 3.1.0 | docker | DCAE Interface to DTI's AAI View | tc677g | published | 2017-06-15 | +| DcaeSyslogCollector | 2.0.0 | docker | DCAE Control Plane Syslog Collector | sh1986 | published | 2017-08-04 | +| cdap.dmaap.spec.example | 0.2.0 | cdap | dmaap spec example. Not a functioning application, only for showing | tc677g | published | 2017-07-24 | +| | | | how to pub/sub dmaap. Pretend this is like MAP with VES in and ou.. | | | | +| cdap.event.proc.enrich.app | 1.0.3 | cdap | CDAP Event Processing Enrich application | an4828 | published | 2017-09-20 | +| cdap.event.proc.map.app | 1.0.3 | cdap | CDAP Event Processing Map application | an4828 | published | 2017-09-20 | + +... + +Data formats: ++--------------------------------------------+---------+-----------------------------------------------------------------------+--------+-----------+------------+ +| Name | Version | Description | Owner | Status | Published | ++--------------------------------------------+---------+-----------------------------------------------------------------------+--------+-----------+------------+ +| FOI_PM_VHSS_data_format | 1.0.0 | CSV pipe delimited data format for VHSS PM files | sr229c | published | 2017-09-05 | +| Map_input | 1.0.0 | The input format for Mapper, that in 1707 is the UCSNMP Collector | an4828 | published | 2017-07-18 | +| | | output format, but will support more formats later | | | | +| Syslog Collector Parsed Json Message | 1.0.0 | Post processed/parsed collected syslog message | sh1986 | published | 2017-08-04 | +| Syslog Collector Syslog Message Input | 1.0.0 | The input message for the DCAE syslog collector is free/unstructured | sh1986 | published | 2017-08-04 | +| | | text | | | | +| TCA Alert Definition | 1.0.0 | The format of the output event from TCA | an4828 | published | 2017-08-10 | +| VES_specification | 5.28.4 | VES spec for 5.4 | vv770d | published | 2017-09-19 | + +... + +``` + +### Show the contents of an item in the catalog + +``` +$ dcae_cli catalog show FOI_PM_VHSS_data_format + +Data format +----------- +{ + "dataformatversion": "1.0.0", + "delimitedschema": { + "delimiter": "|", + "fields": [ + { + "description": "System ID", + "fieldtype": "string", + "name": "SYSTEM" + }, + { + "description": "Date", + "fieldtype": "string", + "name": "DATE" + }, + { + "description": "Time", + "fieldtype": "string", + "name": "TIME" + }, + +... + +``` + +--------------------------------------------------------------------------- + +## `component` + +The `component` command is for validating (adding), listing, showing, verifying generated configuration, running, undeploying, and publishing components that YOU own. + +``` +$ dcae_cli component --help +Usage: dcae_cli component [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + add + dev Set up component in development for... + list Lists components in the onboarding catalog. + publish Pushes COMPONENT to the onboarding catalog + run Runs the latest version of COMPONENT. + show Provides more information about COMPONENT + undeploy Undeploys the latest version of COMPONENT. +``` + +--------------------------------------------------------------------------- + +### Add a Component + +A component must be added to the onboarding catalog in order to be tested by the dcae_cli tool. The process of adding a component also validates it's component specification. In order to add a component, the component docker image (or CDAP jar) must exist locally. + +Components in the onboarding catalog can be run by others, once they are `published.` `Published` components cannot be modified or deleted. Rather a new version can be created instead. + +Validated component specs are used later to generate Tosca models and Cloudify Blueprints for the component, which makes them available for use in the SDC Tool for creating services. + +``` +$ dcae_cli component add --help +Usage: dcae_cli component add [OPTIONS] COMPONENT-SPECIFICATION + +Options: + --update Updates a previously added component if it has not been already + published + --help Show this message and exit. +``` + +``` +$ dcae_cli component add component-spec.json +``` + +--------------------------------------------------------------------------- + +### List Components + +List components in the onboarding catalog that owned by YOUR userid.. + +``` +$ dcae_cli component list +Active profile: solutioning ++-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ +| Name | Version | Type | Description | Status | Modified | #Deployed | ++-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ +| cdap.helloworld.endnode | 0.8.0 | cdap | cdap test component | staged | 2017-05-23 04:14:35.588075 | 0 | +| sandbox.platform.laika | 0.5.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-05-23 04:07:44.065610 | 0 | ++-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ +``` + +The fields `Name`, `Version`, `Type`, `Description` are referenced from the component specification's `self` JSON. +Use the "--deployed" argument to see more details on deploymed components + +--------------------------------------------------------------------------- + +### Run a Component + +The `run` operation is to be used for running your application in its container remotely on the activated environment. Docker containers have the additional option to run locally on your development machine. If the component uses Dmaap, you can specify the Dmaap Connection Object as well. Refer to [Dmaap Connection Object](/components/component-specification/dmaap-connection-objects). + +When you run a component via the dcae_cli Tool, remember the blueprint has not been created and is not used for deployment. + +In order to run the component, the data formats and component must have been added to the onboarding catalog. + +**DOCKER NOTE:** Make sure the Docker image has been uploaded to the shared registry. + +A docker component can be run in either `attached` or `unattached` mode. (Default is unattached). + +Mode | Description +---- | ----------- +attached | component is run in the 'foreground', container logs are streamed to stdout. Ctrl-C is used to terminate the dcae_cli session. +unattached | component is run in the 'background', container logs are viewed via `docker logs` command, container runs until undeployed with dcae_cli `undeploy` command. + + +#### Run a component in attached mode: + +``` +$ dcae_cli -v component run --attached sandbox.platform.laika:0.5.0 + +DCAE.Docker | INFO | Running image 'registry.proto.research.att.com/dcae-rework/laika:0.4.0' as 'mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika' +DCAE.Docker.mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika | INFO | Consul host: realsolcnsl00.dcae.solutioning.homer.att.com + +DCAE.Docker.mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika | INFO | service name: mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika + +DCAE.Docker.mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika | INFO | Generated config: {'multiplier': 3} + +DCAE.Docker.mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika | INFO | * Running on http://0.0.0.0:8080/ (Press CTRL+C to quit) + +DCAE.Docker.mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika | INFO | 135.205.226.140 - - [24/May/2017 03:37:57] "GET /health HTTP/1.1" 200 - + +DCAE.Docker.mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika | INFO | 135.205.226.140 - - [24/May/2017 03:38:12] "GET /health HTTP/1.1" 200 - +``` + +Hit Ctrl-C to terminate session. + +``` +^CDCAE.Docker | INFO | Stopping container 'mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-laika' and cleaning up... +``` + +#### Run a component in unattached mode: + +``` +$ dcae_cli -v component run sandbox.platform.laika:0.5.0 +DCAE.Docker | INFO | Running image 'registry.proto.research.att.com/dcae-rework/laika:0.4.0' as 'mh677g.4811da0e-08d5-429f-93bf-bf6814924577.0-5-0.sandbox-platform-laika' +DCAE.Run | INFO | Deployed /mh677g.4811da0e-08d5-429f-93bf-bf6814924577.0-5-0.sandbox-platform-laika +``` + +**NOTE** You must undeploy this component when finished testing. This is important to conserve resources in the environment. + +#### Run a component that subscribes to Dmaap MR or DR + +``` +$ dcae_cli -v component run --attached --dmaap-file $dmaap-connection-file sandbox.platform.laika:0.5.0 +``` + +--------------------------------------------------------------------------- + +### Undeploy a Component + +The `undeploy` command is used to undeploy any instance of a specified component/version that you have deployed. This includes cleaning up the configuration. + +``` +$ dcae_cli component undeploy sandbox.platform.laika:0.5.0 +DCAE.Undeploy | WARNING | Undeploying components: 1 +DCAE.Undeploy | WARNING | Undeployed components: 1 +``` + +--------------------------------------------------------------------------- + +### Publish a component + +Once a component has been tested, it should be published in the onboarding catalog using the `publish` sub-command . +Publishing will change the status of the component (from `staged` to `published`), indicating that it has been tested, and making it accessible for other developers to use. + +**Note** Before a component can be published, all data_formats that it references must be published. + +``` +dcae_cli component publish sandbox.platform.laika:0.5.0 +``` + +--------------------------------------------------------------------------- + +### Show a Component + +This will print out the contents of a component and is useful to copy a component spec. + +``` +$ dcae_cli component show +``` + +--------------------------------------------------------------------------- + +### Run the `dev` command + +The `dev` command is used as part of a process to see the platform generated configuration. It established the environment variables and is best explained [here](/components/dcae-cli/walkthrough/#view-the-platform-generated-configuration). + +``` +$ dcae_cli component dev component-spec.json +Ready for component development +``` + +--------------------------------------------------------------------------- + +## `data_format` + +The `data_format` command is for validating (adding), listing, showing, publishing data_formats that YOU own. data_formats can also be generated with this command. + +``` +$ dcae_cli data_format --help +Usage: dcae_cli data_format [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + add Tracks a data format file DATA_FORMAT-SPECIFICATION... + generate Create schema from a file or directory... + list Lists all your data formats + publish Publishes data format to make available to others... + show Provides more information about FORMAT +``` + +--------------------------------------------------------------------------- + +### Add a Data Format + +A data_format must be in the onboarding catalog in order to be referenced in the component specification. The process of adding a data_format also validates it. +Data_formats in the onboarding catalog can be run by others, once they are `published.` `Published` data_formats cannot be modified or deleted. Rather a new version can be created instead. + +``` +$ dcae_cli data_format add --help +Usage: dcae_cli data_format add [OPTIONS] DATA_FORMAT-SPECIFICATION + +Options: + --update Updates a previously added data_format if it has not been already + published + --help Show this message and exit. +``` + +``` +dcae_cli data_format add health.json +``` + +--------------------------------------------------------------------------- + +### List Data Formats + +Only data_formats owned by YOUR userid will be shown. + +``` +$ dcae_cli data_format list + +Data formats for mh877g ++-------------------------------+---------+-------------------------------------------+--------+----------------------------+ +| Name | Version | Description | Status | Modified | ++-------------------------------+---------+-------------------------------------------+--------+----------------------------+ +| sandbox.platform.laika.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-05-23 04:02:38.952799 | ++-------------------------------+---------+-------------------------------------------+--------+----------------------------+ +``` + +The fields `name`, `version`, `description` are referenced from the data format specification's `self` JSON. `Status` represents the status of the data format in the catalog. See [Publish a Data Format](#publish-a-data-format) for more info. + +--------------------------------------------------------------------------- + +### Show a Data Format + +This will print out the contents of a data_format and is useful for copying a data_format. + +``` +$ dcae_cli data_format show +``` + +--------------------------------------------------------------------------- + +### Publish a Data Format + +Once a data_format has been tested (by referencing it in a component spec that has been tested), it should be published in the onboarding catalog using the `publish` sub-command . +Publishing will change the status of the data_format (from `staged` to `published`), indicating that it has been tested, and making it accessible for other developers to use. + +``` +$ dcae_cli data_format publish data_format.json +``` + +--------------------------------------------------------------------------- + +### Generate a Data Format + +If you already have a valid input or output file, you can use the generate command to create the it's data format specification. + +``` +$ dcae_cli data_format generate name:version file-or-dir-path +``` + +--------------------------------------------------------------------------- + +## `profiles` + +The`profiles` command is for creating, deleting, listing, showing, activating, and updating (set) profiles. The profile contains environment variables used to connect to different environments. This is used in the running and deployment of a component using the `dcae_cli component run` or `dev` command. + +``` +$ dcae_cli profiles --help +Usage: dcae_cli profiles [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + activate Sets profile NAME as the active profile + create Creates a new profile NAME initialized with... + delete Deletes profile NAME + list Lists available profiles + set Updates profile NAME such that KEY=VALUE + show Prints the profile dictionary +``` + +--------------------------------------------------------------------------- + +### List the available profiles + +``` +$ dcae_cli profiles list +* solutioning + 1710 + 1802 +``` + +The * identifies the active profile. `dcae-cli` is currently installed with profiles for the `solutioning`, `1710`, and `1802` environments. They are intended for the following: + +Environment | Description +----------- | ----------- +solutioning | default environment; used for initial component developer testing with the dcae_cli tool. +1710 | FTL3 (Functional Testing Lab 3) environment, which represents the 1710 release. +1802 | FTL3a (Functional Testing Lab 3a) environment, which represents the 1802 release. + +--------------------------------------------------------------------------- + +### Show the details of a profile + +``` +$ dcae_cli profiles show solutioning +{ + "cdap_broker": "cdap_broker", + "config_binding_service": "config_binding_service", + "consul_host": "realsolcnsl00.dcae.solutioning.homer.att.com", + "docker_host": "realsoldokr00.dcae.solutioning.homer.att.com:2376" +} +``` + +--------------------------------------------------------------------------- + +### Activate a profile + +To switch among profiles, use the activate sub-command. A subsequent `list` will reveal the change made. + +``` +$ dcae_cli profiles activate test +``` + +--------------------------------------------------------------------------- + +### Create a new profile + +If you want to work in a different environment using the dcae_cli tool, you can make your own profile. (The environment must be a working DCAE Platform environment). + +``` +$ dcae_cli profiles create new-profile +``` + +After creating you would assign the variables with the `set` sub-command. Then activate it to use. + +--------------------------------------------------------------------------- + +### Set variables in a profile + +``` +$ dcae_cli profiles set $profile $key $value +``` + +--------------------------------------------------------------------------- + +### Delete a profile + +``` +$ dcae_cli profiles delete new-profile +``` diff --git a/platformdoc/docs/components/dcae-cli/quickstart.md b/platformdoc/docs/components/dcae-cli/quickstart.md index 905b1372..a70cd037 100644 --- a/platformdoc/docs/components/dcae-cli/quickstart.md +++ b/platformdoc/docs/components/dcae-cli/quickstart.md @@ -1,48 +1,84 @@ -# Quickstart +# Overview -The `dcae-cli` is a Python command-line tool built to aide component developers with the development and testing of their micro-service component for the DCAE platform. It will help developers do functional and integration testing of their components locally and on remote environments as simple as possible. +The `dcae-cli` is a Python command-line tool for component developers. With it, the developer can : -The tool requires the component developers to create a valid component specification for their component which is used by the tool. This same component specification will be published in the [onboarding catalog](../../glossary#onboarding-catalog) at the end of development and testing. +* validate the data formats and component specifications +* publish the validated data formats and component specifications into the `onboarding catalog` +* access the `onboarding catalog` to search for existing data formats (for possible reuse) and component specs +* deploy a component onto a local or remote DCAE platform for functional and pair-wise testing (This is done without Cloudify) -The git repository can be found [here](ONAP LINK TBD) +The git repository for the dcae_cli tool can be found [here](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae-cli/browse) -## Pre-requisite +## Pre-requisites ### For Docker -There are two options for development with Docker: developing locally on your machine which requires Docker to be installed and developing remotely by deploying onto remote infrastructure. +There are two options for development with Docker: #### For local development -* You must install [Docker engine](https://docs.docker.com/engine/installation/) locally on your machine. -* You must know the *external ip* of where the Docker engine is running. The external ip is needed so that service discovery will wire up correctly. - - For OSX users, this means making sure the VirtualBox VM that is running your Docker engine has a bridged adapter and getting the ip of that adapter. +* Install [Docker engine](https://docs.docker.com/engine/installation/) locally on your machine. +* Know the *external ip* of where the Docker engine is running. The external ip is needed so that service discovery will connect to it. + - *(For OSX users, this means making sure the VirtualBox VM that is running your Docker engine has a bridged adapter and getting the ip of that adapter).* #### For remote development -You need access to a remote host with Docker engine installed with remote API access. You must have the associated connection information: domain name or IP and port (should be either 2375 or 2376). This information should be set in [an active profile](walkthrough.md#setting-profile). +* Have access to a remote host with Docker engine installed and with remote API access. +* Have the associated connection information: + - domain name or IP and port (port should be either 2375 or 2376). Use this information to establish an active [profile](/components/dcae-cli/commands#activate-a-profile). ### For CDAP -TBD +None at this time. -## Install +### Python, Pip, Virtualenv + +Install python, pip (9.0.1 or higher), and virtualenv if they are not installed. Do these when not in a VPN to avoid possible network issues. + +``` + sudo apt-get -f install python + sudo apt-get -f install python-pip + sudo pip install virtualenv +``` + +Set up a virtual environment and activate ``` -pip install --extra-index-url https://YOUR_NEXUS_PYPI_SERVER/simple dcae-cli + virtualenv cli_tool + source cli_tool/biin/activate ``` -To do an upgrade, use the `--upgrade` flag. +## Install dcae_cli + +``` +pip install --extra-index-url https://nexus01.research.att.com:8443/repository/solutioning01-mte2-pypi/simple dcae-cli +``` -### Configuration +## Check dcae_cli version -When you run the tool for the first time, the tool will create a [configuration directory](http://click.pocoo.org/5/api/#click.get_app_dir) and generate a configuration file. +You can verify the version of the dcae-cli with the following command. To get the latest version of the dcae_cli tool, -Configuration is first sourced from an remote server that the platform team manages. This is overlaid with configuration details that you will be prompted to input particularly your user id. +``` +$ dcae_cli --version +``` -#### `--reinit` +## Upgrade dcae_cli -Configuration can be re-initialized or reset. There is a `--reinit` flag that is to be used to re-initialize your configuration and your environment profiles. +Periodically, upgrade the dcae_cli to get the latest version + +``` +pip install --upgrade --extra-index-url https://nexus01.research.att.com:8443/repository/solutioning01-mte2-pypi/simple dcae-cli +``` + +## Configuration + +When running the tool for the first time, a [configuration directory](http://click.pocoo.org/5/api/#click.get_app_dir) and configuration file will be created. + +The configuration is first sourced from a remote server that is managed by the platform team. You will be prompted to enter your ATTUID to complete this process. + +### Re-initializing Configuration + +Configuration can be re-initialized or reset. There is a `--reinit` flag that is to be used to re-initialize your configuration and your environment profiles. You may be instructed to re-initialize after certain updates are made to the dcae_cli tool. When you re-initialize the configuration, your configuration will be added to or updated from the platform configuration and profiles. No profiles will be deleted via the reinit process. To re-initialize: @@ -50,9 +86,9 @@ To re-initialize: $ dcae_cli --reinit ``` -### Verify +## Verify Installation -Verify that its installed: +To Verify that the dcae_cli tool is installed, run the following command and look for the output below. ``` $ dcae_cli --help @@ -70,10 +106,5 @@ Commands: data_format profiles ``` -## Version - -You can verify the version of the dcae-cli with the following command: -``` -$ dcae_cli --version -``` +Refer to [dcae_cli Commands](/components/dcae-cli/commands). diff --git a/platformdoc/docs/components/dcae-cli/walkthrough.md b/platformdoc/docs/components/dcae-cli/walkthrough.md index da54e12e..683bfabd 100644 --- a/platformdoc/docs/components/dcae-cli/walkthrough.md +++ b/platformdoc/docs/components/dcae-cli/walkthrough.md @@ -1,438 +1,340 @@ # Walk-through -The goal of this quickstart is to provide an overview of the functionalities of the `dcae-cli` and walk you through the capabilities: +This section demonstrates the flow and usage of the dcae_cli tool to onboard a typical component to the DCAE platform. The commands are explained in more detail in [dcae_cli Commands](/components/dcae-cli/commands). -* [Adding data formats](#adding-data-formats) -* [Adding component](#adding-component) -* [Setting profile](#setting-profile) -* [Development and testing](#development-and-testing) -* [Publishing component](#publishing-component) -* [Shared catalog](#shared-catalog) +* [Add (and validate) a data format](#add-a-data-format) +* [Add (and validate) the component](#add-the-component) +* [View the platform generated configuration](#view-the-platform-generated-configuration) +* [If needed, Create the dmaap file for Dmaap Testing](#create-input-file-for-dmaap-testing) +* [If needed, Create the input file for *Sourced at Deployment* Testing](#create-input-file-for-sourced-at-deployment-testing) +* [Run the component](#run-the-component) +* [If needed, Create the DTI entry in CONSUL for DTI Reconfiguration Testing](#create-DTI-entry-for-reconfiguration) +* [Undeploy the component](#undeploy-the-component) +* [Publish the component and data_format](#publish-the-component-and-data_format) to let others know its ready for reuse +* [List the Catalog Contents](#list-the-catalog-contents) to see your published resources -This walk-through uses example projects: +This walk-through uses example projects that can be found in CodeCloud: -* [laika](ONAP URL TBD) -* [CDAP examples](ONAP URL TBD) +* [laika](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/laika/browse) +* [CDAP examples](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_examples/browse) -## Adding data formats - -`data_format` is the sub-command that is used to execute operations that manage [data formats](../data-formats.md). +------------------------------------------------------------------- +## Add a Data Format ``` -$ dcae_cli data_format --help -Usage: dcae_cli data_format [OPTIONS] COMMAND [ARGS]... - -Options: - --help Show this message and exit. - -Commands: - add Tracks a data format file SPECIFICATION... - generate Generates a data format file from JSON input examples - list Lists all your data formats - publish Publishes data format to make publicly... - show Provides more information about FORMAT +$ dcae_cli data_format add $HOME/laika/data-formats/health.json ``` -Your data format must be in the catalog in order to use in the component specification. Check the catalog using the `data_format list` sub-command: - +Verify that the data_format was added ``` -$ dcae_cli data_format list - -Data formats for mh677g -+------+---------+-------------+--------+----------+ -| Name | Version | Description | Status | Modified | -+------+---------+-------------+--------+----------+ -| | | | | | -+------+---------+-------------+--------+----------+ +$ dcae_cli data_format list | grep laika +| sandbox.platform.laika.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-11-07 21:48:47.736518 | ``` -The fields `name`, `version`, `description` are referenced from the data format JSON from the `self` JSON. +(Note: Each of the data formats for your component need to be added, unless already existing in the onboarding catalog ) -There are no data formats so you must add the data formats that your component specification references. Use the `data_format add` sub-command: +------------------------------------------------------------------- -Here's an example command: +## Add the Component ``` -dcae_cli data_format add health.json +$ dcae_cli component add $HOME/laika/component-spec.json ``` -Verify that it was added: - +Verify that the component was added ``` -$ dcae_cli data_format list +$ dcae_cli component list +Active profile: solutioning -Data formats for mh677g -+-------------------------------+---------+-------------------------------------------+--------+----------------------------+ -| Name | Version | Description | Status | Modified | -+-------------------------------+---------+-------------------------------------------+--------+----------------------------+ -| sandbox.platform.laika.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-05-23 04:02:38.952799 | -+-------------------------------+---------+-------------------------------------------+--------+----------------------------+ ++-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ +| Name | Version | Type | Description | Status | Modified | #Deployed | ++-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ +| sandbox.platform.laika | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-11-08 20:27:34.168854 | 0 | ++-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ ``` -Go ahead and add other referenced data formats. +------------------------------------------------------------------- -If you have JSON input you can generate a data format like this: +## View the platform generated configuration -``` -$ dcae_cli data_format --keywords generate myname:1.0.0 myjsoninputfile -``` +The `component dev` command is useful during onboarding. Running this command is part of a multi-step process that sets up a temporary test environment, generates your application configuration, makes it available in that environment, and allows you to view that configuration to help with debugging. -where `myname` is the name of your data format, `1.0.0` is an example version, and `myjsoninputfile` is an example JSON input file (a directory of input JSON files can also be provided). The `--keywords` option adds additional data attributes that can be completed to provide a more detailed data characterization. In any event the output should be reviewed for accuracy. The data format is written to stdout. +Here is a step-by-step example based on a component specification called `component-spec.json`. -## Adding component +### Step 1 - Run the component dev command -`component` is the sub-command that is used to work with operations for components: +(This creates a file called env_$ENV (in the current directory)- where $ENV is the name of the active profile. Note: SERVICE_NAME and HOSTNAME always resolve to the same value). ``` -$ dcae_cli component --help -Usage: dcae_cli component [OPTIONS] COMMAND [ARGS]... - -Options: - --help Show this message and exit. +$ dcae_cli component dev component-spec.json +Ready for component development -Commands: - add - dev Set up component in development for... - list Lists components in the public catalog. - publish Pushes COMPONENT to the public catalog - run Runs the latest version of COMPONENT. - show Provides more information about COMPONENT - undeploy Undeploys the latest version of COMPONENT. -``` +Setup these environment variables. Run "source env_solutioning": -Your component must be accessible from the catalog in order for it to be used. Check the catalog using the `component list` sub-command: +export DOCKER_HOST=realsoldokr00.dcae.solutioning.homer.att.com:2376 +export SERVICE_CHECK_INTERVAL=15s +export CONFIG_BINDING_SERVICE=config_binding_service +export HOSTNAME=ph8547.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-laika +export CONSUL_HOST=realsolcnsl00.dcae.solutioning.homer.att.com +export CDAP_BROKER=cdap_broker +export SERVICE_NAME=ph8547.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-laika +export SERVICE_CHECK_TIMEOUT=1s +export SERVICE_CHECK_HTTP=/health +Press any key to stop and to clean up ``` -$ dcae_cli component list -Active profile: solutioning -+------+---------+------+-------------+--------+----------+-----------+ -| Name | Version | Type | Description | Status | Modified | #Deployed | -+------+---------+------+-------------+--------+----------+-----------+ -| | | | | | | | -+------+---------+------+-------------+--------+----------+-----------+ +### Step 2 - Setup the environment +In another window, setup the temporary testing environment, by executing the environment file created above. -Use the "--deployed" option to see more details on deployments +``` +$ source env_solutioning ``` -The fields `name`, `version`, `type`, `description` are referenced from the component specification's `self` JSON. +(The application configuration is now available under the SERVICE_NAME shown above - `ph8547.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-laika`). -There are no components so you must add your component. Use the `component add` sub-command. The command is the same for docker and cdap components: -``` -$ dcae_cli component add --help -Usage: dcae_cli component add [OPTIONS] SPECIFICATION +### Step 3 - Query CONSUL +Query CONSUL to get the IP/PORT of CONFIG BINDING SERVICE -Options: - --update Updates a locally added component if it has not been already - pushed - --help Show this message and exit. +``` +$ curl http://$CONSUL_HOST:8500/v1/catalog/service/$CONFIG_BINDING_SERVICE +[ + { + "ID": "bfbc220d-4603-7f90-ec2e-611d3c330f20", + "Node":"realsoldokr00", + "Address": "10.226.1.15", + "Datacenter":"solutioning-central", + "TaggedAddresses": { + "lan":"10.226.1.15", + "wan":"10.226.1.15" + }, + "NodeMeta": {}, + "ServiceID": "472b116f9035:config_binding_service:10000", + "ServiceName": "config_binding_service", + "ServiceTags": [], + "ServiceAddress":"135.205.226.126", + "ServicePort":10000, + "ServiceEnableTagOverride": false, + "CreateIndex":1078990, + "ModifyIndex":1078990 + } +] ``` -*Note* use the `--update` flag to replace existing staged instances. - -The `component dev` sub-command can be useful in validating and experimenting when crafting your component specification. See details about `dev` under [Development and testing](#development-and-testing). +Fetch the generated configuration from CONFIG BINDING SERVICE using the 'serviceaddress' and 'serviceport' from above along with $SERVICE_NAME from earlier. -Once we add the components laika and helloworld, let's verify that they got added ok: +``` +$ curl http://135.205.226.126:10000/service_component/ph8547.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-laika +{"streams_subscribes": {}, "services_calls": {}, "multiplier": 3, "streams_publishes": {}} ``` -$ dcae_cli component list -Active profile: solutioning -+-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ -| Name | Version | Type | Description | Status | Modified | #Deployed | -+-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ -| cdap.helloworld.endnode | 0.8.0 | cdap | cdap test component | staged | 2017-05-23 04:14:35.588075 | 0 | -| sandbox.platform.laika | 0.5.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-05-23 04:07:44.065610 | 0 | -+-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ +------------------------------------------------------------------- -Use the "--deployed" option to see more details on deployments -``` +## Create the input file for Dmaap Testing -## Setting profile +Currently, the dcae-cli tool does not have the capability to provision topics or feeds. Therefore, in order to test with `message router` or `data router` feeds, the developer must manually provision the topic or feed and then provide the connection details in the form of a DMaap JSON file for testing. This file is then passed in on the `component run` or `component dev` commands by using the argument `--dmaap-file`. -`profile` is the sub-command that is used to manage profiles. These profiles contain environment variables used to connect to different environments. This is used in the running and deployment of your component using the `dcae_cli component run` command. The `dcae-cli` ships with profiles for `solutioning` and `rework`. +The structure of the DMaaP JSON is an object of config keys with the topic or feed connection details. The config keys are the `config_key` values specified in the component specification streams section where the streams must be type `message router` or `data router`. This file corresponds to the `Dmaap Connection Object` which is generated by the platform and provided to the component at runtime. The exception is that `delivery_url` cannot be provided in the dmaap-file because it is not created until the component is deployed. Refer to [Dmaap Connection Object](/components/component-specification/dmaap-connection-objects), for details on creating the dmaap-file for testing. -``` -$ dcae_cli profiles --help -Usage: dcae_cli profiles [OPTIONS] COMMAND [ARGS]... +------------------------------------------------------------------- -Options: - --help Show this message and exit. +## Create the input file for *Sourced at Deployment* Testing -Commands: - activate Sets profile NAME as the active profile - create Creates a new profile NAME initialized with... - delete Deletes profile NAME - list Lists available profiles - set Updates profile NAME such that KEY=VALUE - show Prints the profile dictionary -``` +Components may have configuration parameters whose values are to be sourced at deployment time. For example, there are components whose configuration parameters are to come from DTI events which are only available when the component is deployed. This is established in the [component specification](/components/component-specification/common-specification/#parameters) by setting the property `sourced_at_deployment` to `true` for each applicable parameter. -To see what variables a profile contains, you can use the `show` command, as in `dcae_cli profiles show PROFILE_NAME` +Then, use the `--inputs-file` command-line argument when running the component `dev` or `run` command for your component. This is to simulate providing the dynamic, deployment time values for those parameters marked as `sourced_at_deployment`. -Use the `create` sub-command to create your own profile and assign new values using the `set` command. Afterwards you will need to `activate` the profile you wish to use. First take a look at which profile is active: +For example, if your component specification has the following configuration parameters: ``` -$ dcae_cli profiles list - rework -* solutioning +"parameters": [{ + "name": "vnf-ip", + "value": "", + "sourced_at_deployment": true +}, +{ + "name": "static-param", + "value": 5 +}] ``` -The active profile is `solutioning` so to activate *rework* to use `rework`: +Pass in an input file that looks like: ``` -$ dcae_cli profiles activate rework +{ + "vnf-ip": "10.100.1.100" +} ``` -Check +The application configuration would look like: ``` -$ dcae_cli profiles list -* rework - solutioning +{ + "vnf-ip": "10.100.1.100", + "static-param": 5 +} ``` -## Development and testing - -The following operations under the sub-command `component` are aimed to help developers with testing: - -* `run` -* `undeploy` -* `dev` +------------------------------------------------------------------- -### `run` +## Run the component -The `run` operation is to be used for running your application in its container remotely on the activated environment. Docker containers have the additional option to run locally on your development machine. +The `run` operation is to be used for running your application in its container remotely on the activated environment. Docker containers have the additional option to run locally on your development machine. If the component uses Dmaap, you can specify the Dmaap Connection Object as well. Refer to [Dmaap Connection Object](/components/component-specification/dmaap-connection-objects). -In order to run your application, you must have added your data formats and your component to your catalog. +In order to run the component, the data formats and component must have been added to the onboarding catalog. -Let's verify that your component is in the catalog: +To verify what's in the catalog: ``` -$ dcae_cli component list +$ dcae_cli catalog list --expanded Active profile: solutioning - +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ | Name | Version | Type | Description | Status | Modified | #Deployed | +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ -| cdap.helloworld.endnode | 0.8.0 | cdap | cdap test component | staged | 2017-05-23 04:14:35.588075 | 0 | -| sandbox.platform.laika | 0.5.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-05-23 04:07:44.065610 | 0 | +| sandbox.platform.laika | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-11-08 20:27:34.168854 | 0 | +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ -Use the "--deployed" option to see more details on deployments ``` -#### Docker - -**NOTE** Make sure your Docker image has been uploaded to the shared registry. -For Docker containers, you can run either attached or unattached. Attached means that the dcae-cli tool will launch the container and not terminate. The dcae-cli while attached will stream in the logs of the Docker container. Doing a Ctrl-C will terminate the run session which means undeploy your container and force a clean up automatically. +For Docker -Running unattached means simply deploy your container. You will need to execute `undeploy` when you are done testing. -#### CDAP +**NOTE** Make sure the Docker image has been uploaded to the shared registry. -**NOTE** Make sure your CDAP jar has been uploaded to Nexus. +A docker component can be run in either `attached` or `unattached` mode. (Default is unattached). -TODO +Mode | Description +---- | ----------- +attached | component is run in the 'foreground', container logs are streamed to stdout. Ctrl-C is used to terminate the dcae_cli session. +unattached | component is run in the 'background', container logs are viewed via `docker logs` command, container runs until undeployed with dcae_cli `undeploy` command. -### `undeploy` -The `undeploy` operation is to be used to undeploy any instances of a specified component, version that you have deployed. This includes cleaning up of configuration. +#### Run a component in attached mode: -Let's undeploy `sandbox.platform.laika` that was deployed from the previous section: - -``` -$ dcae_cli component undeploy sandbox.platform.laika:0.5.0 -DCAE.Undeploy | WARNING | Undeploying components: 1 -DCAE.Undeploy | WARNING | Undeployed components: 1 ``` +$ dcae_cli -v component run --attached sandbox.platform.laika:0.7.0 +DCAE.Docker | INFO | Running image 'nexus01.research.att.com:18443/repository/solutioning01-mte2-docker/dcae-platform/laika:0.7.0' as 'ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika' +DCAE.Docker.ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika | INFO | Consul host: realsolcnsl00.dcae.solutioning.homer.att.com -### `dev` +DCAE.Docker.ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika | INFO | service name: ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika -The `dev` operation is a convenient operation that can be useful for the development and testing of your component. It can be used to: +DCAE.Docker.ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika | INFO | get_config returned the following configuration: {"streams_subscribes": {}, "multiplier": 3, "services_calls": {}, "streams_publishes": {}} -* Help validate your experimental component specification before uploading to the catalog -* Generate the application configuration from the component specification and make it available in a test environment. This allows you to view your resulting configuration for local development and to help debug potential related issues. +DCAE.Docker.ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika | INFO | * Running on http://0.0.0.0:8080/ (Press CTRL+C to quit) -Let's say you have a component specification called `component-spec.json`: +DCAE.Docker.ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika | INFO | 135.205.226.156 - - [08/Nov/2017 23:27:30] "GET /health HTTP/1.1" 200 - -``` -$ dcae_cli component dev component-spec.json -Ready for component development -Setup these environment varibles. Run "source env_solutioning": +Hit Ctrl-C to terminate session. -export DOCKER_HOST=SOME_DOCKER_HOST:2376 -export SERVICE_CHECK_INTERVAL=15s -export CONFIG_BINDING_SERVICE=config_binding_service -export HOSTNAME=mh677g.95740959-63d2-492a-b964-62a6dce2591d.0-6-0.sandbox-platform-laika -export CONSUL_HOST=SOME_CONSUL_HOST -export CDAP_BROKER=cdap_broker -export SERVICE_NAME=mh677g.95740959-63d2-492a-b964-62a6dce2591d.0-6-0.sandbox-platform-laika -export SERVICE_CHECK_TIMEOUT=1s -export SERVICE_CHECK_HTTP=/health - -Press any key to stop and to clean up +^C +DCAE.Docker | INFO | Stopping container 'ph8547.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-laika' and cleaning up... ``` -Your application configuration is now available under the name `mh677g.95740959-63d2-492a-b964-62a6dce2591d.0-6-0.sandbox-platform-laika`. - -To view the resulting configuration, you can `curl` a request to the config binding service or programmatically fetch your configuration within your application. - -You need to first query Consul to get the ip and port of config binding service: +#### Run a component in unattached mode: ``` -curl http://$CONSUL_HOST:8500/v1/catalog/service/$CONFIG_BINDING_SERVICE -[ - { - "ID": "983d5c94-c508-4a8a-9be3-5912bd09786b", - "Node": "realsolcnsl00", - "Address": "10.226.1.22", - "TaggedAddresses": { - "lan": "10.226.1.22", - "wan": "10.226.1.22" - }, - "NodeMeta": {}, - "ServiceID": "5f371f295c90:config_binding_service:10000", - "ServiceName": "config_binding_service", - "ServiceTags": [], - "ServiceAddress": "XXXX", - "ServicePort": 32770, - "ServiceEnableTagOverride": false, - "CreateIndex": 487, - "ModifyIndex": 487 - } -] +$ dcae_cli -v component run sandbox.platform.laika:0.7.0 +DCAE.Docker | INFO | Running image 'nexus01.research.att.com:18443/repository/solutioning01-mte2-docker/dcae-platform/laika:0.7.0' as 'ph8547.22629ebd-417e-4e61-a9a0-f0cb16d4cef2.0-7-0.sandbox-platform-laika' +DCAE.Run | INFO | Deployed ph8547.22629ebd-417e-4e61-a9a0-f0cb16d4cef2.0-7-0.sandbox-platform-laika. Verifying.. +DCAE.Run | INFO | Container is up and healthy ``` -### DMaaP testing - -Currently, the dcae-cli does not have the capability of provisioning topics. In order to do testing with message router topics or with data router feeds, the developer must provision the topic or the feed manually and provide the connection details in the form of a JSON in a file to the dcae-cli. This file is to be passed in when using the `run` and `dev` commands with the option `--dmaap-file`. - -The structure of the DMaaP JSON is an object of config keys to matching topic or feed connection details. Config keys are the `config_key` values specified in your component specification streams section where the streams must be type message router or data router. Information about the associated connection details can be found on [this page](dmaap-connection-objects.md). Please check it out. +**NOTE** You must undeploy this component when finished testing. This is important to conserve resources in the environment. -For example, if you have a component specification that has the following streams entry: +#### Run a component that subscribes to Dmaap Message Router or Data Router -```json -"streams": { - "publishes": [{ - "format": "ves", - "version": "1.0.0", - "type": "message router", - "config_key": "ves_connection" - }] -} ``` - -Then to deploy and to run your component, you must use the `--dmaap-file` command and pass in a JSON that looks like: - -```json -{ - "ves_connection": { - "type": "message_router", - "dmaap_info": { - "topic_url": "https://we-are-message-router.us:3905/events/some-topic" - } - } -} +$ dcae_cli -v component run $component-that-uses-dmamp --dmaap-file $dmaap-connection-object ``` -The provided DMaaP JSON is used to simulate the output of provisioning and will be used to merge with the generated application configuration at runtime. - -Your final application config will look like: +#### Run a component that expects input that is `sourced at deployment` -```json -{ - "streams_publishes": { - "ves_connection": { - "type": "message_router", - "dmaap_info": { - "topic_url": "https://we-are-message-router.us:3905/events/some-topic" - } - } - } -} ``` - -#### Data router subscribers - -Note for data router subscriber testing, you will need the delivery url in order to provision the subscriber to the feed. This is constructed at deployment time and will be provided by the dcae-cli after you deploy your component. The delivery urls will be displayed to the screen: - +$ dcae_cli -v component run $component-that-expects-dti --inputs-file $input-file-to-simulate-dti ``` -DCAE.Run | WARNING | Your component is a data router subscriber. Here are the delivery urls: - some-sub-dr: http://SOME_IP:32838/identity +------------------------------------------------------------------- -``` +## Create the DTI Entry for Reconfiguration -### *Sourced at deployment* testing +Go the the CONSUL UI for the environment that you are working in. Add a `dti` entry to represent one or more instances of `vnfType-vnfFuncId` for your component. -Components may have configuration parameters whose values are to be sourced at deployment time. For example, there are components whose configuration parameters are to come from DTI events which are only available when the component is getting deployed. These configuration parameters must be setup correctly in the [component specification](http://localhost:8000/components/component-specification/docker-specification/#configuration-parameters) by setting the property `sourced_at_deployment` to `true` for each and every parameter that is expected to come in at deployment time. +For example, in 1802, go (here)[http://zldcrdm5bdcc2cnsl00.2f3fb3.rdm5b.tci.att.com:8500/ui/#/zldcrdm5bdcc2/kv/). -Once your component specification has been updated correctly, you must use the `--inputs-file` command-line argument when running the commands `dev` or `run` with your component. This is to simulate providing the dynamic, deployment time values for those parameters marked as `sourced_at_deployment`. +Do CNTL-F to find your running MS +Click on + to add your entry +Enter your $SERVICE_NAME:dti as the Key +Paste your JSON into the box, remember to `check` the VALIDATE JSON box +Click on CREATE -For example, if your component specification has the following configuration parameters: +Verify that you can retrieve the dti entry you just created as in this example: (Remember to use the 'serviceaddress' and 'serviceport' from above for CONFIG BINDING SERVICE). ``` -"parameters": [{ - "name": "vnf-ip", - "value": "", - "sourced_at_deployment": true -}, -{ - "name": "static-param", - "value": 5 -}] +http://135.203.226.126:10000/dti/<service name>`` ``` +(You should see the entry you created above) -You would have to pass in an inputs file that looks like: +------------------------------------------------------------------- -``` -{ - "vnf-ip": "10.100.1.100" -} -``` +## Run the reconfigure script -Your application configuration would look like: +Execute the components reconfigure script as defined in the Auxilary section of the component spec, such as in this example: ``` -{ - "vnf-ip": "10.100.1.100", - "static-param": 5 -} +/opt/app/reconfigure.sh dti $updated_dti ``` -## Publishing component +(Refer to [DTI Reconfiguration](/components/component-specification/docker-specification/#dti-reconfiguration) + +Verify that your component received and is processed the updated set of vnfType-vnfFuncId instances. -Once components have their component specifications crafted and validated and have been tested, components should be published in the shared onboarding catalog using the `publish` sub-command for both data formats and components. You must publish all data formats of a component before publishing a component. +------------------------------------------------------------------ +## Undeploy the component -Publishing will change the status of a component, be made accessible for other developers to use, and will generate the associated TOSCA models for use in designing of compositions. +The `undeploy` command is used to undeploy any instance of a specified component/version that you have deployed. This includes cleaning up the configuration. + +Undeploy `sandbox.platform.laika:0.7.0` that was deployed above: ``` -dcae_cli component publish sandbox.platform.laika:0.5.0 +$ dcae_cli -v component undeploy sandbox.platform.laika:0.7.0 +DCAE.Undeploy | WARNING | Undeploying components: 1 +DCAE.Undeploy | WARNING | Undeployed components: 1 ``` -## Shared catalog +------------------------------------------------------------------- + +## Publish the component and data_format -`catalog` is the sub-command used to access and to browse the shared onboarding catalog to view components and data formats that have been published and that are being worked on. Components and data formats have two statuses `staged` and `published`. +Once a component has been tested, it (and the data_format(s)) should be published in the onboarding catalog using the `publish` sub-command for both the `data_format` and `component` command. -Staged means that the resource has been simply added and is under development. It is to be used only by the owner. Published means that the resource has been fully developed and tested and is ready to be shared. +**Note** Before a component can be published, all data_formats that it references must be published. -Published components can be deployed by non-owners and published data formats can be used in component specifications of non-owners. +Publishing will change the status of a component or data_format, indicating that it has been tested, make accessible for other developers to use. -There are two available operations: +``` +$ dcae_cli data_format publish sandbox.platform.laika:0.7.0 +Data format has been published + +$dcae_cli component publish sandbox.platform.laika:0.7.0 +Component has been published ``` -$ dcae_cli catalog --help -Usage: dcae_cli catalog [OPTIONS] COMMAND [ARGS]... +------------------------------------------------------------------- -Options: - --help Show this message and exit. +## List the catalog contents + +``` +$dcae_cli catalog list -Commands: - list - show +$ dcae_cli data_format list | grep sandbox +| sandbox.platform.laika | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | ph8547 | published | 2017-11-13 | +| sandbox.platform.laika.health | 0.1.0 | Data format used for the /health endpoint | published | 2017-11-13 17:48:10.121588 | +| sandbox.platform.any | 0.1.0 | Data format used when no data format is required. | published | 2017-11-13 17:47:51.622607 | +| sandbox.platform.laika.identity.response | 0.1.0 | Data format used for the /identity endpoint response which should | published | 2017-11-13 17:47:43.234715 | +| sandbox.platform.laika.identity.request | 0.1.0 | Data format used for the /identity endpoint request. This is | published | 2017-11-13 17:47:36.693643 | +| sandbox.platform.laika.rollcall.response | 0.1.0 | Data format used for the /rollcall endpoint respon.. | published | 2017-11-13 17:46:30.026846 | ``` -Staged components can be viewed under the `list` operation using the `--expanded` flag. diff --git a/platformdoc/docs/components/intro.md b/platformdoc/docs/components/intro.md index 6bd3b2d1..b94fb3e3 100644 --- a/platformdoc/docs/components/intro.md +++ b/platformdoc/docs/components/intro.md @@ -1,14 +1,14 @@ # Overview -DCAE components are services that provide a specific functionality and are written to be composable with other DCAE service components. The DCAE platform is responsible for running and managing DCAE service components reliably. +DCAE components are services that provide a specific functionality and are generally written to be composable with other DCAE components, although a component can run independently as well. The DCAE platform is responsible for running and managing DCAE service components reliably. Currently, the DCAE platform supports two types of components, CDAP applications and Docker containers. For each, there are requirements that must be met for the component to integrate into the DCAE platform (see [CDAP](component-type-cdap.md) and [Docker](component-type-docker.md)). -### Components requires one or more data formats. +### A Component requires one or more data formats. -Components are software applications that do some function. Components don't run independently, they depend upon other components. A component's function could require connecting to other components to fulfill that function. A component could also be providing its function as a service through an interface for other components to use. +A component is a software application that performs a function. It doesn't run independently; it depends upon other components. A component's function could require connecting to other components to fulfill that function. A component could also be providing its function as a service through an interface for other components to use. -Components cannot connect to or be connected with any other component. The upstream and downstream components must *speak* the same vocabulary or *data format*. The output of an one component must match another component's input. This is necessary for components to function correctly and without errors. +A component cannot connect to or be connected with any other component. The upstream and downstream components must *speak* the same vocabulary or *data format*. The output of an one component must match another component's input. This is necessary for components to function correctly and without errors. The platform requires data formats to ensure that a component will be run with other *compatible* components. @@ -17,7 +17,7 @@ Data formats can and should be shared by multiple components. ### Each Component requires a component specification. -The component specification is a JSON artifact that fully specifies the component, it's interfaces, and configuration. It's standardized for CDAP and Docker applications and is validated using a [JSON schema](ONAP URL TBD). +The component specification is a JSON artifact that fully specifies the component, it's interfaces, and configuration. It's standardized for CDAP and Docker applications and is validated using a [JSON schema](https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/component-json-schemas/browse/component-spec-schema.json). The component specification fully specifies all the configuration parameters of the component. This is used by the designer and by policy (future) to configure the runtime behavior of the component. @@ -41,8 +41,8 @@ Onboarding is a process that ensures that the component is compliant with the DC 1. Defining the [data formats](data-formats.md) if they don't already exist. 2. Defining the [component specification](/components/component-specification/common-specification.md). See [docker](component-specification/docker-specification.md) and [CDAP](component-specification/cdap-specification.md). -3. Use the dcae_cli tool to [add the data formats](/components/dcae-cli/walkthrough/#adding-data-formats) and [add the component](/components/dcae-cli/walkthrough/#adding-component) to the onboarding catalog. This process will validate them as well. -4. Use the dcae_cli tool to [deploy](/components/dcae-cli/walkthrough/#development-and-testing) the component. (The component is deployed to the environment indicated in the [profile](/components/dcae-cli/walkthrough/#setting-profile) section. +3. Use the dcae_cli tool to [add the data formats](/components/dcae-cli/commands/#add-a-data-format) and [add the component](/components/dcae-cli/commands/#add-a-component) to the onboarding catalog. This process will validate them as well. +4. Use the dcae_cli tool to [deploy](/components/dcae-cli/commands/#run-a-component) the component. (The component is deployed to the environment indicated in the [profile](/components/dcae-cli/commands/#activate-a-profile) section). 5. Test the component. Also do pairwise-test the component with any other components it connects with. -7. Publish the component and data formats into the Service Design and Creation (SDC) 'catalog'. (Currently, this is a manual step). +7. Publish the component and data formats into the Service Design and Creation (SDC) 'catalog'. (Currently, this is a manual step, not done via the dcae_cli tool). diff --git a/platformdoc/docs/concepts/clamp.md b/platformdoc/docs/concepts/clamp.md new file mode 100644 index 00000000..f2eec839 --- /dev/null +++ b/platformdoc/docs/concepts/clamp.md @@ -0,0 +1,8 @@ + +# Closed Loop / CLAMP + +Closed Loop is a concept where when certain conditions are observed in the network by a component, messages are sent to a particular controller that then responds to those conditions. No manual intervention is needed. Thus the 'loop' is closed automatically. Closed Loop services are configured in the CLAMP UI, which is a non-DCAE component. + +There is no specific configuration in the component specification for closed loop services, and currently it is a manual process between the developer and the CLAMP support team, to discuss which configuration parameters are affected. These parameters can also be policy editable. + +Refer to the [Configuration Grid](/components/component-specification/configuration-grid) for more information. diff --git a/platformdoc/docs/concepts/dti.md b/platformdoc/docs/concepts/dti.md new file mode 100644 index 00000000..4af71187 --- /dev/null +++ b/platformdoc/docs/concepts/dti.md @@ -0,0 +1,10 @@ + +# DTI (DCAE Topology Interface) Reconfiguration + +Services that support DTI Reconfiguration are installed (with input provided by a DESIGNER (via SDC UI) or Operations (via Dashboard) to indicate which `vnfType-vnfFuncId` is to be processed by that service. When deployed, the service run but does not collect data until a DTI Event comes in. + +Whenever a network event occurs, resulting in the deployment or undeployment of a `vnfType-vnfFuncId`, it is recorded in A&AI and reported to the DTI Topology VM. The DTI Event is then received by the DTI HANDLER, which uses its values to retrieve one or more reconfiguration blueprints from INVENTORY, and then uploads those blueprint(s) to CLOUDIFY MANAGER. It then creates deployment(s) from those blueprint(s), populates them with the DTI Event values (as inputs), and then executes the install workflow on those deployment(s). + +The platform then executes the reconfigure script (configured in the Auxilary section of the component specification), passing the entire DTI Event to the component. The component can process the individual event or make a call to the CONFIG BINDING SERVICE to retrieve all of the vnfType-vnfFuncIds that are to be processed by the service. + +Refer to the [DTI Reconfiguration](/components/component-specification/docker-specification/#dti-reconfiguration) for more information. diff --git a/platformdoc/docs/concepts/on_sd.md b/platformdoc/docs/concepts/on_sd.md new file mode 100644 index 00000000..5e6a2f20 --- /dev/null +++ b/platformdoc/docs/concepts/on_sd.md @@ -0,0 +1,52 @@ +NOTE: THIS PAGE IS A STUB RIGHT NOW. NEEDS EDITING AND WORK. + +# CDAP Cuda Service discovery example. + +NOTE: The below URLs were temporary when the app was deployed, they are no longer valid. +However you can re-create all of this using the commands in https://codecloud.web.att.com/projects/ST_DCAECNTR/repos/dcae_cli_cdap_examples/browse/cuda. + +I deployed CUDA and got back the instance name: b30b7b640fa14539890efa768b4649a2.70900c87-1c82-4690-a0b1-e0aa72cbab84.0-4-1.cdap-cuda.rework-central.dcae.ecomp.att.com. + +If another DCAE service component, like a collector or downstream analytics, were to perform a service lookup on this CUDA app, they would eventually get back a broker URL: http://135.205.226.76:32773/application/b30b7b640fa14539890efa768b4649a2.70900c87-1c82-4690-a0b1-e0aa72cbab84.0-4-1.cdap-cuda.rework-central.dcae.ecomp.att.com + +Which returns a JSON that looks like this: + +``` +{ + "appname":"b30b7b640fa14539890efa768b4649a2.70900c87-1c82-4690-a0b1-e0aa72cbab84.0-4-1.cdap-cuda.rework-central.dcae.ecomp.att.com", + "namespace":"cuda", + "healthcheckurl":"http://135.205.226.76:32773/application/b30b7b640fa14539890efa768b4649a2.70900c87-1c82-4690-a0b1-e0aa72cbab84.0-4-1.cdap-cuda.rework-central.dcae.ecomp.att.com/healthcheck", + "metricsurl":"http://135.205.226.76:32773/application/b30b7b640fa14539890efa768b4649a2.70900c87-1c82-4690-a0b1-e0aa72cbab84.0-4-1.cdap-cuda.rework-central.dcae.ecomp.att.com/metrics", + "url":"http://135.205.226.76:32773/application/b30b7b640fa14539890efa768b4649a2.70900c87-1c82-4690-a0b1-e0aa72cbab84.0-4-1.cdap-cuda.rework-central.dcae.ecomp.att.com", + "connectionurl":"http://135.205.226.110:11015/v3/namespaces/cuda/streams/cuda_appInputStream", + "serviceendpoints":[ + { + "url":"http://135.205.226.110:11015/v3/namespaces/cuda/apps/b30b7b640fa14539890efa768b4649a270900c871c824690a0b1e0aa72cbab84041cdapcudareworkcentraldcaeecompattcom/services/cuda_appOutputDataQuery/methods/allrows", + "method":"GET" + }, + { + "url":"http://135.205.226.110:11015/v3/namespaces/cuda/apps/b30b7b640fa14539890efa768b4649a270900c871c824690a0b1e0aa72cbab84041cdapcudareworkcentraldcaeecompattcom/services/cuda_appOutputDataQuery/methods/lastrow", + "method":"GET" + } + ] +} +``` + +There are important service discovery keys in this JSON. The first is connectionurl. That is the full URL to this application’s stream. The collector can POST data to that URL. + +The second important key is serviceendpoints. These are formed using the information found in the spec. These are, again like connectionurl, full URIs that point to the actual data service. You can try hitting one of those in your browser: http://135.205.226.110:11015/v3/namespaces/cuda/apps/b30b7b640fa14539890efa768b4649a270900c871c824690a0b1e0aa72cbab84041cdapcudareworkcentraldcaeecompattcom/services/cuda_appOutputDataQuery/methods/allrows + +And you will get back the output: +``` +[ + { + "[0]":"{\"start\": 1, \"engagements\": [{\"transcript\": [{\"type\": \"chat.scriptlineSent\", \"senderId\": \"ss754c\", \"content\": \"My SSN is $Identity.ssn. $Identity.name, Thanks for choosing AT\u0026T. My name is $Identity.name and I will be happy to assist you with our plans and services.\", \"iso\": \"2016-05-31T21:59:55-07:00\", \"timestamp\": 1464757195034, \"senderName\": \"agent\"}], \"engagementID\": \"744878782903999181\"}], \"numFound\": 1}" + } +] +``` +This is what we call “service discovery”. A component wants to know how to connect to an application and all they know is that applications name. Through a series of HTTP calls which I’m glossing over here, they eventually get back the URL I showed above which returns that JSON containing all important URLs associated with that application. + +When DMaaP is introduced, this conversation is going to change. In that case, instead of discovering an application directly, you may instead discover a feed that feeds into/out of that app. + + + diff --git a/platformdoc/docs/concepts/policy.md b/platformdoc/docs/concepts/policy.md new file mode 100644 index 00000000..3838dc22 --- /dev/null +++ b/platformdoc/docs/concepts/policy.md @@ -0,0 +1,8 @@ + +# Policy (not yet implemented) + +Policy is a concept where certain configuration is specified as being 'set' by Operations via a Policy UI. + +The specific configuration that represents Policy for a given service is designated in the component specification as `policy_editable`, and has a `policy_schema` defined. At any time thereafter, the values can be set in the POLICY UI. Once set, POLICY HANDLER will update the values in CONSUL. If the component is running, it will notify the component that it's policy has changed, and the component will call the CONFIG BINDING SERVICE to obtain a current set of its configuration. + +Refer to the [Configuration Grid](/components/component-specification/configuration-grid) for more information. diff --git a/platformdoc/docs/concepts/service-designer.md b/platformdoc/docs/concepts/service-designer.md new file mode 100644 index 00000000..5f45f318 --- /dev/null +++ b/platformdoc/docs/concepts/service-designer.md @@ -0,0 +1,8 @@ + +# Service Design + +Service Design is a concept where the `Service Designer` role uses the `Service Design and Creation (SDC) Tool to compose and services from available components. In doing so, the Service Designer will need to provide certain input that is required by those components. + +The specific configuration that represents input that at Service Designer can provide is designated in the component specification as `designer_editable`. + +Refer to the [Configuration Grid](/components/component-specification/configuration-grid) for more information. diff --git a/platformdoc/docs/glossary.md b/platformdoc/docs/glossary.md index a9f23214..a48903e1 100644 --- a/platformdoc/docs/glossary.md +++ b/platformdoc/docs/glossary.md @@ -1,19 +1,107 @@ # Glossary -## ASDC +## A&AI - Active and Available Inventory +Inventory DB for all network components -The ECOMP resource catalog. We assume the existince and usage of ASDC, though for near term testing purposes, we use a mock version of the catalog. -We assume all DCAE artifacts: components like collectors, CDAP applications, data formats (see below), etc, are onboarded and searchable in the catalog. -Further, we assume that every catalog artifact has a ``UUID", a globally unique identifier that identifies that artifact. +## CLAMP +Non DCAE Platform Component - Controls the input and processing for Closed Loop services. + +## Closed Loop +Services designed to monitor and report back to a controlling function that automatically deals with the event reported without human interaction. + +## CDAP +Opensource Platform for development of Big Data platforms using Hadoop. Some DCAE service components are written utilizing CDAP. + +## Cloudify +Open Source application and network orchestration framework, based on TOSCA used in DCAE to deploy platform and service components from Cloudify Blueprints. Refer to [Architecture](/architecture/pieces) for more information. + +## Cloudify Blueprints +YAML formatted file used by Cloudify to deploy platform and service components. Contains all the information needed for installation. + +## Consul +Opensource Platform Component that supports Service Discovery, Configuration, and Healthcheck. Refer to [Architecture](/architecture/pieces) for more information. ## Component +Refers to a DCAE service component which is a single micro-service that is written to be run by the DCAE platform and to be composeable to form a DCAE service. That composition occurs in the SDC. + +## Config Binding Service +DCAE Platform Component - Service Components use Config Binding Service to access Consul and retrieve configuration variables. + +## Component Specification +JSON formatted file that fully describes a component and its interfaces + +## Data Format / Data Format Specification +JSON formatted file that fully describes a components input or output + +## dcae_cli Tool +Tool used for development and testing. It validates the component and data format specifications against their respective schemas, provides the capability to view platform generated configuration for the component, assist with Dmaap testing, and DTI testing, by enabling the developer to enter DTI input at the command line. + +## Deployment Handler +DCAE Platform Component - Receives Input from DTI Handler, and talks to Cloudify to deploy components. + +## Design-Time +Refers to when the System Designer uses the SDC Tool to compose services from components in the SDC catalog. The Designer can provide input to assign/override defaults for configuration for any parameter with the property 'designer_editable' set to 'true'. + +## Deploy-Time +Refers to when a service is being deployed. This can be done automatically via the SDC Tool, or manually via the DCAE Dashboard or CLAMP UI. When manually deployed, DevOps can provide input to assign/override defaults for configuration for any parameter with the property 'sourced_at_deployment' set to 'true'. + +## Docker +Opensource Platform for development of containerized applications in the cloud. Many DCAE service components and all DCAE collectors are written utilizing Docker. + +## Dmaap +AT&T data transportation service platform that supports message-based topics and file-based feeds. Runs locally at the Edge and Centrally. -Refers to a DCAE service component which is a single micro-service that is written to be run by the DCAE platform and to be composeable to form a DCAE service. +## DTI - DCAE Topology Interface +Method of receiving input based on Network element changes (the addtion and removal of VNFs). -## Data format +## DTI Event +Data passed between DTI Topology VM and DTI Handler, a notification of a network change -Artifact that describes a data structure and schema. +## DTI Handler +DCAE Platform Component - Receives DTI Event from DTI Topology VM, talks to Deployment Handler + +## DTI Topology VM +DCAE Platform Component - Receives DTI Event from A&AI, when there is a network change + +## Inventory +DCAE Platform Component - Postgres DB containing Cloudify Blueprints for platform and service components. ## Onboarding catalog +Catalog used exclusively by the dcae_cli tool during development and testing. Contains validated components and data_formats to be used among developers during development and testing. + +## Policy (not yet implemented) +Refers to the setting of configuration parameters for a component, by Operations via the Policy UI. + +## Policy Handler (not yet implemented) +DCAE Platform Component that received Policy updates from Policy UI + +## Policy UI (not yet implemented) +Non DCAE Component - Policy User Interace where Operations assigns values to configuraton specified for this. + +## Run-Time +Refers to the when a service is running on the platform. Often used in conjunction with DTI events which occur at Run-time. + +## SCH - Service Change Handler +DCAE Platform Component - Receives updates from SDC and updates Inventory + +## SDC - Service Design and Creation - (formerly ASDC) +Tool used by Service Designers to compose services from SDC catalog artifacts. Once services are created, Cloudify Blueprints can be generated to deployment and installation. + +## SDC Catalog +Catalog of composable Components and Data Formats to be used in the SDC Tool to create services. Currently, there is no access to the SDC Catalog from the dcae_cli tool. Artifacts are manually placed there after testing. Every catalog artifact has a `UUID`, a globally unique identifier that identifies that artifact. + +## Self-Service +Refers to services that are supported by SDC, and that are automatically installed as a result of a Service Designer's composition and submission of a service. Only a handful of services are 'self-service' currently. Most require manual effort to generate the Tosca Model files and Cloudify Blueprints. + +## Service Component +Microservice that provides network monitoring or analytic function on the DCAE platform. + +## Service +Generally composed of multiple service components, which is deployed to the DCAE platform. + +## Tosca Model +Model generated from validately component specification, (stored in SDC catalog for Self-Service components), and used as input to generate Cloudify Blueprints + +## VNF - Virtualized Network Function +A network function that runs on one or more virtualized machines. -The onboarding catalog is the intermediary store for project details, component specifications and data formats. This is to be used by component developers through the `dcae-cli` to browse and to launch existing components and push their own component artifacts in preparation to move their component to the ASDC resource catalog. diff --git a/platformdoc/docs/images/dftool1.png b/platformdoc/docs/images/dftool1.png Binary files differnew file mode 100755 index 00000000..5f9948f4 --- /dev/null +++ b/platformdoc/docs/images/dftool1.png diff --git a/platformdoc/docs/images/dftool2.png b/platformdoc/docs/images/dftool2.png Binary files differnew file mode 100755 index 00000000..5c4e3ff5 --- /dev/null +++ b/platformdoc/docs/images/dftool2.png diff --git a/platformdoc/docs/index.md b/platformdoc/docs/index.md index 93689beb..4a06541f 100644 --- a/platformdoc/docs/index.md +++ b/platformdoc/docs/index.md @@ -2,7 +2,7 @@ ## Welcome -This is the home for the documetation for the DCAE platform where you will find: +This is the home for the documentation for the DCAE platform. * Details of the platform's architecture * User guides for the different roles @@ -12,3 +12,38 @@ This is the home for the documetation for the DCAE platform where you will find: DCAE is composed of a platform and services. The DCAE platform is all of the technical pieces responsible for the deployment and the management of DCAE services. The DCAE services are the components responsible for monitoring network services, and data collection and analytics. +Currently, the current DCAE Platform is a hybrid running both the 'classic (or old) controller' and what's commonly called the 'new controller'. + +### For 1710 + +The classic controller is deploying and managing all CDAP applications, and docker applications that were deployed prior to 1710. +All other docker applications are being deployed and mananged by the new controller. + +### For 1802 + +The classic controller is deploying and managing all CDAP applications, and some docker applications that were deployed prior to 1710. +Some docker applications are being migrated from the classic to the new controller. All new docker applications are being deployed and mananged by the new controller. + + + +## Contacts + +Name | Email | Role +---- | ----- | ---- +Dan Musgrove | dm4812@att.com | Lead +Shrinkant Acharya | sa8763@att.com | Systems Engineer +Kailas Deshmukh | kd046m@att.com | Technical Lead +Anurag Agarwal | aa0918@att.com | Cloudify Manager Support +Henry Thorpe | ht1659@att.com | CDAP Broker Support +Hong Guan | hg4105@att.com | Clamp Integration Support +Ken Lehner | kl525c@att.com | Consul, Config Binding Service and Docker Container Support +Kuldeep Sharma | ks958r@att.com | DTI Topology VM, Docker Platform and CDAP Cluster Blueprint Support +Lisa Revel | lr0306@att.com | TOSCA Model Tool and Blueprint Generation +Patty Heffner | ph8547@att.com | Onboarding to the DCAE Platform +Shadi Haidar | sh1986@att.com | DTI Handler, Deployment Handler, Inventory, and A&AI Broker Support +Sue Steele | ss477j@att.com | CDAP Platform Support +Swapna Anne | sa9226@att.com | Docker Plugins Support +Terry Schmalzried | ts862m@att.com | Cloudify and CDAP Plugin Support +Tony Hansen | tony@att.com | Postgres Support +William Au | sa998j@att.com | Service Change Handler Support + diff --git a/platformdoc/mkdocs.yml b/platformdoc/mkdocs.yml index 3cbe14c1..35d660b0 100644 --- a/platformdoc/mkdocs.yml +++ b/platformdoc/mkdocs.yml @@ -1,16 +1,25 @@ site_name: DCAE Platform +site_url: https://github.research.att.com/pages/DCAE-Rework/dcae-rework.github.io/ + +repo_url: https://github.research.att.com/DCAE-Rework # http://squidfunk.github.io/mkdocs-material/ theme: material pages: - Home: 'index.md' + - Change Log: 'changelog.md' - Architecture: - Technologies: 'architecture/pieces.md' - Services: 'architecture/services.md' - Service Discovery: 'architecture/service-discovery.md' - Component Developers: - Overview: 'components/intro.md' + - Concepts: + - Closed Loop/CLAMP: 'concepts/clamp.md' + - DTI Reconfiguration: 'concepts/dti.md' + - Policy: 'concepts/policy.md' + - Service Design: 'concepts/service-designer.md' - Requirements/Guidelines: - CDAP: 'components/component-type-cdap.md' - Docker: 'components/component-type-docker.md' @@ -18,12 +27,13 @@ pages: - Common Elements: 'components/component-specification/common-specification.md' - CDAP Specific: 'components/component-specification/cdap-specification.md' - Docker Specific: 'components/component-specification/docker-specification.md' - - Generated Configuration: 'components/component-specification/generated-configuration.md' + - DMaaP connection objects: 'components/component-specification/dmaap-connection-objects.md' - Streams Quick Reference: 'components/component-specification/streams-grid.md' - - Configuration Quick Reference: 'components/component-specification/configuration-grid.md' - - Data Formats: 'components/data-formats.md' + - Configuration Types Quick Reference: 'components/component-specification/configuration-grid.md' + - Examples: 'components/component-specification/examples-grid.md' + - Data Format Specification: 'components/data-formats.md' - dcae-cli Tool: - Quickstart: 'components/dcae-cli/quickstart.md' + - Commands: 'components/dcae-cli/commands.md' - Walk-through: 'components/dcae-cli/walkthrough.md' - - DMaaP connection objects: 'components/dcae-cli/dmaap-connection-objects.md' - Glossary: 'glossary.md' diff --git a/platformdoc/pom.xml b/platformdoc/pom.xml deleted file mode 100644 index 6991ba7b..00000000 --- a/platformdoc/pom.xml +++ /dev/null @@ -1,274 +0,0 @@ -<?xml version="1.0"?> -<!-- -================================================================================ -Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. -================================================================================ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -============LICENSE_END========================================================= - -ECOMP is a trademark and service mark of AT&T Intellectual Property. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.onap.dcaegen2</groupId> - <artifactId>dcaegen2</artifactId> - <version>1.2.0-SNAPSHOT</version> - </parent> - <groupId>org.onap.dcaegen2</groupId> - <artifactId>platformdoc</artifactId> - <name>dcaegen2-platformdoc</name> - <version>1.2.0-SNAPSHOT</version> - <url>http://maven.apache.org</url> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <sonar.skip>true</sonar.skip> - <sonar.sources>.</sonar.sources> - <!-- customize the SONARQUBE URL --> - <!-- sonar.host.url>http://localhost:9000</sonar.host.url --> - <!-- below are language dependent --> - <!-- for Python --> - <sonar.language>py</sonar.language> - <sonar.pluginName>Python</sonar.pluginName> - <sonar.inclusions>**/*.py</sonar.inclusions> - <!-- for JavaScaript --> - <!-- - <sonar.language>js</sonar.language> - <sonar.pluginName>JS</sonar.pluginName> - <sonar.inclusions>**/*.js</sonar.inclusions> - --> - </properties> - <build> - <finalName>${project.artifactId}-${project.version}</finalName> - <pluginManagement> - <plugins> - <!-- the following plugins are invoked from oparent, we do not need them --> - <plugin> - <groupId>org.sonatype.plugins</groupId> - <artifactId>nexus-staging-maven-plugin</artifactId> - <version>1.6.7</version> - <configuration> - <skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <!-- This version supports the "deployAtEnd" parameter --> - <version>2.8</version> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - <!-- first disable the default Java plugins at various stages --> - <!-- maven-resources-plugin is called during "*resource" phases by default behavior. it prepares - the resources dir. we do not need it --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> - <version>2.6</version> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - <!-- maven-compiler-plugin is called during "compile" phases by default behavior. we do not need it --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.1</version> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - <!-- maven-jar-plugin is called during "compile" phase by default behavior. we do not need it --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>2.4</version> - <executions> - <execution> - <id>default-jar</id> - <phase/> - </execution> - </executions> - </plugin> - <!-- maven-install-plugin is called during "install" phase by default behavior. it tries to copy stuff under - target dir to ~/.m2. we do not need it --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <version>2.4</version> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - <!-- maven-surefire-plugin is called during "test" phase by default behavior. it triggers junit test. - we do not need it --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.12.4</version> - <configuration> - <skipTests>true</skipTests> - </configuration> - </plugin> - - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.2.1</version> - <configuration> - <executable>${session.executionRootDirectory}/mvn-phase-script.sh</executable> - <environmentVariables> - <!-- make mvn properties as env for our script --> - <MVN_PROJECT_GROUPID>${project.parent.groupId}</MVN_PROJECT_GROUPID> - <MVN_PROJECT_ARTIFACTID>${project.parent.artifactId}</MVN_PROJECT_ARTIFACTID> - <MVN_PROJECT_VERSION>${project.version}</MVN_PROJECT_VERSION> - <MVN_NEXUSPROXY>${onap.nexus.url}</MVN_NEXUSPROXY> - <MVN_RAWREPO_BASEURL_UPLOAD>${onap.nexus.rawrepo.baseurl.upload}</MVN_RAWREPO_BASEURL_UPLOAD> - <MVN_RAWREPO_BASEURL_DOWNLOAD>${onap.nexus.rawrepo.baseurl.download}</MVN_RAWREPO_BASEURL_DOWNLOAD> - <MVN_RAWREPO_SERVERID>${onap.nexus.rawrepo.serverid}</MVN_RAWREPO_SERVERID> - <MVN_DOCKERREGISTRY_SNAPSHOT>${onap.nexus.dockerregistry.snapshot}</MVN_DOCKERREGISTRY_SNAPSHOT> - <MVN_DOCKERREGISTRY_RELEASE>${onap.nexus.dockerregistry.release}</MVN_DOCKERREGISTRY_RELEASE> - <MVN_DOCKERREGISTRY_SNAPSHOT_SERVERID>${onap.nexus.dockerregistry.snapshot.serverid}</MVN_DOCKERREGISTRY_SNAPSHOT_SERVERID> - <MVN_DOCKERREGISTRY_RELEASE_SERVERID>${onap.nexus.dockerregistry.release.serverid}</MVN_DOCKERREGISTRY_RELEASE_SERVERID> - <MVN_PYPISERVER_BASEURL>${onap.nexus.pypiserver.baseurl}</MVN_PYPISERVER_BASEURL> - <MVN_PYPISERVER_SERVERID>${onap.nexus.pypiserver.serverid}</MVN_PYPISERVER_SERVERID> - </environmentVariables> - </configuration> - </plugin> - </plugins> - </pluginManagement> - - <plugins> - <!-- plugin> - <artifactId>maven-assembly-plugin</artifactId> - <version>2.4.1</version> - <configuration> - <descriptors> - <descriptor>assembly/dep.xml</descriptor> - </descriptors> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin --> - <!-- now we configure custom action (calling a script) at various lifecycle phases --> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.2.1</version> - <executions> - <execution> - <id>clean phase script</id> - <phase>clean</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <arguments> - <argument>${project.artifactId}</argument> - <argument>clean</argument> - </arguments> - </configuration> - </execution> - <execution> - <id>generate-sources script</id> - <phase>generate-sources</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <arguments> - <argument>${project.artifactId}</argument> - <argument>generate-sources</argument> - </arguments> - </configuration> - </execution> - <execution> - <id>compile script</id> - <phase>compile</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <arguments> - <argument>${project.artifactId}</argument> - <argument>compile</argument> - </arguments> - </configuration> - </execution> - <execution> - <id>package script</id> - <phase>package</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <arguments> - <argument>${project.artifactId}</argument> - <argument>package</argument> - </arguments> - </configuration> - </execution> - <execution> - <id>test script</id> - <phase>test</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <arguments> - <argument>${project.artifactId}</argument> - <argument>test</argument> - </arguments> - </configuration> - </execution> - <execution> - <id>install script</id> - <phase>install</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <arguments> - <argument>${project.artifactId}</argument> - <argument>install</argument> - </arguments> - </configuration> - </execution> - <execution> - <id>deploy script</id> - <phase>deploy</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <arguments> - <argument>${project.artifactId}</argument> - <argument>deploy</argument> - </arguments> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> |
