From 26d51b60fb298dd53e513ba42b544f5418c30902 Mon Sep 17 00:00:00 2001 From: maximesson Date: Tue, 23 Jul 2019 09:40:54 +0000 Subject: Documentation for generalised DFC Change-Id: Ic240ebf83c58b2f0dd231bd3933838614f716ff8 Issue-ID: DCAEGEN2-1662 Signed-off-by: maximesson --- docs/sections/images/DFC_config.png | Bin 0 -> 170402 bytes docs/sections/services/dfc/architecture.rst | 8 +- docs/sections/services/dfc/configuration.rst | 332 ++++++++++++++++++--------- docs/sections/services/dfc/release-notes.rst | 6 + 4 files changed, 234 insertions(+), 112 deletions(-) create mode 100644 docs/sections/images/DFC_config.png diff --git a/docs/sections/images/DFC_config.png b/docs/sections/images/DFC_config.png new file mode 100644 index 00000000..517c6c95 Binary files /dev/null and b/docs/sections/images/DFC_config.png differ diff --git a/docs/sections/services/dfc/architecture.rst b/docs/sections/services/dfc/architecture.rst index f25b855c..5592bf25 100644 --- a/docs/sections/services/dfc/architecture.rst +++ b/docs/sections/services/dfc/architecture.rst @@ -40,4 +40,10 @@ The event is received from the Message Router (MR), the files are fetched from a (DR). Both fetching of a file and publishing is retried a number of times with an increasing delay between each attempt. After a number of attempts, the DFC will log an error message and give up. Failing of processing of one file does not -affect the handling of others. \ No newline at end of file +affect the handling of others. + +Generalized DFC +"""""""""""""""" +From version 1.2.1 and onwards, the DFC has more general use. Instead of only managing PM files, any kind of files +are handled. The 'changeIdentifier' parameter in the file ready VES event (which is reported from the PNFs) is used +to define the file type of file to be handled. \ No newline at end of file diff --git a/docs/sections/services/dfc/configuration.rst b/docs/sections/services/dfc/configuration.rst index 76e822e7..bb60fa28 100644 --- a/docs/sections/services/dfc/configuration.rst +++ b/docs/sections/services/dfc/configuration.rst @@ -1,120 +1,230 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 +============================= Configuration and Performance ============================= -**datafile** configuration is controlled via a single JSON file called datafile_endpoints.json. -This is located under datafile-app-server/config. - -Json Configuration Explained -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Variables of interest (e.g. variables that should be inspected/modified for a specific runtime environment) are listed below for convenience. The entire file is provided later in this page for reference. - -dmaapConsumerConfiguration -"""""""""""""""""""""""""" - -.. code-block:: json - - "dmaapHostName": - "dmaapPortNumber": - "dmaapTopicName": "/events/unauthenticated.VES_NOTIFICATION_OUTPUT" - "dmaapProtocol": "http" - "dmaapUserName": "" - "dmaapUserPassword": "" - "dmaapContentType": "application/json" - "consumerId": "C12" - "consumerGroup": "OpenDcae-c12" - "timeoutMS": -1 - "messageLimit": 1 - -dmaapProducerConfiguration -"""""""""""""""""""""""""" - -.. code-block:: json - - "dmaapHostName": - "dmaapPortNumber": - "dmaapTopicName": "publish" - "dmaapProtocol": "https" - "dmaapUserName": "dradmin" - "dmaapUserPassword": "dradmin" - "dmaapContentType": "application/octet-stream" - -ftpesConfiguration -"""""""""""""""""" - -.. code-block:: json - - "keyCert": - "keyPassword": - "trustedCa": - "trustedCaPassword": - -securityConfiguration -""""""""""""""""""""" - -.. code-block:: json - - "trustStorePath": - "trustStorePasswordPath": - "keyStorePath": - "keyStorePasswordPath": - "enableDmaapCertAuth": . If false, all information above are ignored. - - - -Sample JSON configuration -""""""""""""""""""""""""" - -The format of the JSON configuration that drives all behavior of DFC is probably best described using an example: - -.. code-block:: json - - { - "configs": { - "dmaap": { - "dmaapConsumerConfiguration": { - "dmaapHostName": "localhost", - "dmaapPortNumber": 2222, - "dmaapTopicName": "/events/unauthenticated.VES_NOTIFICATION_OUTPUT", - "dmaapProtocol": "http", - "dmaapUserName": "", - "dmaapUserPassword": "", - "dmaapContentType": "application/json", - "consumerId": "C12", - "consumerGroup": "OpenDcae-c12", - "timeoutMS": -1, - "messageLimit": 1 - }, - "dmaapProducerConfiguration": { - "dmaapHostName": "localhost", - "dmaapPortNumber": 3907, - "dmaapTopicName": "publish", - "dmaapProtocol": "https", - "dmaapUserName": "dradmin", - "dmaapUserPassword": "dradmin", - "dmaapContentType": "application/octet-stream" - } - }, - "ftp": { - "ftpesConfiguration": { - "keyCert": "config/dfc.jks", - "keyPassword": "secret", - "trustedCa": "config/ftp.jks", - "trustedCaPassword": "secret" - } - }, - "security": { - "trustStorePath" : "change it", - "trustStorePasswordPath" : "change it", - "keyStorePath" : "change it", - "keyStorePasswordPath" : "change it", - "enableDmaapCertAuth" : "false" - } - } - } +Configuration +^^^^^^^^^^^^^ +By default, DFC handles the "PM_MEAS_FILES" change identifier and publishes these files on the "bulk_pm_feed" feed. +But it can also be configured to handle other change identifiers and publish them to other feeds, see picture below. + +.. image:: ../../images/DFC_config.png + +The configuration of DFC is controlled via a blueprint. + +Blueprint Configuration Explained +""""""""""""""""""""""""""""""""" + +For the communication with the Message Router, the user must provide the **host name**, **port**, and **protocol** of the DMaaP Message router. + +.. code-block:: yaml + :emphasize-lines: 2,6,10 + + inputs: + dmaap_mr_host: + type: string + description: dmaap messagerouter host + default: message-router.onap.svc.cluster.local + dmaap_mr_port: + type: integer + description: dmaap messagerouter port + default: 3904 + dmaap_mr_protocol: + type: string + description: dmaap messagerouter protocol + default: "http" + +The user can also specify which version of DFC to use. + +.. code-block:: yaml + :emphasize-lines: 2 + + inputs: + tag_version: + type: string + description: DFC image tag/version + default: "nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server:1.2.0" + +The user can also enable secure communication with the DMaaP Message Router. + +.. code-block:: yaml + :emphasize-lines: 2 + + inputs: + secureEnableCert: + type: boolean + description: enable certificate based connection with DMaap + default: false + +DFC can handle multiple change identifiers. This will require to create manually a new block for each change identifier. + +.. code-block:: yaml + :emphasize-lines: 2 + + streams_publishes: + dfcFeed00ChangeIdentifier: + dmaap_info: <> + type: data_router + +For each feed related to a change identifier the user must provide the **feed name**, and **feed description**. + +.. code-block:: yaml + :emphasize-lines: 2,6 + + inputs: + dfcFeed00Name: + type: string + description: The name of the feed the files will be published to. Should be used by the subscriber. + default: "bulk_pm_feed" + dfcFeed00Description: + type: string + description: A description of the feed the files will be published to. + default: "Feed for Bulk PM files" + +**Note!** The feed name provided should be used by the subscriber/s to set up the subscription to the feed. + +To dynamically create the feeds, DFC uses the DCAE DMaaP plugin. This means that for each feed the user must also add an +item under the "**node_templates**" section of the blueprint. The feed is identified within the blueprint with a feed identifier, "**dfcFeed00**" in the example. + +.. code-block:: yaml + :emphasize-lines: 2 + + node_templates: + dfcFeed00: + type: ccsdk.nodes.Feed + properties: + feed_name: { get_input: dfcFeed00Name } + feed_description: { get_input: dfcFeed00Description } + +To configure DFC micro service itself, the user must also add the change identifier and the feed identifier to the "**application_config**" section +under the tag "**dmaap.dmaapProducerConfiguration**" in the blueprint. **Note!** The identifier should be surrounded by **<<>>** for the feed's configuration to be bound to the DFC configuration in CBS. + +.. code-block:: yaml + :emphasize-lines: 4 + + application_config: + dmaap.dmaapProducerConfiguration: + - changeIdentifier: {get_input: dfcFeed00ChangeIdentifier} + feedInfo: <> + +And, lastly, to set up the publication to the feed, the feed identifier must be added to the "**streams_publishes**" section +of the blueprint. + +.. code-block:: yaml + :emphasize-lines: 2 + + streams_publishes: + - name: dfcFeed00 + type: data_router + location: "loc00" + +Sample blueprint configuration +"""""""""""""""""""""""""""""" + +The format of the blueprint configuration that drives all behavior of DFC is probably best described using an example. +The blueprint below configures DFC to handle the two feeds shown in the picture above. + +.. code-block:: yaml + + inputs: + dmaap_mr_host: + type: string + description: dmaap messagerouter host + default: message-router.onap.svc.cluster.local + dmaap_mr_port: + type: integer + description: dmaap messagerouter port + default: 3904 + dmaap_mr_protocol: + type: string + description: dmaap messagerouter protocol + default: "http" + tag_version: + type: string + description: DFC image tag/version + default: "nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server:1.2.0" + replicas: + type: integer + description: number of instances + default: 1 + secureEnableCert: + type: boolean + description: enable certificate based connection with DMaap + default: false + dfcFeed00Name: + type: string + description: The name of the feed the files will be published to. Should be used by the subscriber. + default: "bulk_pm_feed" + dfcFeed00Description: + type: string + description: A description of the feed the files will be published to. + default: "Feed for Bulk PM files" + dfcFeed01Name: + type: string + description: The name of the feed the files will be published to. Should be used by the subscriber. + default: "log_feed" + dfcFeed01Description: + type: string + description: A description of the feed the files will be published to. + default: "Feed for log files" + + node_templates: + dfcFeed00: + type: ccsdk.nodes.Feed + properties: + feed_name: { get_input: dfcFeed00Name } + feed_description: { get_input: dfcFeed00Description } + dfcFeed01: + type: ccsdk.nodes.Feed + properties: + feed_name: { get_input: dfcFeed01Name } + feed_description: { get_input: dfcFeed01Description } + + datafile-collector: + type: dcae.nodes.ContainerizedServiceComponentUsingDmaap + + relationships: + - type: ccsdk.relationships.publish_files + target: dfcFeed00 + - type: ccsdk.relationships.publish_files + target: dfcFeed01 + + interfaces: + cloudify.interfaces.lifecycle: + start: + inputs: + ports: + - concat: ["8100:0"] + - concat: ["8433:0"] + + properties: + service_component_type: 'dcae-datafile' + application_config: + dmaap.security.enableDmaapCertAuth: { get_input: secureEnableCert } + streams_publishes: + dfcFeed00ChangeIdentifier: + dmaap_info: <> + type: data_router + dfcFeed01ChangeIdentifier: + dmaap_info: <> + type: data_router + streams_subscribes: + dmaap_subscriber: + type: + "message_router" + dmaap_info: + topic_url: + { concat: [{ get_input: dmaap_mr_protocol },"://",{ get_input: dmaap_mr_host }, + ":",{ get_input: dmaap_mr_port },"/events/unauthenticated.VES_NOTIFICATION_OUTPUT/OpenDcae-c12/C12"]} + streams_publishes: + - name: dfcFeed00 + type: data_router + location: "loc00" + - name: dfcFeed01 + type: data_router + location: "loc00" Performance ^^^^^^^^^^^ diff --git a/docs/sections/services/dfc/release-notes.rst b/docs/sections/services/dfc/release-notes.rst index a0277d80..0025b838 100644 --- a/docs/sections/services/dfc/release-notes.rst +++ b/docs/sections/services/dfc/release-notes.rst @@ -4,6 +4,12 @@ Release Notes ============= +Version 1.2.1 +------------- + +The DFC is now generalized: it can handle any kind of files, not only PM files. + + Version 1.1.3 ------------- -- cgit 1.2.3-korg