From 64559741f071b37556b7745ffa2cdcc25f40af94 Mon Sep 17 00:00:00 2001 From: VENKATESH KUMAR Date: Wed, 29 Apr 2020 18:53:53 -0400 Subject: dcae design doc updates Change-Id: I7145f840d9a7de34bb6a615fe992ba22e1ff0380 Signed-off-by: VENKATESH KUMAR Issue-ID: DCAEGEN2-2024 Issue-ID: DCAEGEN2-1865 Signed-off-by: VENKATESH KUMAR --- docs/index.rst | 2 +- docs/sections/DCAE-MOD/DCAE-MOD-Architecture.rst | 173 -- docs/sections/DCAE-MOD/DCAE-MOD-Intro.rst | 45 - docs/sections/DCAE-MOD/DCAE-MOD-User-Guide.rst | 441 ---- docs/sections/DCAE-MOD/Glossary.rst | 184 -- docs/sections/DCAE-MOD/Roles.rst | 168 -- .../Request-body-of-Sample-Component.json | 328 --- .../Request-body-of-Sample-Data-Format.json | 2124 -------------------- .../Sample-Input-Files/tca-deploy.input.json | 21 - .../Sample-Input-Files/ves-deploy.input.json | 30 - docs/sections/DCAE-MOD/mod-index.rst | 14 - docs/sections/apis/deployment-handler.rst | 2 +- docs/sections/components/architecture/pieces.rst | 11 - .../components/architecture/service-discovery.rst | 36 - docs/sections/components/blueprint_generator.rst | 77 - docs/sections/components/component-development.rst | 22 - docs/sections/components/component-json-schema.rst | 753 ------- .../component-specification/cdap-specification.rst | 208 -- .../common-specification.rst | 1442 ------------- .../component-specification.rst | 16 - .../component-specification/configuration-grid.rst | 121 -- .../dmaap-connection-objects.rst | 276 --- .../docker-specification.rst | 351 ---- .../component-specification/streams-grid.rst | 149 -- docs/sections/components/component-type-cdap.rst | 87 - docs/sections/components/component-type-docker.rst | 465 ----- docs/sections/components/data-formats.rst | 238 --- docs/sections/components/dcae-cli/commands.rst | 590 ------ .../sections/components/dcae-cli/dcae-cli-tool.rst | 13 - docs/sections/components/dcae-cli/quickstart.rst | 156 -- docs/sections/components/dcae-cli/walkthrough.rst | 385 ---- docs/sections/components/glossary.rst | 210 -- docs/sections/components/images/IO.graffle | Bin 2748 -> 0 bytes docs/sections/components/images/dmd vision.graffle | Bin 4041 -> 0 bytes docs/sections/components/images/dmdvision.png | Bin 200850 -> 0 bytes docs/sections/components/images/io.png | Bin 131383 -> 0 bytes docs/sections/components/intro.rst | 105 - .../components/requirements-guidelines.rst | 12 - .../DCAE-MOD/DCAE-MOD-Architecture.rst | 178 ++ .../DCAE-MOD/DCAE-MOD-User-Guide.rst | 443 ++++ .../design-components/DCAE-MOD/DCAE-MOD-goals.rst | 43 + docs/sections/design-components/DCAE-MOD/Roles.rst | 168 ++ .../Request-body-of-Sample-Component.json | 328 +++ .../Request-body-of-Sample-Data-Format.json | 2124 ++++++++++++++++++++ .../Sample-Input-Files/tca-deploy.input.json | 21 + .../Sample-Input-Files/ves-deploy.input.json | 30 + .../design-components/blueprint_generator.rst | 84 + .../component-json-schema.rst | 933 +++++++++ .../component-type-docker.rst | 1437 +++++++++++++ .../component-specification/configuration-grid.rst | 118 ++ .../component-specification/data-formats.rst | 235 +++ .../dmaap-connection-objects.rst | 218 ++ .../docker-specification.rst | 347 ++++ .../index-component-specification.rst | 15 + .../component-specification/streams-grid.rst | 149 ++ docs/sections/design-components/glossary.rst | 169 ++ docs/sections/design-components/images/1.png | Bin 0 -> 57705 bytes docs/sections/design-components/images/10.png | Bin 0 -> 38497 bytes docs/sections/design-components/images/11.png | Bin 0 -> 11149 bytes docs/sections/design-components/images/12.png | Bin 0 -> 61495 bytes docs/sections/design-components/images/13.png | Bin 0 -> 251384 bytes docs/sections/design-components/images/14.png | Bin 0 -> 126604 bytes docs/sections/design-components/images/15.png | Bin 0 -> 126203 bytes docs/sections/design-components/images/16.png | Bin 0 -> 72017 bytes docs/sections/design-components/images/17.png | Bin 0 -> 69183 bytes docs/sections/design-components/images/18.png | Bin 0 -> 43446 bytes docs/sections/design-components/images/19.png | Bin 0 -> 744365 bytes docs/sections/design-components/images/2.png | Bin 0 -> 26016 bytes docs/sections/design-components/images/20.png | Bin 0 -> 689792 bytes docs/sections/design-components/images/21.png | Bin 0 -> 78966 bytes docs/sections/design-components/images/22.png | Bin 0 -> 146671 bytes docs/sections/design-components/images/23.png | Bin 0 -> 126591 bytes docs/sections/design-components/images/24.png | Bin 0 -> 113390 bytes docs/sections/design-components/images/25.png | Bin 0 -> 77247 bytes docs/sections/design-components/images/26.png | Bin 0 -> 123519 bytes docs/sections/design-components/images/27.png | Bin 0 -> 298476 bytes docs/sections/design-components/images/3.png | Bin 0 -> 97794 bytes docs/sections/design-components/images/4.png | Bin 0 -> 64151 bytes docs/sections/design-components/images/5.png | Bin 0 -> 47830 bytes docs/sections/design-components/images/6.png | Bin 0 -> 151219 bytes docs/sections/design-components/images/7.png | Bin 0 -> 412560 bytes docs/sections/design-components/images/8.png | Bin 0 -> 433473 bytes docs/sections/design-components/images/9.png | Bin 0 -> 69247 bytes .../images/DCAE-Mod-Architecture.png | Bin 0 -> 44962 bytes .../images/Onboarding-with-DCAE-MOD.png | Bin 0 -> 69293 bytes .../images/nifi-toolbar-components.png | Bin 0 -> 122427 bytes .../design-components/index-onboarding.rst | 18 + docs/sections/design-components/overview.rst | 107 + .../design-components/requirements-guidelines.rst | 279 +++ docs/sections/images/1.png | Bin 57705 -> 0 bytes docs/sections/images/10.png | Bin 38497 -> 0 bytes docs/sections/images/11.png | Bin 11149 -> 0 bytes docs/sections/images/12.png | Bin 61495 -> 0 bytes docs/sections/images/13.png | Bin 251384 -> 0 bytes docs/sections/images/14.png | Bin 126604 -> 0 bytes docs/sections/images/15.png | Bin 126203 -> 0 bytes docs/sections/images/16.png | Bin 72017 -> 0 bytes docs/sections/images/17.png | Bin 69183 -> 0 bytes docs/sections/images/18.png | Bin 43446 -> 0 bytes docs/sections/images/19.png | Bin 744365 -> 0 bytes docs/sections/images/2.png | Bin 26016 -> 0 bytes docs/sections/images/20.png | Bin 689792 -> 0 bytes docs/sections/images/21.png | Bin 78966 -> 0 bytes docs/sections/images/22.png | Bin 146671 -> 0 bytes docs/sections/images/23.png | Bin 126591 -> 0 bytes docs/sections/images/24.png | Bin 113390 -> 0 bytes docs/sections/images/25.png | Bin 77247 -> 0 bytes docs/sections/images/26.png | Bin 123519 -> 0 bytes docs/sections/images/27.png | Bin 298476 -> 0 bytes docs/sections/images/3.png | Bin 97794 -> 0 bytes docs/sections/images/4.png | Bin 64151 -> 0 bytes docs/sections/images/5.png | Bin 47830 -> 0 bytes docs/sections/images/6.png | Bin 151219 -> 0 bytes docs/sections/images/7.png | Bin 412560 -> 0 bytes docs/sections/images/8.png | Bin 433473 -> 0 bytes docs/sections/images/9.png | Bin 69247 -> 0 bytes docs/sections/images/DCAE-Mod-Architecture.png | Bin 44962 -> 0 bytes docs/sections/images/Onboarding-with-DCAE-MOD.png | Bin 69293 -> 0 bytes docs/sections/images/nifi-toolbar-components.png | Bin 122427 -> 0 bytes docs/sections/release-notes.rst | 51 +- 120 files changed, 7485 insertions(+), 9265 deletions(-) delete mode 100644 docs/sections/DCAE-MOD/DCAE-MOD-Architecture.rst delete mode 100644 docs/sections/DCAE-MOD/DCAE-MOD-Intro.rst delete mode 100644 docs/sections/DCAE-MOD/DCAE-MOD-User-Guide.rst delete mode 100644 docs/sections/DCAE-MOD/Glossary.rst delete mode 100644 docs/sections/DCAE-MOD/Roles.rst delete mode 100644 docs/sections/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json delete mode 100644 docs/sections/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Data-Format.json delete mode 100644 docs/sections/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json delete mode 100644 docs/sections/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json delete mode 100644 docs/sections/DCAE-MOD/mod-index.rst delete mode 100755 docs/sections/components/architecture/pieces.rst delete mode 100755 docs/sections/components/architecture/service-discovery.rst delete mode 100644 docs/sections/components/blueprint_generator.rst delete mode 100644 docs/sections/components/component-development.rst delete mode 100644 docs/sections/components/component-json-schema.rst delete mode 100755 docs/sections/components/component-specification/cdap-specification.rst delete mode 100755 docs/sections/components/component-specification/common-specification.rst delete mode 100644 docs/sections/components/component-specification/component-specification.rst delete mode 100755 docs/sections/components/component-specification/configuration-grid.rst delete mode 100755 docs/sections/components/component-specification/dmaap-connection-objects.rst delete mode 100755 docs/sections/components/component-specification/docker-specification.rst delete mode 100755 docs/sections/components/component-specification/streams-grid.rst delete mode 100755 docs/sections/components/component-type-cdap.rst delete mode 100755 docs/sections/components/component-type-docker.rst delete mode 100755 docs/sections/components/data-formats.rst delete mode 100644 docs/sections/components/dcae-cli/commands.rst delete mode 100644 docs/sections/components/dcae-cli/dcae-cli-tool.rst delete mode 100755 docs/sections/components/dcae-cli/quickstart.rst delete mode 100755 docs/sections/components/dcae-cli/walkthrough.rst delete mode 100644 docs/sections/components/glossary.rst delete mode 100644 docs/sections/components/images/IO.graffle delete mode 100644 docs/sections/components/images/dmd vision.graffle delete mode 100644 docs/sections/components/images/dmdvision.png delete mode 100644 docs/sections/components/images/io.png delete mode 100755 docs/sections/components/intro.rst delete mode 100644 docs/sections/components/requirements-guidelines.rst create mode 100644 docs/sections/design-components/DCAE-MOD/DCAE-MOD-Architecture.rst create mode 100644 docs/sections/design-components/DCAE-MOD/DCAE-MOD-User-Guide.rst create mode 100644 docs/sections/design-components/DCAE-MOD/DCAE-MOD-goals.rst create mode 100644 docs/sections/design-components/DCAE-MOD/Roles.rst create mode 100644 docs/sections/design-components/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json create mode 100644 docs/sections/design-components/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Data-Format.json create mode 100644 docs/sections/design-components/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json create mode 100644 docs/sections/design-components/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json create mode 100644 docs/sections/design-components/blueprint_generator.rst create mode 100644 docs/sections/design-components/component-specification/component-json-schema.rst create mode 100755 docs/sections/design-components/component-specification/component-type-docker.rst create mode 100755 docs/sections/design-components/component-specification/configuration-grid.rst create mode 100755 docs/sections/design-components/component-specification/data-formats.rst create mode 100755 docs/sections/design-components/component-specification/dmaap-connection-objects.rst create mode 100755 docs/sections/design-components/component-specification/docker-specification.rst create mode 100644 docs/sections/design-components/component-specification/index-component-specification.rst create mode 100755 docs/sections/design-components/component-specification/streams-grid.rst create mode 100644 docs/sections/design-components/glossary.rst create mode 100644 docs/sections/design-components/images/1.png create mode 100644 docs/sections/design-components/images/10.png create mode 100644 docs/sections/design-components/images/11.png create mode 100644 docs/sections/design-components/images/12.png create mode 100644 docs/sections/design-components/images/13.png create mode 100644 docs/sections/design-components/images/14.png create mode 100644 docs/sections/design-components/images/15.png create mode 100644 docs/sections/design-components/images/16.png create mode 100644 docs/sections/design-components/images/17.png create mode 100644 docs/sections/design-components/images/18.png create mode 100644 docs/sections/design-components/images/19.png create mode 100644 docs/sections/design-components/images/2.png create mode 100644 docs/sections/design-components/images/20.png create mode 100644 docs/sections/design-components/images/21.png create mode 100644 docs/sections/design-components/images/22.png create mode 100644 docs/sections/design-components/images/23.png create mode 100644 docs/sections/design-components/images/24.png create mode 100644 docs/sections/design-components/images/25.png create mode 100644 docs/sections/design-components/images/26.png create mode 100644 docs/sections/design-components/images/27.png create mode 100644 docs/sections/design-components/images/3.png create mode 100644 docs/sections/design-components/images/4.png create mode 100644 docs/sections/design-components/images/5.png create mode 100644 docs/sections/design-components/images/6.png create mode 100644 docs/sections/design-components/images/7.png create mode 100644 docs/sections/design-components/images/8.png create mode 100644 docs/sections/design-components/images/9.png create mode 100644 docs/sections/design-components/images/DCAE-Mod-Architecture.png create mode 100644 docs/sections/design-components/images/Onboarding-with-DCAE-MOD.png create mode 100644 docs/sections/design-components/images/nifi-toolbar-components.png create mode 100644 docs/sections/design-components/index-onboarding.rst create mode 100755 docs/sections/design-components/overview.rst create mode 100644 docs/sections/design-components/requirements-guidelines.rst delete mode 100644 docs/sections/images/1.png delete mode 100644 docs/sections/images/10.png delete mode 100644 docs/sections/images/11.png delete mode 100644 docs/sections/images/12.png delete mode 100644 docs/sections/images/13.png delete mode 100644 docs/sections/images/14.png delete mode 100644 docs/sections/images/15.png delete mode 100644 docs/sections/images/16.png delete mode 100644 docs/sections/images/17.png delete mode 100644 docs/sections/images/18.png delete mode 100644 docs/sections/images/19.png delete mode 100644 docs/sections/images/2.png delete mode 100644 docs/sections/images/20.png delete mode 100644 docs/sections/images/21.png delete mode 100644 docs/sections/images/22.png delete mode 100644 docs/sections/images/23.png delete mode 100644 docs/sections/images/24.png delete mode 100644 docs/sections/images/25.png delete mode 100644 docs/sections/images/26.png delete mode 100644 docs/sections/images/27.png delete mode 100644 docs/sections/images/3.png delete mode 100644 docs/sections/images/4.png delete mode 100644 docs/sections/images/5.png delete mode 100644 docs/sections/images/6.png delete mode 100644 docs/sections/images/7.png delete mode 100644 docs/sections/images/8.png delete mode 100644 docs/sections/images/9.png delete mode 100644 docs/sections/images/DCAE-Mod-Architecture.png delete mode 100644 docs/sections/images/Onboarding-with-DCAE-MOD.png delete mode 100644 docs/sections/images/nifi-toolbar-components.png diff --git a/docs/index.rst b/docs/index.rst index 8d3496cd..a3ed0e03 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,7 @@ Data Collection, Analytics, and Events (DCAE) ./sections/tls_enablement.rst ./sections/sdk/index.rst ./sections/configuration.rst - ./sections/components/component-development.rst + ./sections/design-components/index-onboarding.rst ./sections/services/serviceindex.rst ./sections/release-notes.rst diff --git a/docs/sections/DCAE-MOD/DCAE-MOD-Architecture.rst b/docs/sections/DCAE-MOD/DCAE-MOD-Architecture.rst deleted file mode 100644 index 4aaf5a08..00000000 --- a/docs/sections/DCAE-MOD/DCAE-MOD-Architecture.rst +++ /dev/null @@ -1,173 +0,0 @@ -===================== -DCAE MOD Architecture -===================== - - -DCAE MOD is composed of a mix of components developed in ONAP and other -components taken from the Apache Nifi project and modified for -appropriate use. - -The goal of the project is to stream line the construction, management, -and evolution of DCAE flows from role to role, from environment to -environment, and from release to release. - -DCAE MOD is composed of three functional areas: onboarding, design, and -distribution. - - -Below illustrations describe the architecture of DCAE-MOD and show the -usage flow in DCAE MOD- - -|image0| - -|image1| - -Onboarding API --------------- - -It is a component developed to onboard -models/components/microservices (spec files) into DCAE MOD. - -Genprocessor ------------- - -It has been developed in Java. This project is a tool to -experiment with generating a Nifi Processor POJO from a DCAE component -spec. - -Nifi Web UI ------------ - -It is a component taken from the Apache Nifi Project but modified for -use in the MOD project. - -Apache NiFi is a dataflow system based on the concepts of flow-based -programming. It supports powerful and scalable directed graphs of data -routing, transformation, and system mediation logic. NiFi has a -web-based user interface for design, control, feedback, and monitoring -of dataflows. It is highly configurable along several dimensions of -quality of service, such as loss-tolerant versus guaranteed delivery, -low latency versus high throughput, and priority-based queuing. NiFi -provides fine-grained data provenance for all data received, forked, -joined cloned, modified, sent, and ultimately dropped upon reaching its -configured end-state. - -The NiFi UI provides mechanisms for creating automated dataflows, as -well as visualizing, editing, monitoring, and administering those -dataflows. The UI can be broken down into several segments, each -responsible for different functionality of the application. This section -provides screenshots of the application and highlights the different -segments of the UI. Each segment is discussed in further detail later in -the document. - -The users of Apache Nifi will find that it is used very differently than -intended to serve our purpose in the DCAE-MOD project. - - -Registry API ------------- - -This component taken from the Apache Nifi project, is a REST API that -provides an interface to a registry with operations for saving, -versioning, reading NiFi flows and components. - -Distributor API ---------------- - -It is a component developed using the Flask framework in Python. -It is a HTTP API to manage distribution targets for DCAE design. -Distribution targets are DCAE runtime environments that have been -registered and are enabled to accept flow design changes that are to be -orchestrated in that environment. - -Flow Based Programming (FBP) ----------------------------- - -NiFi’s fundamental design concepts closely relate to the main ideas of -Flow Based -Programming `[fbp] `__. - -For more information on how some of the main NiFi concepts map to FBP, -check out https://nifi.apache.org/docs/nifi-docs/html/overview.html - -Runtime API ------------ - -It is developed in Java’s Spring Boot framework. It -is a HTTP API to support runtime environment for DCAE-MOD. It has two -major functionalities: - -1. It accepts changes on the flow-graph via fbp protocol - -2. It generates and distributes blueprints based on the change made on - the flow-graph - -Blueprint Generator -------------------- - -This tool allows the user to create a blueprint from a component spec json file. -This tool is used by the runtime api. - -Inventory API -------------- - -DCAE Inventory is a web -service that provides the following: - -1. Real-time data on all DCAE services and their components - -2. Comprehensive details on available DCAE service types - -DCAE Inventory is a composite API that relies on other APIs to obtain -resources on underlying components and uses these resources to compose a -DCAE service resource. In addition, DCAE Inventory will store data that -is unique to the DCAE service level including: - -1. DCAE service metadata - -2. DCAE service type description and composition details - -3. Relationships between DCAE service and DCAE service types and their - respective VNF and VNF types - -DCAE Inventory has a REST interface to service client requests. It has a -well-defined query interface that would filter result sets based upon -resource attributes. - -Here, think of it as a back end API for the DCAE dashboard. The runtime -posts Cloudify Blueprints to this API so they show up in the DCAE -dashboard. - -DCAE Dashboard --------------- - -The DCAE dashboard provides visibility into running DCAE services for -operational purposes. It queries the DCAE Inventory for aggregate -details on all the running DCAE services and for getting up-to-date -status information on DCAE services and their components. - -End-to-End Flow ---------------- - -A model/component/microservice can be onboarded by a ms Developer by -posting a spec file on the onboarding API. Alternatively, an Acumos -model can be onboarded using the Acumos Adapter. Once successfully -onboarded, the genprocessor converts converts them to jars and onboards -them into Nifi i.e DCAE MOD. These artifacts are now available to use -from the Modified Nifi Web UI i.e DCAE Designer. - -The registry api offers version control and retrieval for flows. The -distributor api can be used to set distribution targets. Once a flow is -designed and distributed, it goes to the distributor api which is -supposed to post graph changes (in accordance with fbp) to the runtime -api. The runtime api generates and distributes blueprints based on the -change made on the flow-graph. These blueprints received by the DCAE -inventory can then be viewed and deployed from the DCAE dashboard. - - - -.. |image0| image:: ../images/DCAE-Mod-Architecture.png - -.. |image1| image:: ../images/Onboarding-with-DCAE-MOD.png - -.. |image2| image:: ../images/nifi-toolbar-components.png diff --git a/docs/sections/DCAE-MOD/DCAE-MOD-Intro.rst b/docs/sections/DCAE-MOD/DCAE-MOD-Intro.rst deleted file mode 100644 index 44b3489f..00000000 --- a/docs/sections/DCAE-MOD/DCAE-MOD-Intro.rst +++ /dev/null @@ -1,45 +0,0 @@ -============ -Introduction -============ - -MOD stands for "micro-service onboarding and design" and the project is -an effort to reboot the onboarding and design experience in DCAE. - - -**Goals and Stretch Goals** ---------------------------- - - -- Due to resource constraints, there are mismatched capabilities between SDC/DCAE-DS and DCAE mS deployment. - -- Due to #1, mS developers upload handcrafted blueprint, and stay involved throughout the deployment process. This also ties mS development to specific Cloudify implementation. - -- There is no Service Assurance flow design in SDC/DCAE-DS, and so there are no reusable flow designs for the Service Designer. - -- There is extensive reliance on developers’ involvement in providing [Inputs.json] as runtime configurations for mS deployment. - -- There is no E2E tracking of the microservice lifecycle. - - -**To address these problems, the new DCAE MOD, replacing the mS onboarding & DCAE-DS in SDC, aims to -** - - - -- Move DCAE mS onboarding & design from SDC project to DCAE Project. - -- Provide simplified mS Onboarding, Service Assurance flow design, & mS microservice design time & runtime configurations to support developers, service designers, and operations. - -- Auto-generate blueprint at the end of the design process, not onboarded before the design process. - -- Support Policy onboarding & artifact distribution to Policy/CLAMP to support Self Service Control Loop. - -- Streamline the process of constructing to deploying flows, Provide the ability to track flows - capture and store the progress and evolution of flows and Provide clear coordination and accountability i.e Provide catalog & data for microservice lifecycle tracking. It fits the ECOMP's release process and must provide clear visibility along the entire process and across different environments. - -- Support automated adaptation of ML model from Acumos to DCAE design & runtime environment through the Acumos Adapter. - -- DCAE-MOD is developed by the DCAE team to ensure consistency across all DCAE implementation, with the long term objective to integrate with SDC as part of the Design Platform. - -- Integrate with ONAP User Experience portals (initially ONAP portal, later SDC portal). - - - diff --git a/docs/sections/DCAE-MOD/DCAE-MOD-User-Guide.rst b/docs/sections/DCAE-MOD/DCAE-MOD-User-Guide.rst deleted file mode 100644 index 9a75f7f3..00000000 --- a/docs/sections/DCAE-MOD/DCAE-MOD-User-Guide.rst +++ /dev/null @@ -1,441 +0,0 @@ -=================== -DCAE MOD User Guide -=================== - - -- `Types of Users and Usage - Instructions: <#DCAEMODUserGuide(draft)-TypesofUsersand>`__ - -- `1.    Deployment of DCAE MOD components via Helm - charts <#DCAEMODUserGuide(draft)-1.DeploymentofD>`__ - - - `Using DCAE MOD without an Ingress - Controller <#DCAEMODUserGuide(draft)-UsingDCAEMODwit>`__ - -- `2.    Configuring DCAE - mod <#DCAEMODUserGuide(draft)-2.ConfiguringDC>`__ - -- `3.    Design & Distribution - Flow <#DCAEMODUserGuide(draft)-3.Design&Distri>`__ - -Types of Users and Usage Instructions: -====================================== - -+-------+-----------------------------+-----------------------------+ -| Sr.No | User | Usage Instructions | -+=======+=============================+=============================+ -| 1. | Developers who are looking | -        Access the Nifi  | -| | to onboard their mS | Web UI url provided to you | -| | | | -| | | -        Follow steps  2.c | -| | | to 2.f | -| | | | -| | | -        You should be able | -| | | to see your microservices | -| | | in the Nifi Web UI by | -| | | clicking and dragging | -| | | ‘Processor’ on the canvas, | -| | | and searching for the name | -| | | of the | -| | | micros | -| | | ervice/component/processor. | -+-------+-----------------------------+-----------------------------+ -| 2. | Designers who are building | -        Access the Nifi  | -| | the flows through UI and | Web UI url provided to you | -| | triggering distribution | | -| | | -        Follow steps 3 to | -| | | the end of the document | -+-------+-----------------------------+-----------------------------+ -| 3. | Infrastructure/ Admins who | -        Follow start to | -| | want to stand up DCAE Mod | the end | -| | and validate it | | -+-------+-----------------------------+-----------------------------+ - - -1.    Deployment of DCAE MOD components via Helm charts -======================================================= - -The DCAE MOD components are deployed using the standard ONAP OOM -deployment process.   When deploying ONAP using the helm deploy command, -DCAE MOD components are deployed when the dcaemod.enabled flag is set to -true, either via a --set option on the command line or by an entry in an -overrides file.  In this respect, DCAE MOD is no different from any -other ONAP subsystem. - -The default DCAE MOD deployment relies on an nginx ingress controller -being available in the Kubernetes cluster where DCAE MOD is being -deployed.   The Rancher RKE installation process sets up a suitable -ingress controller.   In order to enable the use of the ingress -controller, it is necessary to override the OOM default global settings -for ingress configuration.   Specifically, the installation needs to set -the following configuration in an override file:: - - global: - ingress: - enabled: true - virtualhost: - enabled: false - -When DCAE MOD is deployed with an ingress controller, several endpoints -are exposed outside the cluster at the ingress controller's external IP -address and port.   (In the case of a Rancher RKE installation, there is -an ingress controller on every worker node, listening at the the -standard HTTP port (80).)  These exposed endpoints are needed by users -using machines outside the Kubernetes cluster. - -+--------------+--------------------------------------------------+--------------------------+ -| **Endpoint** | ** Routes to (cluster | **Description** | -| | internal address)** | | -+==============+==================================================+==========================+ -| /nifi | http://dcaemod-designtool:8080/nifi | Design tool Web UI | -| | | | -+--------------+--------------------------------------------------+--------------------------+ -| /nifi-api | http://dcaemod-designtool:8080/nifi-api | Design tool API | -| | | | -+--------------+--------------------------------------------------+--------------------------+ -| /nifi-jars | http://dcaemod-nifi-registry:18080/nifi-jars | Flow registry listing of | -| | | JAR files built from | -| | | component specs | -+--------------+--------------------------------------------------+--------------------------+ -| /onboarding | http://dcaemod-onboarding-api:8080/onboarding | Onboarding API | -| | | | -+--------------+--------------------------------------------------+--------------------------+ -| /distributor | http://dcaemod-distributor-api:8080/distributor | Distributor API | -| | | | -+--------------+--------------------------------------------------+--------------------------+ - -| To access the design Web UI, for example, a user would use the URL : - http://*ingress_controller_address:ingress_controller_port*/nifi. -| *ingress_controller_address* is the the IP address or DNS FQDN of the - ingress controller and -| *ingress_controller_port* is the port on which the ingress controller - is listening for HTTP requests.  (If the port is 80, the HTTP default, - then there is no need to specify a port.) - -There are two additional *internal* endpoints that users need to know, -in order to configure a registry client and a distribution target in the -design tool's controller settings. - -+------------------------+--------------------------------------------+ -| **Configuration Item** | **Endpoint URL** | -+========================+============================================+ -| Registry client | http://dcaemod-nifi-registry:18080 | -+------------------------+--------------------------------------------+ -| Distribution target | http://dcaemod-runtime-api:9090 | -+------------------------+--------------------------------------------+ - -Using DCAE MOD without an Ingress Controller - - -Not currently supported - -2.    Configuring DCAE mod -========================== - -**a. Configure Nifi Registry url** - -Next check Nifi settings by selecting the Hamburger button in the Nifi -UI. It should lead you to the Nifi Settings screen - -|image16| - -|image3| - -Add a registry client. The Registry client url will be -http://dcaemod-nifi-registry:18080 - -|image4| - - -**b. Add distribution target which will be the runtime api url** - -Set the distribution target in the controller settings - -|image17| - -Distribution target URL will be -`http://dcaemod-runtime-api:9090 `__ - - - -Now let’s access the Nifi (DCAE designer) UI - http:///nifi - -IPAddress is the host address or the DNS FQDN, if there is one, for one of the Kubernetes nodes. - -|image0| - - -**c. Get the artifacts to test and onboard.** - -Let's fetch the artifacts/ spec files - -**Sample Component DCAE-VES-Collector :** https://git.onap.org/dcaegen2/collectors/ves/tree/dpo/spec/vescollector-componentspec.json - -**Sample Data Format :** https://git.onap.org/dcaegen2/collectors/ves/tree/dpo/data-formats/VES-5.28.4-dataformat.json - -For the purpose of onboarding, a Sample Request body should be of the type -:: - - { "owner": "", "spec": } - -where the json object inside the spec field can be a component spec json. - -Request bodies of this type will be used in the onboarding requests you make using curl or the onboarding swagger interface. - -**The prepared Sample Request body for a component dcae-ves-collector looks like -so –** - -See :download:`VES Collector Spec <./Sample-Input-Files/Request-body-of-Sample-Component.json>` - -**The prepared Sample request body for a sample data format  looks like so -** - -See :download:`VES data Format <./Sample-Input-Files/Request-body-of-Sample-Data-Format.json>` - - - -**d. To onboard a data format and a component** - -Each component has a description that tells what it does. - -These requests would be of the type - -curl -X POST http:///onboarding/dataformats     -H "Content-Type: application/json" -d -@ - -curl -X POST http:///onboarding/components     -H "Content-Type: application/json" -d -@ - -In our case, - -curl -X POST http:///onboarding/dataformats     -H "Content-Type: application/json" -d @ - -curl -X POST http:///onboarding/components    -H "Content-Type: application/json" -d @ - - - - -**e. Verify the resources were created using** - -curl -X GET http:///onboarding/dataformats - -curl -X GET http:///onboarding/components - -**f. Verify the genprocessor (which polls onboarding periodically to convert component specs to nifi processor), converted the component** - -Open http:///nifi-jars in a browser. - -These jars should now be available for you to use in the nifi UI as -processors - -|image1| - -3.    Design & Distribution Flow -================================ - - -**a**. To start creating flows, we need to create a process group first. The -name of the process group will be the name of the flow. Drag and Drop on -the canvas, the ‘Processor Group’ icon from the DCAE Designer bar on the -top. - -|image2| - - -Now enter the process group by double clicking it, - -You can now drag and drop on the canvas ‘Processor’ icon from the top -DCAE Designer tab. You can search for a particular component in the -search box that appears when you attempt to drag the ‘Processor’ icon to -the canvas. - -|image5| - -If the Nifi registry linking worked, you should see the “Import” button -when you try to add a Processor or Process group to the Nifi canvas, -like so- - -|image6| - -By clicking on the import button, we can import already created saved -and version controlled flows from the Nifi registry, if they are -present. - -|image7| - -We can save created flows by version controlling them like so starting -with a 'right click' anywhere on the canvas- - -|image8| - -Ideally you would name the flow and process group the same, because -functionally they are similar. - -|image9| - -When the flow is checked in, the bar at the bottom shows a green -checkmark - -|image10| - -Note: Even if you move a component around on the canvas, and its -position on the canvas changes, it is recognized as a change, and it -will have to recommitted. - -You can add additional components in your flow and connect them. - -DcaeVesCollector connects to DockerTcagen2. - -|image11| - -|image12| - -|image13| - -Along the way you need to also provide topic names in the settings -section. These can be arbitrary names. - -|image14| - -To recap, see how DcaeVesCollector connects to DockerTcagen2. Look at -the connection relationships. Currently there is no way to validate -these relationships. Notice how it is required to name the topics by -going to Settings. - -The complete flow after joining our components looks like so - -|image15| - - -**b. Submit/ Distribute the flow:** - -Once your flow is complete and saved in the Nifi registry, you can -choose to submit it for distribution. - -|image18| - -If the flow was submitted successfully to the runtime api, you should -get a pop up a success message like so - - -|image19| - -At this step, the design was packaged and sent to Runtime api. - -The runtime is supposed to generate the blueprint out of the packaged -design/flow and push it to the DCAE inventory and the DCAE Dasboard. - -**c. Checking the components in the DCAE Dashboard** - -You should see the generated artifact/ blueprint in the DCAE Dashboard -dashboard at https://:30418/ccsdk-app/login_external.htm in -our deployment. The name for each component will be appended by the flow -name followed by underscore followed by the component’s name. - -The credentials to access the DCAE Dashboard are- - -:: - -Login: su1234 - -Password: fusion - - -|image20| - -|image21| - -|image22| - -The generated Blueprint can be viewed. - -|image23| - -Finally, the generated Blueprint can be deployed. - -|image24| - -You can use/import the attached input configurations files to deploy. Drag and Drop these sample JSON files to fill in the configuration values. -See :download:`VES Collector Input Configuration <./Sample-Input-Files/ves-deploy.input.json>` -See :download:`Tcagen2 Input Configuration <./Sample-Input-Files/tca-deploy.input.json>` - -|image25| - -|image26| - -.. |image0| image:: ../images/1.png - :width: 6.5in - :height: 1.08333in -.. |image1| image:: ../images/2.png - :width: 6.5in - :height: 1.58333in -.. |image2| image:: ../images/3.png - :width: 5.83333in - :height: 3.58333in -.. |image3| image:: ../images/4.png - :width: 4.91667in - :height: 2.16667in -.. |image4| image:: ../images/5.png - :width: 6.5in - :height: 2.66667in -.. |image5| image:: ../images/6.png - :width: 6.5in - :height: 3.33333in -.. |image6| image:: ../images/7.png - :width: 4.91667in - :height: 2.25in -.. |image7| image:: ../images/8.png - :width: 4.91667in - :height: 2.58333in -.. |image8| image:: ../images/9.png - :width: 6.5in - :height: 4.58333in -.. |image9| image:: ../images/10.png - :width: 6.5in - :height: 4in -.. |image10| image:: ../images/11.png - :width: 4.91667in - :height: 0.41667in -.. |image11| image:: ../images/12.png - :width: 6.33333in - :height: 3.16667in -.. |image12| image:: ../images/13.png - :width: 6in - :height: 2.66667in -.. |image13| image:: ../images/14.png - :width: 6.5in - :height: 3.41667in -.. |image14| image:: ../images/15.png - :width: 6.5in - :height: 3.58333in -.. |image15| image:: ../images/16.png - :width: 6.5in - :height: 2.25in -.. |image16| image:: ../images/17.png - :width: 6.5in - :height: 2.83333in -.. |image17| image:: ../images/18.png - :width: 6.5in - :height: 3.08333in -.. |image18| image:: ../images/19.png - :width: 4.91667in - :height: 1.91667in -.. |image19| image:: ../images/20.png - :width: 4.91667in - :height: 2.41667in -.. |image20| image:: ../images/21.png - :width: 6.5in - :height: 2.41667in -.. |image21| image:: ../images/22.png - :width: 6.5in - :height: 3in -.. |image22| image:: ../images/23.png - :width: 6.5in - :height: 2.16667in -.. |image23| image:: ../images/24.png - :width: 6.5in - :height: 2.83333in -.. |image24| image:: ../images/25.png - :width: 6.5in - :height: 3in -.. |image25| image:: ../images/26.png -.. |image26| image:: ../images/27.png - - diff --git a/docs/sections/DCAE-MOD/Glossary.rst b/docs/sections/DCAE-MOD/Glossary.rst deleted file mode 100644 index b4c2beb2..00000000 --- a/docs/sections/DCAE-MOD/Glossary.rst +++ /dev/null @@ -1,184 +0,0 @@ -======== -Glossary -======== - - -A&AI - Active and Available Inventory -------------------------------------- -Inventory DB for all network components - - -CLAMP ------ -Non DCAE Platform Component - Controls the input and processing for -Closed Loop services. - - -Closed Loop ------------ -Services designed to monitor and report back to a controlling function -that automatically deals with the event reported without human -interaction. - - - -Cloudify --------- -Open Source application and network orchestration framework, based on -TOSCA used in DCAE to deploy platform and service components from -Cloudify Blueprints. Refer to `Architecture `__ -for more information. - - -Cloudify Blueprints -------------------- -YAML formatted file used by Cloudify to deploy platform and service -components. Contains all the information needed for installation. - - -Consul ------- -Opensource Platform Component that supports Service Discovery, -Configuration, and Healthcheck. Refer to -`Architecture `__ for more information. - -Component ---------- -Refers to a DCAE service component which is a single micro-service that -is written to be run by the DCAE platform and to be composeable to form -a DCAE service. That composition occurs in the SDC. - - -Config Binding Service ----------------------- -DCAE Platform Component - Service Components use Config Binding Service -to access Consul and retrieve configuration variables. - - -Component Specification ------------------------ -JSON formatted file that fully describes a component and its interfaces - - -Data Format / Data Format Specification ---------------------------------------- -JSON formatted file that fully describes a components input or output - - -Deployment Handler ------------------- -DCAE Platform Component - Receives Input from DTI Handler, and talks to -Cloudify to deploy components. - - -Design-Time ------------ -Refers to when the System Designer uses the SDC Tool to compose services -from components in the SDC catalog. The Designer can provide input to -assign/override defaults for configuration for any parameter with the -property 'designer\_editable' set to 'true'. - - -Deploy-Time ------------ -Refers to when a service is being deployed. This can be done -automatically via the SDC Tool, or manually via the DCAE Dashboard or -CLAMP UI. When manually deployed, DevOps can provide input to -assign/override defaults for configuration for any parameter with the -property 'sourced\_at\_deployment' set to 'true'. - - -Docker ------- -Opensource Platform for development of containerized applications in the -cloud. Many DCAE service components and all DCAE collectors are written -utilizing Docker. - - -Dmaap ------ -AT&T data transportation service platform that supports message-based -topics and file-based feeds. Runs locally at the Edge and Centrally. - - -Inventory ---------- -DCAE Platform Component - Postgres DB containing Cloudify Blueprints for -platform and service components. - - -Policy ------- -Refers to the setting of configuration parameters for a component, by -Operations via the Policy UI. - - -Policy Handler --------------- -DCAE Platform Component that received Policy updates from Policy UI - - -Policy UI ---------- -Non DCAE Component - Policy User Interace where Operations assigns -values to configuraton specified for this. - - -Run-Time --------- -Refers to the when a service is running on the platform. Often used in -conjunction with DTI events which occur at Run-time. - - -SCH - Service Change Handler ----------------------------- -DCAE Platform Component - Receives updates from SDC and updates -Inventory - - -SDC - Service Design and Creation ---------------------------------- -Tool used by Service Designers to compose services from SDC catalog -artifacts. Once services are created, Cloudify Blueprints can be -generated to deployment and installation. - - -SDC Catalog ------------ -Catalog of composable Components and Data Formats to be used in the SDC -Tool to create services. Currently, there is no access to the SDC -Catalog from the dcae\_cli tool. Artifacts are manually placed there -after testing. Every catalog artifact has a ``UUID``, a globally unique -identifier that identifies that artifact. - -Self-Service ------------- -Refers to services that are supported by SDC, and that are automatically -installed as a result of a Service Designer's composition and submission -of a service. Only a handful of services are 'self-service' currently. -Most require manual effort to generate the Tosca Model files and -Cloudify Blueprints. - - -Service Component ------------------ -Microservice that provides network monitoring or analytic function on -the DCAE platform. - - -Service -------- -Generally composed of multiple service components, which is deployed to -the DCAE platform. - - -Tosca Model ------------ -Model generated from validated component specification, (stored in SDC -catalog for Self-Service components), and used as input to generate -Cloudify Blueprints - - -VNF - Virtualized Network Function ----------------------------------- -A network function that runs on one or more virtualized machines. diff --git a/docs/sections/DCAE-MOD/Roles.rst b/docs/sections/DCAE-MOD/Roles.rst deleted file mode 100644 index c3460841..00000000 --- a/docs/sections/DCAE-MOD/Roles.rst +++ /dev/null @@ -1,168 +0,0 @@ -===== -Roles -===== - - -Here is master list of all roles involved in ECOMP with DCAE: - -- System engineer - -- Component developer/expert - components are also referred to as - micro-services but include collectors, analytics - -- Designer - -- Tester - -- Operations - -- Platform developer - -- Manager - - -System engineer ---------------- - -Person who knows the high-level technical requirements for DCAE's -upcoming release cycle and dictates the development needs.  This person -is responsible for the service assurance flows.  This person expresses -the nodes and connections of a flow at a high level in a new graph or an -existing graph and assigns nodes to component developer/experts to be -implemented. - -This person must know: - -- What newly added flows should look like at a high level - -- What changes that are needed to existing flows - -- Target environments/sites/locations that need the flows at what SLA - -- Data requirements e.g. volume, rate, format, retention - -This person creates a top-level representation of the flow and assigns -the pieces to developers or experts for implementation. - - -Component developer/expert --------------------------- - -Person who is responsible for defining an assigned node's subgraph.  -This person can be: - -- A developer who might be onboarding a new component or a new version - of an existing component to fulfill the system engineer's - requirements - -- A domain expert who selects a suitable existing component, wires and - configures.  This expert knows the intricacies of a class of - components (e.g. Acumos machine learning). - -Developers ----------- - -They must know: - -- The target DCAE runtime and can develop a component to successfully - run on the runtime - -- The DCAE onboarding process for components including the development - testing procedure - -- Best practices of data flow management (data provenance?) - -- Lifecycle of DCAE components specifically impact of changes to - existing running instances globally - -- The resource requirement of the developed component - -Experts -------- - -They must know: - -- The target DCAE runtime - -- The technical capabilities of a set of components in order to best - select - -- The technical needs of the set of components in order to properly - configure and connect - -Designer --------- - - -Person who is responsible for connecting remote nodes to flows and -configuring all nodes in a flow in the context of the flow and in the -context of the greater graph.  An example of the former is connecting a -flow with a collector to a black boxed vMME.  An example of the latter -is assigning the threshold to a threshold-crossing-analytics component -when it is connected to a specific VES collector who is connected to a -specific vMME. - -This person knows: - -- The VNFs to monitor and the technical details to correctly connect - with them - -- Enough about the capability of a component and understands the - characteristics and requirements of a flow to properly - assign designer_editable configuration parameters - -This person has the ability to promote flows through the development -process (i.e. FTL to IST to ETE) and will coordinate with testers to -make sure the progression happens. - -Tester ------- - -Person who is responsible for testing a promoted new flow or newly -edited flow.  Once the designer has promoted a flow to a tester's -environment, the tester will have ready access to the deployment -artifacts necessary to apply the runtime changes that will reflect the -flow design and verify the resulting functionality matches to the system -engineer's expectations. - -Person knows and owns a DCAE runtime. - -Read access to the design tool would be useful for troubleshooting. - - -Operations ----------- - -Person who is responsible that DCAE both platform and service assurance -flows are all operational in production environments and locations.  -Once a flow has been fully certified, the required deployment artifact -is provided to operations and operations is responsible for applying the -runtime changes to reflect the flow design. - -Person knows and owns a DCAE runtime. - -Read access to the design tool would be useful for troubleshooting. - - - -Manager -------- - - -Person who are accountable to the business of the successful delivery of -a set of service assurance flows.  Read access to the design tool -specifically high level reports are useful to understand if goals are -being met and to better measure project success. - -Platform developer ------------------- - - -Person who is responsible for the development of the DCAE platform which -ranges from onboarding, design, and runtime.  In the scope of onboarding -and design, they are also responsible (unless there's an internal -operational team) for the operational concerns which means the tooling -built in this effort will need to be continually supported.  -Their **users** are all the above.  The design tool is intended to span -across multiple environments thus must run where all the required -parties have access. diff --git a/docs/sections/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json b/docs/sections/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json deleted file mode 100644 index 68667836..00000000 --- a/docs/sections/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json +++ /dev/null @@ -1,328 +0,0 @@ -{ - "owner": "mS-Developer", - "spec": { - "self": { - "version": "1.5.4", - "name": "dcae-ves-collector", - "description": "Collector for receiving VES events through restful interface", - "component_type": "docker" - }, - "streams": { - "subscribes": [], - "publishes": [ - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-fault" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-measurement" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-syslog" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-heartbeat" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-other" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-mobileflow" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-statechange" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-thresholdCrossingAlert" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-voicequality" - }, - { - "format": "VES_specification", - "version": "5.28.4", - "type": "message router", - "config_key": "ves-sipsignaling" - }, - { - "format": "VES_specification", - "version": "7.30.1", - "type": "message router", - "config_key": "ves-pnfRegistration" - }, - { - "format": "VES_specification", - "version": "7.30.1", - "type": "message router", - "config_key": "ves-notification" - }, - { - "format": "VES_specification", - "version": "7.30.1", - "type": "message router", - "config_key": "ves-perf3gpp" - } - ] - }, - "services": { - "calls": [], - "provides": [ - { - "route": "/eventListener/v1", - "verb": "POST", - "request": { - "format": "VES_specification", - "version": "4.27.2" - }, - "response": { - "format": "ves.coll.response", - "version": "1.0.0" - } - }, - { - "route": "/eventListener/v2", - "verb": "POST", - "request": { - "format": "VES_specification", - "version": "4.27.2" - }, - "response": { - "format": "ves.coll.response", - "version": "1.0.0" - } - }, - { - "route": "/eventListener/v3", - "verb": "POST", - "request": { - "format": "VES_specification", - "version": "4.27.2" - }, - "response": { - "format": "ves.coll.response", - "version": "1.0.0" - } - }, - { - "route": "/eventListener/v4", - "verb": "POST", - "request": { - "format": "VES_specification", - "version": "4.27.2" - }, - "response": { - "format": "ves.coll.response", - "version": "1.0.0" - } - }, - { - "route": "/eventListener/v5", - "verb": "POST", - "request": { - "format": "VES_specification", - "version": "5.28.4" - }, - "response": { - "format": "ves.coll.response", - "version": "1.0.0" - } - }, - { - "route": "/eventListener/v7", - "verb": "POST", - "request": { - "format": "VES_specification", - "version": "7.30.1" - }, - "response": { - "format": "ves.coll.response", - "version": "1.0.0" - } - } - ] - }, - "parameters": [ - { - "name": "collector.service.port", - "value": 8080, - "description": "standard http port collector will open for listening;", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "collector.service.secure.port", - "value": 8443, - "description": "secure http port collector will open for listening ", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": true - }, - { - "name": "collector.keystore.file.location", - "value": "/opt/app/dcae-certificate/cert.jks", - "description": "fs location of keystore file in vm", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "collector.keystore.passwordfile", - "value": "/opt/app/dcae-certificate/jks.pass", - "description": "location of keystore password file in vm", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "collector.truststore.file.location", - "value": "/opt/app/dcae-certificate/trust.jks", - "description": "fs location of truststore file in vm", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "collector.truststore.passwordfile", - "value": "/opt/app/dcae-certificate/trust.pass", - "description": "location of truststore password file in vm", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "collector.dmaap.streamid", - "value": "fault=ves-fault|syslog=ves-syslog|heartbeat=ves-heartbeat|measurementsForVfScaling=ves-measurement|measurement=ves-measurement|mobileFlow=ves-mobileflow|other=ves-other|stateChange=ves-statechange|thresholdCrossingAlert=ves-thresholdCrossingAlert|voiceQuality=ves-voicequality|sipSignaling=ves-sipsignaling|notification=ves-notification|pnfRegistration=ves-pnfRegistration|perf3gpp=ves-perf3gpp", - "description": "domain-to-streamid mapping used by VESCollector to distributes events based on domain. Both primary and secondary config_key are included for resilency (multiple streamid can be included commma separated). The streamids MUST match to topic config_keys. For single site without resiliency deployment - configkeys with -secondary suffix can be removed", - "sourced_at_deployment": true, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "auth.method", - "value": "noAuth", - "description": "Property to manage application mode, possible configurations: noAuth - default option - no security (http) , certOnly - auth by certificate (https), basicAuth - auth by basic auth username and password (https),certBasicAuth - auth by certificate and basic auth username / password (https),", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "header.authlist", - "value": "sample1,$2a$10$pgjaxDzSuc6XVFEeqvxQ5u90DKJnM/u7TJTcinAlFJVaavXMWf/Zi|userid1,$2a$10$61gNubgJJl9lh3nvQvY9X.x4e5ETWJJ7ao7ZhJEvmfJigov26Z6uq|userid2,$2a$10$G52y/3uhuhWAMy.bx9Se8uzWinmbJa.dlm1LW6bYPdPkkywLDPLiy", - "description": "List of id and base 64 encoded password.For each onboarding VNF - unique userid and password should be assigned and communicated to VNF owner. Password value should be base64 encoded in config here", - "policy_editable": false, - "sourced_at_deployment": true, - "designer_editable": true - }, - { - "name": "collector.schema.checkflag", - "value": 1, - "description": "Schema check validation flag. When enabled, collector will validate input VES events against VES Schema defined on collector.schema.file ", - "sourced_at_deployment": false, - "policy_editable": false, - "designer_editable": false - }, - { - "name": "collector.schema.file", - "value": "{\"v1\":\"./etc/CommonEventFormat_27.2.json\",\"v2\":\"./etc/CommonEventFormat_27.2.json\",\"v3\":\"./etc/CommonEventFormat_27.2.json\",\"v4\":\"./etc/CommonEventFormat_27.2.json\",\"v5\":\"./etc/CommonEventFormat_28.4.1.json\",\"v7\":\"./etc/CommonEventFormat_30.1.1.json\"}", - "description": "VES schema file name per version used for validation", - "designer_editable": true, - "sourced_at_deployment": false, - "policy_editable": false - }, - { - "name": "event.transform.flag", - "value": 1, - "description": "flag to enable tranformation rules defined under eventTransform.json; this is applicable when event tranformation rules preset should be activated for transforming elements, starting at 1", - "type": "string" - }, - "url": { - "description": "Hyperlink to a detailed error resource e.g., an HTML page for browser user agents", - "type": "string" - }, - "variables": { - "description": "List of zero or more strings that represent the contents of the variables used by the message text", - "type": "string" - } - }, - "required": [ - "messageId", - "text" - ] - }, - "sipSignalingFields": { - "description": "sip signaling fields", - "type": "object", - "properties": { - "additionalInformation": { - "description": "additional sip signaling fields if needed", - "type": "array", - "items": { - "$ref": "#/definitions/field" - } - }, - "compressedSip": { - "description": "the full SIP request/response including headers and bodies", - "type": "string" - }, - "correlator": { - "description": "this is the same for all events on this call", - "type": "string" - }, - "localIpAddress": { - "description": "IP address on VNF", - "type": "string" - }, - "localPort": { - "description": "port on VNF", - "type": "string" - }, - "remoteIpAddress": { - "description": "IP address of peer endpoint", - "type": "string" - }, - "remotePort": { - "description": "port of peer endpoint", - "type": "string" - }, - "sipSignalingFieldsVersion": { - "description": "version of the sipSignalingFields block", - "type": "number" - }, - "summarySip": { - "description": "the SIP Method or Response (‘INVITE’, ‘200 OK’, ‘BYE’, etc)", - "type": "string" - }, - "vendorVnfNameFields": { - "$ref": "#/definitions/vendorVnfNameFields" - } - }, - "required": [ - "correlator", - "localIpAddress", - "localPort", - "remoteIpAddress", - "remotePort", - "sipSignalingFieldsVersion", - "vendorVnfNameFields" - ] - }, - "stateChangeFields": { - "description": "stateChange fields", - "type": "object", - "properties": { - "additionalFields": { - "description": "additional stateChange fields if needed", - "type": "array", - "items": { - "$ref": "#/definitions/field" - } - }, - "newState": { - "description": "new state of the entity", - "type": "string", - "enum": [ - "inService", - "maintenance", - "outOfService" - ] - }, - "oldState": { - "description": "previous state of the entity", - "type": "string", - "enum": [ - "inService", - "maintenance", - "outOfService" - ] - }, - "stateChangeFieldsVersion": { - "description": "version of the stateChangeFields block", - "type": "number" - }, - "stateInterface": { - "description": "card or port name of the entity that changed state", - "type": "string" - } - }, - "required": [ - "newState", - "oldState", - "stateChangeFieldsVersion", - "stateInterface" - ] - }, - "suppressedNvPairs": { - "description": "List of specific NvPairsNames to suppress within a given Name-Value Field for event Throttling", - "type": "object", - "properties": { - "nvPairFieldName": { - "description": "Name of the field within which are the nvpair names to suppress", - "type": "string" - }, - "suppressedNvPairNames": { - "description": "Array of nvpair names to suppress within the nvpairFieldName", - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "nvPairFieldName", - "suppressedNvPairNames" - ] - }, - "syslogFields": { - "description": "sysLog fields", - "type": "object", - "properties": { - "additionalFields": { - "description": "additional syslog fields if needed provided as name=value delimited by a pipe ‘|’ symbol, for example: 'name1=value1|name2=value2|…'", - "type": "string" - }, - "eventSourceHost": { - "description": "hostname of the device", - "type": "string" - }, - "eventSourceType": { - "description": "type of event source; examples: other, router, switch, host, card, port, slotThreshold, portThreshold, virtualMachine, virtualNetworkFunction", - "type": "string" - }, - "syslogFacility": { - "description": "numeric code from 0 to 23 for facility--see table in documentation", - "type": "integer" - }, - "syslogFieldsVersion": { - "description": "version of the syslogFields block", - "type": "number" - }, - "syslogMsg": { - "description": "syslog message", - "type": "string" - }, - "syslogPri": { - "description": "0-192 combined severity and facility", - "type": "integer" - }, - "syslogProc": { - "description": "identifies the application that originated the message", - "type": "string" - }, - "syslogProcId": { - "description": "a change in the value of this field indicates a discontinuity in syslog reporting", - "type": "number" - }, - "syslogSData": { - "description": "syslog structured data consisting of a structured data Id followed by a set of key value pairs", - "type": "string" - }, - "syslogSdId": { - "description": "0-32 char in format name@number for example ourSDID@32473", - "type": "string" - }, - "syslogSev": { - "description": "numerical Code for severity derived from syslogPri as remaider of syslogPri / 8", - "type": "string", - "enum": [ - "Alert", - "Critical", - "Debug", - "Emergency", - "Error", - "Info", - "Notice", - "Warning" - ] - }, - "syslogTag": { - "description": "msgId indicating the type of message such as TCPOUT or TCPIN; NILVALUE should be used when no other value can be provided", - "type": "string" - }, - "syslogVer": { - "description": "IANA assigned version of the syslog protocol specification - typically 1", - "type": "number" - } - }, - "required": [ - "eventSourceType", - "syslogFieldsVersion", - "syslogMsg", - "syslogTag" - ] - }, - "thresholdCrossingAlertFields": { - "description": "fields specific to threshold crossing alert events", - "type": "object", - "properties": { - "additionalFields": { - "description": "additional threshold crossing alert fields if needed", - "type": "array", - "items": { - "$ref": "#/definitions/field" - } - }, - "additionalParameters": { - "description": "performance counters", - "type": "array", - "items": { - "$ref": "#/definitions/counter" - } - }, - "alertAction": { - "description": "Event action", - "type": "string", - "enum": [ - "CLEAR", - "CONT", - "SET" - ] - }, - "alertDescription": { - "description": "Unique short alert description such as IF-SHUB-ERRDROP", - "type": "string" - }, - "alertType": { - "description": "Event type", - "type": "string", - "enum": [ - "CARD-ANOMALY", - "ELEMENT-ANOMALY", - "INTERFACE-ANOMALY", - "SERVICE-ANOMALY" - ] - }, - "alertValue": { - "description": "Calculated API value (if applicable)", - "type": "string" - }, - "associatedAlertIdList": { - "description": "List of eventIds associated with the event being reported", - "type": "array", - "items": { - "type": "string" - } - }, - "collectionTimestamp": { - "description": "Time when the performance collector picked up the data; with RFC 2822 compliant format: Sat, 13 Mar 2010 11:29:05 -0800", - "type": "string" - }, - "dataCollector": { - "description": "Specific performance collector instance used", - "type": "string" - }, - "elementType": { - "description": "type of network element - internal ATT field", - "type": "string" - }, - "eventSeverity": { - "description": "event severity or priority", - "type": "string", - "enum": [ - "CRITICAL", - "MAJOR", - "MINOR", - "WARNING", - "NORMAL" - ] - }, - "eventStartTimestamp": { - "description": "Time closest to when the measurement was made; with RFC 2822 compliant format: Sat, 13 Mar 2010 11:29:05 -0800", - "type": "string" - }, - "interfaceName": { - "description": "Physical or logical port or card (if applicable)", - "type": "string" - }, - "networkService": { - "description": "network name - internal ATT field", - "type": "string" - }, - "possibleRootCause": { - "description": "Reserved for future use", - "type": "string" - }, - "thresholdCrossingFieldsVersion": { - "description": "version of the thresholdCrossingAlertFields block", - "type": "number" - } - }, - "required": [ - "additionalParameters", - "alertAction", - "alertDescription", - "alertType", - "collectionTimestamp", - "eventSeverity", - "eventStartTimestamp", - "thresholdCrossingFieldsVersion" - ] - }, - "vendorVnfNameFields": { - "description": "provides vendor, vnf and vfModule identifying information", - "type": "object", - "properties": { - "vendorName": { - "description": "VNF vendor name", - "type": "string" - }, - "vfModuleName": { - "description": "ASDC vfModuleName for the vfModule generating the event", - "type": "string" - }, - "vnfName": { - "description": "ASDC modelName for the VNF generating the event", - "type": "string" - } - }, - "required": [ - "vendorName" - ] - }, - "vNicPerformance": { - "description": "describes the performance and errors of an identified virtual network interface card", - "type": "object", - "properties": { - "receivedBroadcastPacketsAccumulated": { - "description": "Cumulative count of broadcast packets received as read at the end of the measurement interval", - "type": "number" - }, - "receivedBroadcastPacketsDelta": { - "description": "Count of broadcast packets received within the measurement interval", - "type": "number" - }, - "receivedDiscardedPacketsAccumulated": { - "description": "Cumulative count of discarded packets received as read at the end of the measurement interval", - "type": "number" - }, - "receivedDiscardedPacketsDelta": { - "description": "Count of discarded packets received within the measurement interval", - "type": "number" - }, - "receivedErrorPacketsAccumulated": { - "description": "Cumulative count of error packets received as read at the end of the measurement interval", - "type": "number" - }, - "receivedErrorPacketsDelta": { - "description": "Count of error packets received within the measurement interval", - "type": "number" - }, - "receivedMulticastPacketsAccumulated": { - "description": "Cumulative count of multicast packets received as read at the end of the measurement interval", - "type": "number" - }, - "receivedMulticastPacketsDelta": { - "description": "Count of multicast packets received within the measurement interval", - "type": "number" - }, - "receivedOctetsAccumulated": { - "description": "Cumulative count of octets received as read at the end of the measurement interval", - "type": "number" - }, - "receivedOctetsDelta": { - "description": "Count of octets received within the measurement interval", - "type": "number" - }, - "receivedTotalPacketsAccumulated": { - "description": "Cumulative count of all packets received as read at the end of the measurement interval", - "type": "number" - }, - "receivedTotalPacketsDelta": { - "description": "Count of all packets received within the measurement interval", - "type": "number" - }, - "receivedUnicastPacketsAccumulated": { - "description": "Cumulative count of unicast packets received as read at the end of the measurement interval", - "type": "number" - }, - "receivedUnicastPacketsDelta": { - "description": "Count of unicast packets received within the measurement interval", - "type": "number" - }, - "transmittedBroadcastPacketsAccumulated": { - "description": "Cumulative count of broadcast packets transmitted as read at the end of the measurement interval", - "type": "number" - }, - "transmittedBroadcastPacketsDelta": { - "description": "Count of broadcast packets transmitted within the measurement interval", - "type": "number" - }, - "transmittedDiscardedPacketsAccumulated": { - "description": "Cumulative count of discarded packets transmitted as read at the end of the measurement interval", - "type": "number" - }, - "transmittedDiscardedPacketsDelta": { - "description": "Count of discarded packets transmitted within the measurement interval", - "type": "number" - }, - "transmittedErrorPacketsAccumulated": { - "description": "Cumulative count of error packets transmitted as read at the end of the measurement interval", - "type": "number" - }, - "transmittedErrorPacketsDelta": { - "description": "Count of error packets transmitted within the measurement interval", - "type": "number" - }, - "transmittedMulticastPacketsAccumulated": { - "description": "Cumulative count of multicast packets transmitted as read at the end of the measurement interval", - "type": "number" - }, - "transmittedMulticastPacketsDelta": { - "description": "Count of multicast packets transmitted within the measurement interval", - "type": "number" - }, - "transmittedOctetsAccumulated": { - "description": "Cumulative count of octets transmitted as read at the end of the measurement interval", - "type": "number" - }, - "transmittedOctetsDelta": { - "description": "Count of octets transmitted within the measurement interval", - "type": "number" - }, - "transmittedTotalPacketsAccumulated": { - "description": "Cumulative count of all packets transmitted as read at the end of the measurement interval", - "type": "number" - }, - "transmittedTotalPacketsDelta": { - "description": "Count of all packets transmitted within the measurement interval", - "type": "number" - }, - "transmittedUnicastPacketsAccumulated": { - "description": "Cumulative count of unicast packets transmitted as read at the end of the measurement interval", - "type": "number" - }, - "transmittedUnicastPacketsDelta": { - "description": "Count of unicast packets transmitted within the measurement interval", - "type": "number" - }, - "valuesAreSuspect": { - "description": "Indicates whether vNicPerformance values are likely inaccurate due to counter overflow or other condtions", - "type": "string", - "enum": [ - "true", - "false" - ] - }, - "vNicIdentifier": { - "description": "vNic identification", - "type": "string" - } - }, - "required": [ - "valuesAreSuspect", - "vNicIdentifier" - ] - }, - "voiceQualityFields": { - "description": "provides statistics related to customer facing voice products", - "type": "object", - "properties": { - "additionalInformation": { - "description": "additional voice quality fields if needed", - "type": "array", - "items": { - "$ref": "#/definitions/field" - } - }, - "calleeSideCodec": { - "description": "callee codec for the call", - "type": "string" - }, - "callerSideCodec": { - "description": "caller codec for the call", - "type": "string" - }, - "correlator": { - "description": "this is the same for all events on this call", - "type": "string" - }, - "endOfCallVqmSummaries": { - "$ref": "#/definitions/endOfCallVqmSummaries" - }, - "phoneNumber": { - "description": "phone number associated with the correlator", - "type": "string" - }, - "midCallRtcp": { - "description": "Base64 encoding of the binary RTCP data excluding Eth/IP/UDP headers", - "type": "string" - }, - "vendorVnfNameFields": { - "$ref": "#/definitions/vendorVnfNameFields" - }, - "voiceQualityFieldsVersion": { - "description": "version of the voiceQualityFields block", - "type": "number" - } - }, - "required": [ - "calleeSideCodec", - "callerSideCodec", - "correlator", - "midCallRtcp", - "vendorVnfNameFields", - "voiceQualityFieldsVersion" - ] - } - } - } - } -} diff --git a/docs/sections/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json b/docs/sections/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json deleted file mode 100644 index c3faf800..00000000 --- a/docs/sections/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "spring.data.mongodb.uri": "mongodb://dcae-mongohost/dcae-tcagen2", - "tca.aai.enable_enrichment": true, - "docker-tcagen2_memory_request": "512Mi", - "replicas": 1, - "docker-tcagen2_cpu_limit": "250m", - "tca.aai.username": "DCAE", - "image": "nexus3.onap.org:10001/onap/org.onap.dcaegen2.analytics.tca-gen2.dcae-analytics-tca-web:1.0.1", - "service_component_name_override": "dcaegen2-tcagen2", - "use_tls": true, - "always_pull_image": true, - "tca.aai.url": "http://aai.onap.svc.cluster.local", - "location_id": "", - "envs": {}, - "tca_handle_in_subscribe_url": "http://message-router:3904/events/unauthenticated.VES_MEASUREMENT_OUTPUT/", - "tca_handle_out_publish_url": "http://message-router:3904/events/unauthenticated.TCAGEN2_OUTPUT/", - "external_port_0": 0, - "docker-tcagen2_memory_limit": "512Mi", - "docker-tcagen2_cpu_request": "250m", - "tca.aai.password": "DCAE" -} \ No newline at end of file diff --git a/docs/sections/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json b/docs/sections/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json deleted file mode 100644 index d86ff0bc..00000000 --- a/docs/sections/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "always_pull_image": true, - "collector.dmaap.streamid": "fault=ves-fault|syslog=ves-syslog|heartbeat=ves-heartbeat|measurementsForVfScaling=ves-measurement|measurement=ves-measurement|mobileFlow=ves-mobileflow|other=ves-other|stateChange=ves-statechange|thresholdCrossingAlert=ves-thresholdCrossingAlert|voiceQuality=ves-voicequality|sipSignaling=ves-sipsignaling|notification=ves-notification|pnfRegistration=ves-pnfRegistration|perf3gpp=ves-perf3gpp", - "dcae-ves-collector_cpu_limit": "250m", - "dcae-ves-collector_cpu_request": "250m", - "dcae-ves-collector_memory_limit": "512Mi", - "dcae-ves-collector_memory_request": "512Mi", - "envs": {}, - "external_port_0": 30235, - "external_port_1": 0, - "header.authlist": "sample1,$2a$10$pgjaxDzSuc6XVFEeqvxQ5u90DKJnM/u7TJTcinAlFJVaavXMWf/Zi|userid1,$2a$10$61gNubgJJl9lh3nvQvY9X.x4e5ETWJJ7ao7ZhJEvmfJigov26Z6uq|userid2,$2a$10$G52y/3uhuhWAMy.bx9Se8uzWinmbJa.dlm1LW6bYPdPkkywLDPLiy", - "image": "nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.ves.vescollector:1.5.4", - "location_id": "", - "replicas": 1, - "service_component_name_override": "dcae-ves-collector-http", - "use_tls": true, - "ves_fault_publish_url": "http://message-router:3904/events/unauthenticated.SEC_FAULT_OUTPUT/", - "ves_heartbeat_publish_url": "http://message-router:3904/events/unauthenticated.SEC_HEARTBEAT_OUTPUT/", - "ves_measurement_publish_url": "http://message-router:3904/events/unauthenticated.VES_MEASUREMENT_OUTPUT/", - "ves_mobileflow_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", - "ves_notification_publish_url": "http://message-router:3904/events/unauthenticated.VES_NOTIFICATION_OUTPUT/", - "ves_other_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", - "ves_perf3gpp_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", - "ves_pnfRegistration_publish_url": "http://message-router:3904/events/unauthenticated.VES_PNFREG_OUTPUT/", - "ves_sipsignaling_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", - "ves_statechange_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", - "ves_syslog_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", - "ves_thresholdCrossingAlert_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", - "ves_voicequality_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/" -} \ No newline at end of file diff --git a/docs/sections/DCAE-MOD/mod-index.rst b/docs/sections/DCAE-MOD/mod-index.rst deleted file mode 100644 index 8e72fb36..00000000 --- a/docs/sections/DCAE-MOD/mod-index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -DCAE Model Onboarding and Design (DCAE MOD) -=========================================== - -.. toctree:: - :maxdepth: 1 - - ./DCAE-MOD-Intro.rst - ./DCAE-MOD-Architecture.rst - ./Roles.rst - ./DCAE-MOD-User-Guide.rst - ./Glossary.rst \ No newline at end of file diff --git a/docs/sections/apis/deployment-handler.rst b/docs/sections/apis/deployment-handler.rst index ab4c0c5c..a35bc8bd 100644 --- a/docs/sections/apis/deployment-handler.rst +++ b/docs/sections/apis/deployment-handler.rst @@ -1,7 +1,7 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. -deployment-handler +Deployment-Handler ================== .. toctree:: diff --git a/docs/sections/components/architecture/pieces.rst b/docs/sections/components/architecture/pieces.rst deleted file mode 100755 index 9e57c02b..00000000 --- a/docs/sections/components/architecture/pieces.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -Platform technologies -===================== - -- `Cloudify `__ -- `Consul `__ -- `Docker `__ -- `CDAP `__ -- `Registrator `__ diff --git a/docs/sections/components/architecture/service-discovery.rst b/docs/sections/components/architecture/service-discovery.rst deleted file mode 100755 index 315f4520..00000000 --- a/docs/sections/components/architecture/service-discovery.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -Service Discovery -================= - -Service discovery is an architecture pattern used for components -(micro-services) to locate each other. The DCAE platform uses -`server-side -discovery `__ -and is using `Consul `__ as the service registry -solution. - -Service Registration --------------------- - -All components are required to register with Consul in order to be -discovered. There are two methods of registration: self and 3rd party. -The DCAE platform uses 3rd party registration which means components -don’t actually make the registration calls but defers that -responsibility to a platform service. - -Implementation for Docker -~~~~~~~~~~~~~~~~~~~~~~~~~ - -`Registrator `__ is an open -source application that is responsible for registering all components -that run as Docker containers. Registrator watches the local Docker -engine’s activity log and will register and unregister a Docker -container when the container is started and stopped. - -Implementation for CDAP -~~~~~~~~~~~~~~~~~~~~~~~ - -The CDAP broker is a REST web service that is responsible for -registering all components that run as CDAP applications. diff --git a/docs/sections/components/blueprint_generator.rst b/docs/sections/components/blueprint_generator.rst deleted file mode 100644 index 3a014ea3..00000000 --- a/docs/sections/components/blueprint_generator.rst +++ /dev/null @@ -1,77 +0,0 @@ - -Blueprint Generator (DCAE) -========================== - -What is the Blueprint Generator? -++++++++++++++++++++++++++++++++ -The blueprint generator is a java rewrite of the tosca lab python tool. The point of this tool is to be able to take a component spec for a given micro-service and translate that component spec into a blueprint yaml file that can be used during deployment. - - -Steps to run the blueprint generator: -+++++++++++++++++++++++++++++++++++++ - -1. Download the jar file from Nexus by clicking `here `_ or running - ``https://nexus.onap.org/service/local/repositories/releases/content/org/onap/dcaegen2/platform/mod/blueprint-generator/1.3.1/blueprint-generator-1.3.1-executable.jar`` - -2. To execute the application, run the following command: - ``java -jar blueprint-generator-1.3.1-executable.jar blueprint`` - -3. This execution will provide the help, as you have not provided the required flags. - -4. When ready you can run the program again except with the required flags. - -5. OPTIONS: - -p: The path to where the final blueprint yaml file will be created (required) - - -i: The path to the JSON spec file (required) - - -n: Name of the blueprint (optional) - - -t: the path to the import yaml file (optional) - - -d: If this flag is present the bp generator will be created with dmaap plugin (optional) - - -o: This flag will create a service component override for your deployment equal to the value you put (optional) - -6. An example running this program would look like this: - ``java -jar blueprint-generator-1.3.1-executable.jar -p blueprint_output -i ComponentSpecs/TestComponentSpec.json -n TestAppBlueprint`` - - -Extra information ------------------ - -1. The component spec must be of the same format as stated in the onap `readthedocs `_ page - -2. If the tag says required then the program will not run without those tags being there - -3. If the tag says optional then it is not necessary to run the program with those tags - -4. If you do not add a -n tag the blueprint name will default to what it is in the component spec - -5. If the directory you specified in the -p tag does not already exist the directory will be created for you - -6. The -t flag will override the default imports set for the blueprints. To see an example of how the import yaml file should be structured see the testImports.yaml file under the folder TestCases - - -How to create policy models: -+++++++++++++++++++++++++++++++++++++ - -1. Policy model creation can be done with the same jar as downloaded for the blueprint generation. - -2. Run the same command as the blueprint generator except replace the ``blueprint`` positional with ``policy`` - -3. Example command: - ``java -jar blueprint-generator-1.3.1-executable.jar policy`` - -4. Options: - - -i: The path to the JSON spec file (required) - - -p: The Output path for all of the models (required) - -Extra information ------------------ - -1. Not all component specs will be able to create policy models - -2. Multiple policy model files may be create from a single component spec \ No newline at end of file diff --git a/docs/sections/components/component-development.rst b/docs/sections/components/component-development.rst deleted file mode 100644 index 196e7a09..00000000 --- a/docs/sections/components/component-development.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -Onboarding Pre-requisite (Service Component) -============================================ - -.. toctree:: - :maxdepth: 1 - - ./intro.rst - ../DCAE-MOD/mod-index.rst - ./architecture/pieces.rst - ./architecture/service-discovery.rst - ./architecture/blueprint_generator.rst - ./requirements-guidelines.rst - ./component-specification/component-specification.rst - ./component-json-schema.rst - ./data-formats.rst - ./dcae-cli/dcae-cli-tool.rst - ./glossary.rst - - diff --git a/docs/sections/components/component-json-schema.rst b/docs/sections/components/component-json-schema.rst deleted file mode 100644 index 9ad54565..00000000 --- a/docs/sections/components/component-json-schema.rst +++ /dev/null @@ -1,753 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _dcae-component-schema: - -DCAE Component JSON Schema -========================== - -:: - - { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Component specification schema", - "type": "object", - "properties": { - "self": { - "type": "object", - "properties": { - "version": { - "$ref": "#/definitions/version" - }, - "description": { - "type": "string" - }, - "component_type": { - "type": "string", - "enum": [ - "docker", - "cdap" - ] - }, - "name": { - "$ref": "#/definitions/name" - } - }, - "required": [ - "version", - "name", - "description", - "component_type" - ] - }, - "streams": { - "type": "object", - "properties": { - "publishes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/publisher_http" }, - { "$ref": "#/definitions/publisher_message_router" }, - { "$ref": "#/definitions/publisher_data_router" } - ] - } - }, - "subscribes": { - "type": "array", - "uniqueItems": true, - "items": { - "oneOf": [ - { "$ref": "#/definitions/subscriber_http" }, - { "$ref": "#/definitions/subscriber_message_router" }, - { "$ref": "#/definitions/subscriber_data_router" } - ] - } - } - }, - "required": [ - "publishes", - "subscribes" - ] - }, - "services": { - "type": "object", - "properties": { - "calls": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/caller" - } - }, - "provides": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/provider" - } - } - }, - "required": [ - "calls", - "provides" - ] - }, - "parameters" : { - "anyOf" : [ - {"$ref": "#/definitions/docker-parameters"}, - {"$ref": "#/definitions/cdap-parameters"} - ] - }, - "auxilary": { - "oneOf" : [ - {"$ref": "#/definitions/auxilary_cdap"}, - {"$ref": "#/definitions/auxilary_docker"} - ] - }, - "artifacts": { - "type": "array", - "description": "List of component artifacts", - "items": { - "$ref": "#/definitions/artifact" - } - } - }, - "required": [ - "self", - "streams", - "services", - "parameters", - "auxilary", - "artifacts" - ], - "additionalProperties": false, - "definitions": { - "cdap-parameters": { - "description" : "There are three seperate ways to pass parameters to CDAP: app config, app preferences, program preferences. These are all treated as optional.", - "type": "object", - "properties" : { - "program_preferences": { - "description" : "A list of {program_id, program_type, program_preference} objects where program_preference is an object passed into program_id of type program_type", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/program_preference" - } - }, - "app_preferences" : { - "description" : "Parameters Passed down to the CDAP preference API", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "app_config" : { - "description" : "Parameters Passed down to the CDAP App Config", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - } - }, - "program_preference": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/program_type" - }, - "program_id": { - "type": "string" - }, - "program_pref":{ - "description" : "Parameters that the CDAP developer wants pushed to this program's preferences API. Optional", - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - } - }, - "required": ["program_type", "program_id", "program_pref"] - }, - "program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "docker-parameters": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/parameter" - } - }, - "parameter": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "used for complex data type in the future. 'type' must be map or array for entry_schema to kick_in. ", - "type": "string" - }, - "designer_editable": { - "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", - "type": "boolean" - }, - "policy_editable": { - "description": "A required property that declares a parameter as editable by DevOps in Policy UI (true) or not (false).", - "type": "boolean" - }, - "sourced_at_deployment": { - "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", - "type": "boolean" - }, - "policy_schema" :{ - "type": "array", - "uniqueItems": true, - "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "value", - "description", - "designer_editable", - "policy_editable", - "sourced_at_deployment" - ], - "additionalProperties": false, - "dependencies": { "policy_schema": ["policy_editable"]} - }, - "policy_schema_parameter": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "description": "Default value for the parameter" - }, - "description": { - "description": "Description for the parameter.", - "type": "string" - }, - "type": { - "description": "The required data type for the parameter.", - "type": "string", - "enum": [ "string", "number", "boolean", "datetime", "list", "map" ] - }, - "required": { - "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", - "type": "boolean", - "default": true - }, - "constraints": { - "description": "The optional list of sequenced constraint clauses for the parameter.", - "type": "array", - "items": { - "$ref": "#/definitions/parameter-constraints" - } - }, - "entry_schema": { - "description": "The optional key that is used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is either list or map. If the type is list and the entry type is a simple type (string, number, boolean, datetime), follow with a simple string to describe the entry type. If the type is list and the entry type is a map, follow with an array to describe the keys for the entry map. If the type is list and the entry type is also list, this is not currently supported here. If the type is map, then follow with an array to describe the keys for this map. ", - "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/policy_schema_parameter"} - } - }, - "required": [ - "name", - "type" - ], - "additionalProperties": false - }, - "parameter-constraints": { - "type": "object", - "additionalProperties": false, - "properties": { - "equal": { - "description": "Constrains a property or parameter to a value equal to ('=') the value declared." - }, - "greater_than": { - "description": "Constrains a property or parameter to a value greater than ('>') the value declared.", - "type": "number" - }, - "greater_or_equal": { - "description": "Constrains a property or parameter to a value greater than or equal to ('>=') the value declared.", - "type": "number" - }, - "less_than": { - "description": "Constrains a property or parameter to a value less than '<') the value declared.", - "type": "number" - }, - "less_or_equal": { - "description": "Constrains a property or parameter to a value less than or equal to ('<=') the value declared.", - "type": "number" - }, - "valid_values": { - "description": "Constrains a property or parameter to a value that is in the list of declared values.", - "type": "array" - }, - "length": { - "description": "Constrains the property or parameter to a value of a given length.", - "type": "number" - }, - "min_length": { - "description": "Constrains the property or parameter to a value to a minimum length.", - "type": "number" - }, - "max_length": { - "description": "Constrains the property or parameter to a value to a maximum length.", - "type": "number" - } - } - }, - "stream_message_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "message router", "message_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "publisher_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "publisher_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "config_key": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - } - }, - "required": [ - "format", - "version", - "config_key", - "type" - ] - }, - "subscriber_http": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "http", - "https" - ] - } - }, - "required": [ - "format", - "version", - "route", - "type" - ] - }, - "subscriber_message_router": { - "$ref": "#/definitions/stream_message_router" - }, - "subscriber_data_router": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - }, - "route": { - "type": "string" - }, - "type": { - "description": "Type of stream to be used", - "type": "string", - "enum": [ - "data router", "data_router" - ] - }, - "config_key": { - "description": "Data router subscribers require config info to setup their endpoints to handle requests. For example, needs username and password", - "type": "string" - } - }, - "required": [ - "format", - "version", - "route", - "type", - "config_key" - ] - }, - "provider" : { - "oneOf" : [ - {"$ref": "#/definitions/docker-provider"}, - {"$ref": "#/definitions/cdap-provider"} - ] - }, - "cdap-provider" : { - "type": "object", - "properties" : { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "service_name" : { - "type" : "string" - }, - "service_endpoint" : { - "type" : "string" - }, - "verb" : { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } - }, - "required" : [ - "request", - "response", - "service_name", - "service_endpoint", - "verb" - ] - }, - "docker-provider": { - "type": "object", - "properties": { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "route": { - "type": "string" - }, - "verb": { - "type": "string", - "enum": ["GET", "PUT", "POST", "DELETE"] - } - }, - "required": [ - "request", - "response", - "route" - ] - }, - "caller": { - "type": "object", - "properties": { - "request": { - "$ref": "#/definitions/formatPair" - }, - "response": { - "$ref": "#/definitions/formatPair" - }, - "config_key": { - "type": "string" - } - }, - "required": [ - "request", - "response", - "config_key" - ] - }, - "formatPair": { - "type": "object", - "properties": { - "format": { - "$ref": "#/definitions/name" - }, - "version": { - "$ref": "#/definitions/version" - } - } - }, - "name": { - "type": "string" - }, - "version": { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "artifact": { - "type": "object", - "description": "Component artifact object", - "properties": { - "uri": { - "type": "string", - "description": "Uri to artifact" - }, - "type": { - "type": "string", - "enum": ["jar", "docker image"] - } - }, - "required": ["uri", "type"] - }, - - "auxilary_cdap": { - "title": "cdap component specification schema", - "type": "object", - "properties": { - "streamname": { - "type": "string" - }, - "artifact_name" : { - "type": "string" - }, - "artifact_version" : { - "type": "string", - "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" - }, - "namespace":{ - "type": "string", - "description" : "optional" - }, - "programs": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/definitions/cdap_program" - } - } - }, - "required": [ - "streamname", - "programs", - "artifact_name", - "artifact_version" - ] - }, - "cdap_program_type": { - "type": "string", - "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] - }, - "cdap_program": { - "type": "object", - "properties": { - "program_type": { - "$ref": "#/definitions/cdap_program_type" - }, - "program_id": { - "type": "string" - } - }, - "required": ["program_type", "program_id"] - }, - - "auxilary_docker": { - "title": "Docker component specification schema", - "type": "object", - "properties": { - "healthcheck": { - "description": "Define the health check that Consul should perfom for this component", - "type": "object", - "oneOf": [ - { "$ref": "#/definitions/docker_healthcheck_http" }, - { "$ref": "#/definitions/docker_healthcheck_script" } - ] - }, - "ports": { - "description": "Port mapping to be used for Docker containers. Each entry is of the format :.", - "type": "array", - "items": { - "type": "string" - } - }, - "reconfigs": { - "properties": { - "dti": { - "description": "Script command that will be executed for reconfiguration", - "type": "string" - } - } - }, - "volumes": { - "description": "Volume mapping to be used for Docker containers. Each entry is of the format below", - "type": "array", - "items": { - "type": "object", - "properties": { - "host":{ - "type":"object", - "path": {"type": "string"} - }, - "container":{ - "type":"object", - "bind": { "type": "string"}, - "mode": { "type": "string"} - } - } - } - } - }, - "required": [ - "healthcheck" - ], - "additionalProperties": false - }, - "docker_healthcheck_http": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "http", - "https" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "endpoint": { - "description": "Relative endpoint used by Consul to check health by making periodic HTTP GET calls", - "type": "string" - } - }, - "required": [ - "type", - "endpoint" - ] - }, - "docker_healthcheck_script": { - "properties": { - "type": { - "description": "Consul health check type", - "type": "string", - "enum": [ - "script", - "docker" - ] - }, - "interval": { - "description": "Interval duration in seconds i.e. 10s", - "default": "15s", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds i.e. 10s", - "default": "1s", - "type": "string" - }, - "script": { - "description": "Script command that will be executed by Consul to check health", - "type": "string" - } - }, - "required": [ - "type", - "script" - ] - } - } - } diff --git a/docs/sections/components/component-specification/cdap-specification.rst b/docs/sections/components/component-specification/cdap-specification.rst deleted file mode 100755 index 2b26442a..00000000 --- a/docs/sections/components/component-specification/cdap-specification.rst +++ /dev/null @@ -1,208 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _cdap-specification: - -Component specification (CDAP) -============================== - -The CDAP component specification contains the following groups of -information. Many of these are common to both CDAP and Docker components -and are therefore described in the common specification. - -- :any:`Metadata ` -- :any:`Interfaces ` including the - associated :any:`Data Formats ` -- :any:`Parameters ` - for specifying parameters in your - AppConfig, AppPreferences, and ProgramPreferences to the Designer and - Policy. This of course is CDAP-specific and is described below. -- :any:`Auxiliary Details ` -- :any:`List of artifacts ` - -Current Limitations and TODOs ------------------------------ - -- The integration of DMD is likely to significantly change the - :any:`Interfaces ` section in this specification. - -.. _parameters: - -Parameters ----------- - -There is a ``parameters`` section in your component specification. This -section contains three optional keys: `app_config <#appconfig>`__, -`app_preferences <#apppreferences>`__, and -`program_preferences <#programpreferences>`__: - -:: - - "parameters" : { - "app_config" : [ ...], - "app_preferences" : [ ...], - "program_preferences" : [...] - // any additional keys are ignored - } - -- Each section details the parameters that are a part of each of these - CDAP constructs (see below). -- All such parameters will be exposed to the designer and to policy for - override. -- These parameters should have default values specified by the - component developer where necessary, i.e., parameters that *must* - come from the designer/policy should not have defaults. -- All of these keys are optional because not every CDAP application - uses preferences and not every application uses the AppConfig. - However, you should specify at least one, or else your application - will have no parameters exposed to policy or to the DCAE designer, - which means it would be non-configurable. -- Despite the AppConfig being optional to *specify* in the case that - you have no parameters in your AppConfig, it is *required for - processing* in your application. That is because the DCAE platform - will place important information into your AppConfig as discussed - below. - -Parameter -~~~~~~~~~ - -The following CDAP specific definitions use ``param1`` to refer to the -common parameter layout in -:any:`Parameter ` - -AppConfig -~~~~~~~~~ - -The ``app_config`` key refers to the `CDAP AppConfig `_. -It is expected to be a JSON: - -:: - - "app_config" : [ // list of JSON - param1, // common parameter layout - ... - ] - -Unfortunately, at the time of writing, the AppConfig is a Java map of -``string:string``, which means you cannot have more complex structures -(than string) as any value in your AppConfig. However, there is a way to -bypass this constraint: you can pass a JSON by encoding the JSON as a -string. E.g., the ``json.dumps()`` and it’s converse ``loads`` methods -in Python: - -:: - - >>> import json - >>> json.dumps({"foo" : "bar"}) # This is a real JSON - '{"foo": "bar"}' # It is now a string: pass this in as your parameter value - >>> json.loads('{"foo": "bar"}') # Do the equivelent of this in your application - {u'foo': u'bar'} # ...and you'll get back a real JSON - >>> - -The final AppConfig (after the designer and policy override parameter -values) is passed into CDAP’s AppConfig API when starting the -application. - - -AppPreferences -~~~~~~~~~~~~~~ - -In addition to the CDAP AppConfig, the platform supports `Application Preferences `_. -The format of the ``app_preferences`` value is the same as the above: - -:: - - "app_preferences" : [ // list of JSON - param1, // common parameter layout - ... - ] - -The final Application Preferences JSON (after the designer and policy -override parameter values) is passed into CDAP’s Preferences API when -starting your application. - - -ProgramPreferences -~~~~~~~~~~~~~~~~~~ - -Preferences can also be specified `per program `_ -in CDAP. This key’s value is a list of JSON with the following format: - -:: - - "program_preferences" : [ // note: this is a list of JSON - { - "program_id" : "program name 1", // the name of this CDAP program - "program_type" : "e.g., flows", // "must be one of flows, mapreduce, schedules, spark, workflows, workers, or services", - "program_pref" : [ // list of JSON - param1, // common parameter layout - ... - ] - }, - // repeat for each program that receives a program_preferences JSON - ] - -Each ``program_pref`` JSON is passed into the CDAP API as the preference -for ``program_id``. - -NOTE: for CDAP, this section is very likely to change when DMD is -available. The *future* vision is -that you would publish your data as a series of files on HDFS, and DMD -will pick them up and send them to the appropriate DMaaP feeds or -directly when needed. - -.. _auxiliary-details: - -Auxiliary Details ------------------ - -*auxiliary* contains details about CDAP specific parameters. - -+----------------------+----------------------+----------------------+ -| Property Name | Type | Description | -+======================+======================+======================+ -| streamname | string | *Required*. | -+----------------------+----------------------+----------------------+ -| artifact_name | string | | -+----------------------+----------------------+----------------------+ -| artifact_version | string | the version of your | -| | | CDAP JAR artifact | -+----------------------+----------------------+----------------------+ -| namespace | string | the CDAP namespace | -| | | to deploy into, | -| | | default is ‘default’ | -+----------------------+----------------------+----------------------+ -| programs | array | contains each CDAP | -| | | entity represented | -| | | in the artifact | -+----------------------+----------------------+----------------------+ -| program_type | string | CDAP entity (eg | -| | | “flows”) | -+----------------------+----------------------+----------------------+ -| program_id | string | name of CDAP entity | -| | | (eg “WhoFlow”) | -+----------------------+----------------------+----------------------+ - -Example: - -.. code:: json - - "auxiliary": { - "streamname" : "who", - "artifact_name" : "HelloWorld", - "artifact_version" : "3.4.3", - "namespace" : "hw", - "programs" : [ - {"program_type" : "flows", "program_id" : "WhoFlow"}, - {"program_type" : "services", "program_id" : "Greeting"}, - ... - ], - } - -The ``programs`` key is identical to the ``program_preferences`` key -discussed `above <#programpreferences>`__ except: - -- each JSON in the list does not contain ``program_pref`` -- this is required! You must include all of your programs in this, as - it is used to start each program as well as for DCAE to perform - periodic healthchecks on your application. Don’t forget about your - services; they are programs too. diff --git a/docs/sections/components/component-specification/common-specification.rst b/docs/sections/components/component-specification/common-specification.rst deleted file mode 100755 index ae846bd0..00000000 --- a/docs/sections/components/component-specification/common-specification.rst +++ /dev/null @@ -1,1442 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _common-specification: - -Common Elements of the Component Specification -============================================== - -This page describes the component specification (JSON) sections that are -common to both Docker and CDAP components. Differences for each are -pointed out below. Elements that are very different, and described in -the CDAP or Docker specific pages. - -.. _metaschema: - -Meta Schema Definition ----------------------- - -The component specification is represented (and validated) against this -`Component Spec json -schema `__ -and described below: - -The “Meta Schema” implementation defines how component specification -JSON schemas can be written to define user input. It is itself a JSON -schema (thus it is a “meta schema”). It requires the name of the -component entry, component type (either ‘cdap’ or ‘docker’) and a -description under “self” object. The meta schema version must be -specified as the value of the “version” key. Then the input schema -itself is described. - -There are four types of schema descriptions objects - jsonschema for -inline standard JSON Schema definitions of JSON inputs, delimitedschema -for delimited data input using a JSON description defined by AT&T, -unstructured for unstructured text, and reference that allows a pointer -to another artifact for a schema. The reference allows for XML and Protocol Buffer schema, -but can be used as a pointer to JSON, Delimited Format, and Unstructured -schemas as well. - -.. _metadata: - -Component Metadata ------------------- - -Metadata refers to the properties found under the ``self`` JSON. This -group of properties is used to uniquely identify this component -specification and identify the component that this specification is used -to capture. - -Example: - -:: - - "self": { - "version": "1.0.0", - "name": "yourapp.component.kpi_anomaly", - "description": "Classifies VNF KPI data as anomalous", - "component_type": "docker" - }, - -``self`` Schema: - -+-------------+--------+----------------+ -| Property | Type | Description | -| Name | | | -+=============+========+================+ -| version | string | *Required*. | -| | | Semantic | -| | | version | -| | | for this | -| | | specification | -+-------------+--------+----------------+ -| name | string | *Required*. | -| | | Full | -| | | name of | -| | | this | -| | | component | -| | | which is | -| | | also | -| | | used as | -| | | this | -| | | component's | -| | | catalog | -| | | id. | -+-------------+--------+----------------+ -| description | string | *Required* | -| | | Human-readable | -| | | text | -| | | describing | -| | | the | -| | | component | -| | | and the | -| | | components | -| | | functional | -| | | purpose. | -+-------------+--------+----------------+ -| component_t\| string | *Required* | -| ype | | Identify | -| | | what | -| | | containe\ | -| | | rization | -| | | technolo\ | -| | | gy | -| | | this | -| | | componen\ | -| | | t | -| | | uses: | -| | | *docker* | -| | | or | -| | | *cdap*. | -| | | | -+-------------+--------+----------------+ - -.. _interfaces: - -Interfaces ----------- - -Interfaces are the JSON objects found under the ``streams`` key and the -``services`` key. These are used to describe the interfaces that the -component uses and the interfaces that the component provides. The -description of each interface includes the associated :any:`data -format `. - -Streams -~~~~~~~ - -- The ``streams`` JSON is for specifying data produced for consumption - by other components, and the streams expected to subscribe to that is - produced by other components. These are “fire and forget” type - interfaces where the publisher of a stream does not expect or parse a - response from the subscriber. -- The term ``stream`` here is abstract and neither refers to “CDAP - streams” or “DMaaP feeds”. While a stream is very likely a DMaaP - feed, it could be a direct stream of data being routed via HTTP too. - It abstractly refers to a sequence of data leaving a publisher. -- Streams have anonymous publish/subscribe semantics, which decouples - the production of information from its consumption. Like the component - specification, the data format specification is represented/validated against this - `Data Format json schema `__ -- In general, components are not aware of who they are communicating - with. -- Instead, components that are interested in data, subscribe to the - relevant stream; components that generate data publish to the - relevant stream. -- There can be multiple publishers and subscribers to a stream. Streams - are intended for unidirectional, streaming communication. - -Streams interfaces that implement an HTTP endpoint must support POST. - -Streams are split into: - -+-------------+----+----------+ -| Property | Ty\| Descript\| -| Name | pe | ion | -+=============+====+==========+ -| subscribes | JS\| *Require\| -| | ON | d*. | -| | li\| List of | -| | st | all | -| | | availabl\| -| | | e | -| | | stream | -| | | interfac\| -| | | es | -| | | that | -| | | this | -| | | componen\| -| | | t | -| | | has that | -| | | can be | -| | | used for | -| | | subscrib\| -| | | ing | -+-------------+----+----------+ -| publishes | JS\| *Require\| -| | ON | d*. | -| | li\| List of | -| | st | all | -| | | stream | -| | | interfac\| -| | | es | -| | | that | -| | | this | -| | | componen\| -| | | t | -| | | will | -| | | publish | -| | | onto | -+-------------+----+----------+ - -Subscribes -^^^^^^^^^^ - -Example: - -.. code:: json - - "streams": { - "subscribes": [{ - "format": "dcae.vnf.kpi", - "version": "1.0.0", - "route": "/data", // for CDAP this value is not used - "type": "http" - }], - ... - } - -This describes that ``yourapp.component.kpi_anomaly`` exposes an HTTP -endpoint called ``/data`` which accepts requests that have the data -format of ``dcae.vnf.kpi`` version ``1.0.0``. - -``subscribes`` Schema: - -+-------------+----+--------------------+ -| Property | Ty\| Descript\ | -| Name | pe | ion | -+=============+====+====================+ -| format | st\| *Require\ | -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | id of | -| | | the data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\ | -| | | e | -+-------------+----+--------------------+ -| version | st\| *Require\ | -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | version | -| | | of the | -| | | data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\ | -| | | e | -+-------------+----+--------------------+ -| route | st\| *Require\ | -| | ri\| d | -| | ng | for HTTP | -| | | and data | -| | | router*. | -| | | The HTTP | -| | | route | -| | | that | -| | | this | -| | | interfac\ | -| | | e | -| | | listens | -| | | on | -+-------------+----+--------------------+ -| config_key | st\| *Require\ | -| | ri\| d \ | -| | ng | for \ | -| | | message_router\ | -| | | and data \ | -| | | router*. | -| | | The HTTP | -| | | route | -| | | that | -| | | this | -| | | interfac\ | -| | | e | -| | | listens | -| | | on | -+-------------+----+--------------------+ -| type | st\| *Require\ | -| | ri\| d*. | -| | ng | Type of | -| | | stream: | -| | | ``http`` | -| | | , | -| | | ``message_router`` | -| | | , | -| | | ``data_router`` | -+-------------+----+--------------------+ - -.. _message-router: - -Message router -'''''''''''''' - -Message router subscribers are http clients rather than http services -and performs a http a ``GET`` call. Thus, message router subscribers -description is structured like message router publishers and requires -``config_key``: - -.. code:: json - - "streams": { - "subscribes": [{ - "format": "dcae.some-format", - "version": "1.0.0", - "config_key": "some_format_handle", - "type": "message router" - }], - ... - } - - -.. _data-router: - -Data router -''''''''''' - -Data router subscribers are http or https services that handle ``PUT`` -requests from data router. Developers must provide the ``route`` or url -path/endpoint that is expected to handle data router requests. This will -be used to construct the delivery url needed to register the subscriber -to the provisioned feed. Developers must also provide a ``config_key`` -because there is dynamic configuration information associated with the -feed that the application will need e.g. username and password. See the -page on :any:`DMaaP connection objects ` for more details on -the configuration information. - -Example (not tied to the larger example): - -.. code:: json - - "streams": { - "subscribes": [{ - "config_key": "some-sub-dr", - "format": "sandbox.platform.any", - "route": "/identity", - "type": "data_router", - "version": "0.1.0" - }], - ... - } - -Publishes -^^^^^^^^^ - -Example: - -.. code:: json - - "streams": { - ... - "publishes": [{ - "format": "yourapp.format.integerClassification", - "version": "1.0.0", - "config_key": "prediction", - "type": "http" - }] - }, - -This describes that ``yourapp.component.kpi_anomaly`` publishes by making -POST requests to streams that support the data format -``yourapp.format.integerClassification`` version ``1.0.0``. - -``publishes`` Schema: - -+-------------+----+--------------------+ -| Property | Ty\| Descript\ | -| Name | pe | ion | -+=============+====+====================+ -| format | st\| *Require\ | -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | id of | -| | | the data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\ | -| | | e | -+-------------+----+--------------------+ -| version | st\| *Require\ | -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | version | -| | | of the | -| | | data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\ | -| | | e | -+-------------+----+--------------------+ -| config_key | st\| *Require\ | -| | ri\| d*. | -| | ng | The JSON | -| | | key in | -| | | the | -| | | generate\ | -| | | d | -| | | applicat | -| | | ion | -| | | configur\ | -| | | ation | -| | | that | -| | | will be | -| | | used to | -| | | pass the | -| | | downstre\ | -| | | am | -| | | componen\ | -| | | t’s | -| | | (the | -| | | subscrib\ | -| | | er’s) | -| | | connecti\ | -| | | on | -| | | informat\ | -| | | ion. | -+-------------+----+--------------------+ -| type | st\| *Require\ | -| | ri\| d*. | -| | ng | Type of | -| | | stream: | -| | | ``http`` | -| | | , | -| | | ``message_router`` | -| | | , | -| | | ``data_router`` | -+-------------+----+--------------------+ - -.. message-router-1: - -Message router -'''''''''''''' - -Message router publishers are http clients of DMaap message_router. -Developers must provide a ``config_key`` because there is dynamic -configuration information associated with the feed that the application -needs to receive e.g. topic url, username, password. See the page on -:any:`DMaaP connection objects ` for more details on -the configuration information. - -Example (not tied to the larger example): - -.. code:: json - - "streams": { - ... - "publishes": [{ - "config_key": "some-pub-mr", - "format": "sandbox.platform.any", - "type": "message_router", - "version": "0.1.0" - }] - } - -.. data-router-1: - -Data router -''''''''''' - -Data router publishers are http clients that make ``PUT`` requests to -data router. Developers must also provide a ``config_key`` because there -is dynamic configuration information associated with the feed that the -application will need to receive e.g. publish url, username, password. -See the page on :any:`DMaaP connection objects ` for more details on -the configuration information. - -Example (not tied to the larger example): - -.. code:: json - - "streams": { - ... - "publishes": [{ - "config_key": "some-pub-dr", - "format": "sandbox.platform.any", - "type": "data_router", - "version": "0.1.0" - }] - } - -Quick Reference -^^^^^^^^^^^^^^^ - -Refer to this :doc:`Quick Reference ` for a -comparison of the Streams ‘Publishes’ and ‘Subscribes’ sections. - -Services -~~~~~~~~ - -- The publish / subscribe model is a very flexible communication - paradigm, but its many-to-many one-way transport is not appropriate - for RPC request / reply interactions, which are often required in a - distributed system. -- Request / reply is done via a Service, which is defined by a pair of - messages: one for the request and one for the reply. - -Services are split into: - -+-------------+----+----------+ -| Property | Ty\| Descript\| -| Name | pe | ion | -+=============+====+==========+ -| calls | JS\| *Require\| -| | ON | d*. | -| | li\| List of | -| | st | all | -| | | service | -| | | interfac\| -| | | es | -| | | that | -| | | this | -| | | componen\| -| | | t | -| | | will | -| | | call | -+-------------+----+----------+ -| provides | JS\| *Require\| -| | ON | d*. | -| | li\| List of | -| | st | all | -| | | service | -| | | interfac\| -| | | es | -| | | that | -| | | this | -| | | componen\| -| | | t | -| | | exposes | -| | | and | -| | | provides | -+-------------+----+----------+ - -Calls -^^^^^ - -The JSON ``services/calls`` is for specifying that the component relies -on an HTTP(S) service—the component sends that service an HTTP request, -and that service responds with an HTTP reply. An example of this is how -string matching (SM) depends on the AAI Broker. SM performs a -synchronous REST call to the AAI broker, providing it the VMNAME of the -VNF, and the AAI Broker responds with additional details about the VNF. -This dependency is expressed via ``services/calls``. In contrast, the -output of string matching (the alerts it computes) is sent directly to -policy as a fire-and-forget interface, so that is an example of a -``stream``. - -Example: - -.. code:: json - - "services": { - "calls": [{ - "config_key": "vnf-db", - "request": { - "format": "dcae.vnf.meta", - "version": "1.0.0" - }, - "response": { - "format": "dcae.vnf.kpi", - "version": "1.0.0" - } - }], - ... - } - -This describes that ``yourapp.component.kpi_anomaly`` will make HTTP -calls to a downstream component that accepts requests of data format -``dcae.vnf.meta`` version ``1.0.0`` and is expecting the response to be -``dcae.vnf.kpi`` version ``1.0.0``. - -``calls`` Schema: - -+-------------+----+----------+ -| Property | Ty\| Descript\| -| Name | pe | ion | -+=============+====+==========+ -| request | JS\| *Require\| -| | ON | d*. | -| | ob\| Descript\| -| | je\| ion | -| | ct | of the | -| | | expected | -| | | request | -| | | for this | -| | | downstre\| -| | | am | -| | | interfac\| -| | | e | -+-------------+----+----------+ -| response | JS\| *Require\| -| | ON | d*. | -| | ob\| Descript\| -| | je\| ion | -| | ct | of the | -| | | expected | -| | | response | -| | | for this | -| | | downstre\| -| | | am | -| | | interfac\| -| | | e | -+-------------+----+----------+ -| config_key | st\| *Require\| -| | ri\| d*. | -| | ng | The JSON | -| | | key in | -| | | the | -| | | generate\| -| | | d | -| | | applicat | -| | | ion | -| | | configur\| -| | | ation | -| | | that | -| | | will be | -| | | used to | -| | | pass the | -| | | downstre\| -| | | am | -| | | componen | -| | | t | -| | | connecti\| -| | | on | -| | | informat\| -| | | ion. | -+-------------+----+----------+ - -The JSON object schema for both ``request`` and ``response``: - -+-------------+----+----------+ -| Property | Ty\| Descript\| -| Name | pe | ion | -+=============+====+==========+ -| format | st\| *Require\| -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | id of | -| | | the data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\| -| | | e | -+-------------+----+----------+ -| version | st\| *Require\| -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | version | -| | | of the | -| | | data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\| -| | | e | -+-------------+----+----------+ - -Provides -^^^^^^^^ - -Example: - -.. code:: json - - "services": { - ... - "provides": [{ - "route": "/score-vnf", - "request": { - "format": "dcae.vnf.meta", - "version": "1.0.0" - }, - "response": { - "format": "yourapp.format.integerClassification", - "version": "1.0.0" - } - }] - }, - -This describes that ``yourapp.component.kpi_anomaly`` provides a service -interface and it is exposed on the ``/score-vnf`` HTTP endpoint. The -endpoint accepts requests that have the data format ``dcae.vnf.meta`` -version ``1.0.0`` and gives back a response of -``yourapp.format.integerClassification`` version ``1.0.0``. - -``provides`` Schema for a Docker component: - -+-------------+----+----------+ -| Property | Ty\| Descript\| -| Name | pe | ion | -+=============+====+==========+ -| request | JS\| *Require\| -| | ON | d*. | -| | ob\| Descript\| -| | je\| ion | -| | ct | of the | -| | | expected | -| | | request | -| | | for this | -| | | interfac\| -| | | e | -+-------------+----+----------+ -| response | JS\| *Require\| -| | ON | d*. | -| | ob\| Descript\| -| | je\| ion | -| | ct | of the | -| | | expected | -| | | response | -| | | for this | -| | | interfac\| -| | | e | -+-------------+----+----------+ -| route | st\| *Require\| -| | ri\| d*. | -| | ng | The HTTP | -| | | route | -| | | that | -| | | this | -| | | interfac\| -| | | e | -| | | listens | -| | | on | -+-------------+----+----------+ - -The JSON object schema for both ``request`` and ``response``: - -+-------------+----+----------+ -| Property | Ty\| Descript\| -| Name | pe | ion | -+=============+====+==========+ -| format | st\| *Require\| -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | id of | -| | | the data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\| -| | | e | -+-------------+----+----------+ -| version | st\| *Require\| -| | ri\| d*. | -| | ng | Data | -| | | format | -| | | version | -| | | of the | -| | | data | -| | | format | -| | | that is | -| | | used by | -| | | this | -| | | interfac\| -| | | e | -+-------------+----+----------+ - -Note, for CDAP, there is a slight variation due to the way CDAP exposes -services: - -:: - - "provides":[ // note this is a list of JSON - { - "request":{ ...}, - "response":{ ...}, - "service_name":"name CDAP service", - "service_endpoint":"greet", // E.g the URL is /services/service_name/methods/service_endpoint - "verb":"GET" // GET, PUT, or POST - } - ] - -``provides`` Schema for a CDAP component: - -+-------------+----+-----------+ -| Property | Ty\| Descript\ | -| Name | pe | ion | -+=============+====+===========+ -| request | JS\| *Require\ | -| | ON | d*. | -| | ob\| Descript\ | -| | je\| ion | -| | ct | of the | -| | | expected | -| | | request | -| | | data | -| | | format | -| | | for this | -| | | interfac\ | -| | | e | -+-------------+----+-----------+ -| response | JS\| *Require\ | -| | ON | d*. | -| | ob\| Descript\ | -| | je\| ion | -| | ct | of the | -| | | expected | -| | | response | -| | | for this | -| | | interfac\ | -| | | e | -+-------------+----+-----------+ -| service_nam\| st\| *Require\ | -| e | ri\| d*. | -| | ng | The CDAP | -| | | service | -| | | name (eg | -| | | “Greetin\ | -| | | g”) | -+-------------+----+-----------+ -| service_end | st\| *Require\ | -| point | ri\| d*. | -| | ng | The CDAP | -| | | service | -| | | endpoint | -| | | for this | -| | | service_n\| -| | | ame | -| | | (eg | -| | | “/greet” | -| | | ) | -+-------------+----+-----------+ -| verb | st\| *Require\ | -| | ri\| d*. | -| | ng | ‘GET’, | -| | | ‘PUT’ or | -| | | ‘POST’ | -+-------------+----+-----------+ - -.. _common-specification-parameters: - -Parameters ----------- - -``parameters`` is where to specify the component’s application -configuration parameters that are not connection information. - -+---------------+------------+----------------------------------+ -| Property Name | Type | Description | -+===============+============+==================================+ -| parameters | JSON array | Each entry is a parameter object | -+---------------+------------+----------------------------------+ - -Parameter object has the following available properties: - -+--------------+----+----------+------+ -| Property | Ty\| Descript\| Defa\| -| Name | pe | ion | ult | -+==============+====+==========+======+ -| name | st\| *Require\| | -| | ri\| d*. | | -| | ng | The | | -| | | property | | -| | | name | | -| | | that | | -| | | will be | | -| | | used as | | -| | | the key | | -| | | in the | | -| | | generate\| | -| | | d | | -| | | config | | -+--------------+----+----------+------+ -| value | an\| *Require\| | -| | y | d*. | | -| | | The | | -| | | default | | -| | | value | | -| | | for the | | -| | | given | | -| | | paramete\| | -| | | r | | -+--------------+----+----------+------+ -| description | st\| *Require\| | -| | ri\| d*. | | -| | ng | Human-re\| | -| | | adable | | -| | | text | | -| | | describi\| | -| | | ng | | -| | | the | | -| | | paramete\| | -| | | r | | -| | | like | | -| | | what its | | -| | | for | | -+--------------+----+----------+------+ -| type | st\| The | | -| | ri\| required | | -| | ng | data | | -| | | type for | | -| | | the | | -| | | paramete\| | -| | | r | | -+--------------+----+----------+------+ -| required | bo\| An | true | -| | ol\| optional | | -| | ea\| key that | | -| | n | declares | | -| | | a | | -| | | paramete\| | -| | | r | | -| | | as | | -| | | required | | -| | | (true) | | -| | | or not | | -| | | (false) | | -+--------------+----+----------+------+ -| constraints | ar\| The | | -| | ra\| optional | | -| | y | list of | | -| | | sequence | | -| | | d | | -| | | constrai\| | -| | | nt | | -| | | clauses | | -| | | for the | | -| | | paramete\| | -| | | r. | | -| | | See | | -| | | below | | -+--------------+----+----------+------+ -| entry_schem\ | st\| The | | -| a | ri\| optional | | -| | ng | key that | | -| | | is used | | -| | | to | | -| | | declare | | -| | | the name | | -| | | of the | | -| | | Datatype | | -| | | definiti\| | -| | | on | | -| | | for | | -| | | entries | | -| | | of set | | -| | | types | | -| | | such as | | -| | | the | | -| | | TOSCA | | -| | | ‘list’ | | -| | | or | | -| | | ‘map’. | | -| | | Only 1 | | -| | | level is | | -| | | supporte\| | -| | | d | | -| | | at this | | -| | | time | | -+--------------+----+----------+------+ -| designer_ed\ | bo\| An | | -| itable | ol\| optional | | -| | ea\| key that | | -| | n | declares | | -| | | a | | -| | | paramete\| | -| | | r | | -| | | to be | | -| | | editable | | -| | | by | | -| | | designer | | -| | | (true) | | -| | | or not | | -| | | (false) | | -+--------------+----+----------+------+ -| sourced_at_d\| bo\| An | | -| eployment | ol\| optional | | -| | ea\| key that | | -| | n | declares | | -| | | a | | -| | | paramete\| | -| | | r’s | | -| | | value to | | -| | | be | | -| | | assigned | | -| | | at | | -| | | deployme\| | -| | | nt | | -| | | time | | -| | | (true) | | -+--------------+----+----------+------+ -| policy_edit\ | bo\| An | | -| able | ol\| optional | | -| | ea\| key that | | -| | n | declares | | -| | | a | | -| | | paramete\| | -| | | r | | -| | | to be | | -| | | editable | | -| | | by | | -| | | policy | | -| | | (true) | | -| | | or not | | -| | | (false) | | -+--------------+----+----------+------+ -| policy_sche\ | ar\| The | | -| ma | ra\| optional | | -| | y | list of | | -| | | schema | | -| | | definiti\| | -| | | ons | | -| | | used for | | -| | | policy. | | -| | | See | | -| | | below | | -+--------------+----+----------+------+ - -Example: - -.. code:: json - - "parameters": [ - { - "name": "threshold", - "value": 0.75, - "description": "Probability threshold to exceed to be anomalous" - } - ] - -Many of the parameter properties have been copied from TOSCA model -property definitions and are to be used for service design composition -and policy creation. See `section 3.5.8 *Property -definition* `__. - -The property ``constraints`` is a list of objects where each constraint -object: - -+--------------+----+----------+ -| Property | Ty\| Descript\| -| Name | pe | ion | -+==============+====+==========+ -| equal | | Constrai\| -| | | ns | -| | | a | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value | -| | | equal to | -| | | (‘=’) | -| | | the | -| | | value | -| | | declared | -+--------------+----+----------+ -| greater_tha\ | nu\| Constrai\| -| n | mb\| ns | -| | er | a | -| | | property | -| | | or | -| | | paramete | -| | | r | -| | | to a | -| | | value | -| | | greater | -| | | than | -| | | (‘>’) | -| | | the | -| | | value | -| | | declared | -+--------------+----+----------+ -| greater_or_e\| nu\| Constrai\| -| qual | mb\| ns | -| | er | a | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value | -| | | greater | -| | | than or | -| | | equal to | -| | | (‘>=’) | -| | | the | -| | | value | -| | | declared | -+--------------+----+----------+ -| less_than | nu\| Constrai\| -| | mb\| ns | -| | er | a | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value | -| | | less | -| | | than | -| | | (‘<’) | -| | | the | -| | | value | -| | | declared | -+--------------+----+----------+ -| less_or_equ\ | nu\| Constrai\| -| al | mb\| ns | -| | er | a | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value | -| | | less | -| | | than or | -| | | equal to | -| | | (‘<=’) | -| | | the | -| | | value | -| | | declared | -+--------------+----+----------+ -| valid_value\ | ar\| Constrai\| -| s | ra\| ns | -| | y | a | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value | -| | | that is | -| | | in the | -| | | list of | -| | | declared | -| | | values | -+--------------+----+----------+ -| length | nu\| Constrai\| -| | mb\| ns | -| | er | the | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value of | -| | | a given | -| | | length | -+--------------+----+----------+ -| min_length | nu\| Constrai\| -| | mb\| ns | -| | er | the | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value to | -| | | a | -| | | minimum | -| | | length | -+--------------+----+----------+ -| max_length | nu\| Constrai\| -| | mb\| ns | -| | er | the | -| | | property | -| | | or | -| | | paramete\| -| | | r | -| | | to a | -| | | value to | -| | | a | -| | | maximum | -| | | length | -+--------------+----+----------+ - -``threshold`` is the configuration parameter and will get set to 0.75 -when the configuration gets generated. - -The property ``policy_schema`` is a list of objects where each -policy_schema object: - -+-------------+----+----------+------+ -| Property | Ty\| Descript\| Defa\| -| Name | pe | ion | ult | -+=============+====+==========+======+ -| name | st\| *Require\| | -| | ri\| d*. | | -| | ng | paramete\| | -| | | r | | -| | | name | | -+-------------+----+----------+------+ -| value | st\| default | | -| | ri\| value | | -| | ng | for the | | -| | | paramete\| | -| | | r | | -+-------------+----+----------+------+ -| description | st\| paramete\| | -| | ri\| r | | -| | ng | descript\| | -| | | ion | | -+-------------+----+----------+------+ -| type | en\| *Require\| | -| | um | d*. | | -| | | data | | -| | | type of | | -| | | the | | -| | | paramete\| | -| | | r, | | -| | | ‘string’ | | -| | | , | | -| | | ‘number’ | | -| | | , | | -| | | ‘boolean | | -| | | ’, | | -| | | ‘datetim\| | -| | | e’, | | -| | | ‘list’, | | -| | | or ‘map’ | | -+-------------+----+----------+------+ -| required | bo\| is | true | -| | ol\| paramete\| | -| | ea\| r | | -| | n | required | | -| | | or not? | | -+-------------+----+----------+------+ -| constraints | ar\| The | | -| | ra\| optional | | -| | y | list of | | -| | | sequence\| | -| | | d | | -| | | constrai\| | -| | | nt | | -| | | clauses | | -| | | for the | | -| | | paramete\| | -| | | r. | | -| | | See | | -| | | above | | -+-------------+----+----------+------+ -| entry_schem\| st\| The | | -| a | ri\| optional | | -| | ng | key that | | -| | | is used | | -| | | to | | -| | | declare | | -| | | the name | | -| | | of the | | -| | | Datatype | | -| | | definiti\| | -| | | on | | -| | | for | | -| | | certain | | -| | | types. | | -| | | entry_sc\| | -| | | hema | | -| | | must be | | -| | | defined | | -| | | when the | | -| | | type is | | -| | | either | | -| | | list or | | -| | | map. If | | -| | | the type | | -| | | is list | | -| | | and the | | -| | | entry | | -| | | type is | | -| | | a simple | | -| | | type | | -| | | (string, | | -| | | number, | | -| | | bookean, | | -| | | datetime | | -| | | ), | | -| | | follow | | -| | | with an | | -| | | string | | -| | | to | | -| | | describe | | -| | | the | | -| | | entry | | -+-------------+----+----------+------+ -| | If | | | -| | th\| | | -| | e | | | -| | ty\| | | -| | pe | | | -| | is | | | -| | li\| | | -| | st | | | -| | an\| | | -| | d | | | -| | th\| | | -| | e | | | -| | en\| | | -| | tr\| | | -| | y | | | -| | ty\| | | -| | pe | | | -| | is | | | -| | a | | | -| | ma\| | | -| | p, | | | -| | fo\| | | -| | ll\| | | -| | ow | | | -| | wi\| | | -| | th | | | -| | an | | | -| | ar\| | | -| | ra\| | | -| | y | | | -| | to | | | -| | de\| | | -| | sc\| | | -| | ri\| | | -| | be | | | -| | th\| | | -| | e | | | -| | ke\| | | -| | ys | | | -| | fo\| | | -| | r | | | -| | th\| | | -| | e | | | -| | en\| | | -| | tr\| | | -| | y | | | -| | ma\| | | -| | p | | | -+-------------+----+----------+------+ -| | If | | | -| | th\| | | -| | e | | | -| | ty\| | | -| | pe | | | -| | is | | | -| | li\| | | -| | st | | | -| | an\| | | -| | d | | | -| | th\| | | -| | e | | | -| | en\| | | -| | tr\| | | -| | y | | | -| | ty\| | | -| | pe | | | -| | is | | | -| | a | | | -| | li\| | | -| | st | | | -| | , | | | -| | th\| | | -| | at | | | -| | is | | | -| | no\| | | -| | t | | | -| | cu\| | | -| | rr\| | | -| | en\| | | -| | tl\| | | -| | y | | | -| | su\| | | -| | pp\| | | -| | or\| | | -| | te\| | | -| | d \| | | -+-------------+----+----------+------+ -| | If | | | -| | th\| | | -| | e | | | -| | ty\| | | -| | pe | | | -| | is | | | -| | ma\| | | -| | p, | | | -| | fo\| | | -| | ll\| | | -| | ow | | | -| | wi\| | | -| | th | | | -| | an | | | -| | ar\| | | -| | ay | | | -| | to | | | -| | de\| | | -| | sc\| | | -| | ri\| | | -| | be | | | -| | th\| | | -| | e | | | -| | ke\| | | -| | ys | | | -| | fo\| | | -| | r | | | -| | th\| | | -| | e | | | -| | ma\| | | -| | p | | | -+-------------+----+----------+------+ - -Generated Application Configuration ------------------------------------ - -The Platform generates configuration for the component (based on the component spec) at deployment time. The generated application configuration will be made up of the Parameters, Streams, and Services sections, after any provisioning for streams and services. The component developer can see what this configuration will look like by reviewing the :any:`component dev command `. - - -.. _artifacts: - -Artifacts ---------- - -``artifacts`` contains a list of artifacts associated with this -component. For Docker, this is the full path (including the registry) to -the Docker image. For CDAP, this is the full path to the CDAP jar. - -+---------------+------------+---------------------------------+ -| Property Name | Type | Description | -+===============+============+=================================+ -| artifacts | JSON array | Each entry is a artifact object | -+---------------+------------+---------------------------------+ - -``artifact`` Schema: - -+---------------+--------+--------------------------------------------+ -| Property Name | Type | Description | -+===============+========+============================================+ -| uri | string | *Required*. Uri to the artifact, full path | -+---------------+--------+--------------------------------------------+ -| type | string | *Required*. ``docker image`` or ``jar`` | -+---------------+--------+--------------------------------------------+ - -.. _component_spec: - -Working with Component Specs -============================ - -Components can be added to the onboarding catalog (which first validates the component spec) by using the :doc:`dcae_cli Tool <../dcae-cli/quickstart/>` -Here you can also list the components, show the contents of a component, publish the component, validate the generated configuration for the component, deploy (run) and undeploy the component. For a list of these capabilities, see :any:`Component Commands ` - diff --git a/docs/sections/components/component-specification/component-specification.rst b/docs/sections/components/component-specification/component-specification.rst deleted file mode 100644 index 56649d31..00000000 --- a/docs/sections/components/component-specification/component-specification.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -Component Specification -======================= - -.. toctree:: - :maxdepth: 1 - - ./common-specification.rst - ./cdap-specification.rst - ./docker-specification.rst - ./dmaap-connection-objects.rst - ./streams-grid.rst - ./configuration-grid.rst - diff --git a/docs/sections/components/component-specification/configuration-grid.rst b/docs/sections/components/component-specification/configuration-grid.rst deleted file mode 100755 index da9b4ab7..00000000 --- a/docs/sections/components/component-specification/configuration-grid.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -Configuration Quick Reference -============================= - -Default Values -^^^^^^^^^^^^^^ - -The component developer can provide default values for any ``parameter`` -in the component specification. These defaults will be passed to the -component in its generated configuration. - -Overridden/Entered Values -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Depending on the other properties set for the parameter, the default -value can be overridden at ‘design-time’, ‘deploy-time’ or once the -microservice is running (‘run-time’). (*In the future, when Policy is -supported, configuration will also be able to be provided/changed in the -Policy UI at any time).* - - -+--------+--------+--------+--------+------------+ -| | Design\| CLAMP | Policy | Deploy-Time| -| | -Time | Input | Input | Input | -| | Input | | (futur\| | -| | | | e) | | -+========+========+========+========+============+ -| Descri\| Applie\| Applie\| (not | Applies to | -| ption | s | s | yet | manually | -| | to SDC | to | suppor\| deployed | -| | self-s\| compon\| ted) | services | -| | ervice | ents | | | -| | compon\| deploy\| | | -| | ents | ed | | | -| | | by | | | -| | | CLAMP | | | -+--------+--------+--------+--------+------------+ -| Input | Servic\| CLAMP | Operat\| DevOps | -| provid\| e | | ions | | -| ed | Design | | | | -| by | er | | | | -+--------+--------+--------+--------+------------+ -| How it | In the | In the | In the | In the DCAE| -| is | SDC UI | CLAMP | POLICY | Dashboard | -| provid\| | UI | GUI | (or Jenkins| -| ed | | | | job) | -+--------+--------+--------+--------+------------+ -| Compon\| ‘desig\| None. | ‘polic\| ‘sourced\ | -| ent | ner-ed\| Develo\| y_edit\| _at_deploy\| -| Specif\| itable\| per | able’ | ment’ must | -| icatio\| ’ | provid\| must | be set to | -| n | set to | es | be set | ‘true’ | -| Detail\| ‘true’ | CLAMP | to | | -| s | | an | ‘true’ | | -| | | email | and | | -| | | with | ‘polic\| | -| | | parame\| y_sche\| | -| | | ters | ma’ | | -| | | to be | must | | -| | | suppor\| be | | -| | | ted | provid\| | -| | | | ed | | -| | | | | | -| | | | | | -+--------+--------+--------+--------+------------+ -| Additi\| | | For | | -| onal | | | Docker | | -| Info | | | only: | | -| for | | | In the | | -| Compon\| | | auxili\| | -| ent | | | ary | | -| Develo\| | | sectio\| | -| per | | | n: | | -| | | | {“poli\| | -| | | | cy”: | | -| | | | {“trig\| | -| | | | ger_ty\| | -| | | | pe”: | | -| | | | “polic\| | -| | | | y”,“sc\| | -| | | | ript_p\| | -| | | | ath”: | | -| | | | “/opt/\| | -| | | | app/re\| | -| | | | config\| | -| | | | ure.sh | | -| | | | ”} | | -| | | | } | | -| | | | Script | | -| | | | interf\| | -| | | | ace | | -| | | | would | | -| | | | then | | -| | | | be | | -| | | | “/opt/\| | -| | | | app/re\| | -| | | | config\| | -| | | | ure.sh | | -| | | | ” | | -| | | | $trigg\| | -| | | | er_typ\| | -| | | | e | | -| | | | $updat\| | -| | | | ed_pol\| | -| | | | icy" | | -| | | | where | | -| | | | $updat\| | -| | | | ed_pol\| | -| | | | icy | | -| | | | is | | -| | | | json | | -| | | | provid\| | -| | | | ed | | -| | | | by the | | -| | | | Policy | | -| | | | Handle\| | -| | | | r. | | -+--------+--------+--------+--------+------------+ - diff --git a/docs/sections/components/component-specification/dmaap-connection-objects.rst b/docs/sections/components/component-specification/dmaap-connection-objects.rst deleted file mode 100755 index dfd4980f..00000000 --- a/docs/sections/components/component-specification/dmaap-connection-objects.rst +++ /dev/null @@ -1,276 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _dmaap-connection-objects: - -DMaaP connection objects -======================== - -DMaaP Connection Objects are JSON objects that: - -1. At Runtime - this is generated by the DCAE Platform and passed to the - component in its application_configuration to be used to connect to - the DMaaP feed or topic. Components will receive the entire object - with all properties populated (default will be \`null) unless - specified otherwise. -2. During dcae_cli testing - this is provided through the command-line - argument ``--dmaap-file`` to test the component with manually - provisioned feeds and topics. Developers are not required to provide - the entire object. The required properties are labeled below with - “*Required as input*”. - -.. _dmaap-message-router: - -Message Router --------------- - -Publishers and subscribers have the same generated -``Dmaap Connection Object`` structure. Here’s an example for any given -config-key: (This is what will be in application_configuration) - -.. code:: json - - { - "type": "message_router", - "aaf_username": "some-user", - "aaf_password": "some-password", - "dmaap_info": { - "client_role": "com.dcae.member", - "client_id": "1500462518108", - "location": "mtc00", - "topic_url": "https://we-are-message-router.us:3905/events/some-topic" - } - } - -At the top-level: - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| type | string | *Required as input*. Must | -| | | be ``message_router`` for | -| | | message router topics | -+--------------------------------+---------+---------------------------+ -| aaf_username | string | AAF username message | -| | | router clients use to | -| | | authenticate with secure | -| | | topics | -+--------------------------------+---------+---------------------------+ -| aaf_password | string | AAF password message | -| | | router clients use to | -| | | authenticate with secure | -| | | topics | -+--------------------------------+---------+---------------------------+ -| dmaap_info | JSON | *Required as input*. | -| | object | Contains the topic | -| | | connection details | -+--------------------------------+---------+---------------------------+ - -The ``dmaap_info`` object contains: - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| client_role | string | AAF client role that’s | -| | | requesting publish or | -| | | subscribe access to the | -| | | topic | -+--------------------------------+---------+---------------------------+ -| client_id | string | Client id for given AAF | -| | | client | -+--------------------------------+---------+---------------------------+ -| location | string | DCAE location for the | -| | | publisher or subscriber, | -| | | used to set up routing | -+--------------------------------+---------+---------------------------+ -| topic_url | string | *Required as input*. URL | -| | | for accessing the topic | -| | | to publish or receive | -| | | events | -+--------------------------------+---------+---------------------------+ - -The –dmaap-file argument (to the component ``run`` or ``dev`` command), -must minimally contain: - -.. code:: json - - { - "type": "message_router", - "dmaap_info": { - "topic_url": "https://we-are-message-router.us:3905/events/some-topic" - } - } - -.. _dmaap-data-router: - -Data Router ------------ - -Publisher -~~~~~~~~~ - -Here’s an example of what the generated ``Dmaap Connection Object`` for -Data Router Publisher looks like: (This is what will be in -application_configuration) - -.. code:: json - - { - "type": "data_router", - "dmaap_info": { - "location": "mtc00", - "publish_url": "https://we-are-data-router.us/feed/xyz", - "log_url": "https://we-are-data-router.us/feed/xyz/logs", - "username": "some-user", - "password": "some-password", - "publisher_id": "123456" - } - } - -At the top-level: - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| type | string | *Required as input*. Must | -| | | be ``data_router`` for | -| | | data router feeds | -+--------------------------------+---------+---------------------------+ -| dmaap_info | JSON | *Required as input*. | -| | object | Contains the feed | -| | | connection details | -+--------------------------------+---------+---------------------------+ - -The ``dmaap_info`` object contains: - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| location | string | DCAE location for the | -| | | publisher, used to set up | -| | | routing | -+--------------------------------+---------+---------------------------+ -| publish_url | string | *Required as input*. URL | -| | | to which the publisher | -| | | makes Data Router publish | -| | | requests | -+--------------------------------+---------+---------------------------+ -| log_url | string | URL from which log data | -| | | for the feed can be | -| | | obtained | -+--------------------------------+---------+---------------------------+ -| username | string | Username the publisher | -| | | uses to authenticate to | -| | | Data Router | -+--------------------------------+---------+---------------------------+ -| password | string | Password the publisher | -| | | uses to authenticate to | -| | | Data Router | -+--------------------------------+---------+---------------------------+ -| publisher_id | string | Publisher id in Data | -| | | Router | -+--------------------------------+---------+---------------------------+ - -The –dmaap-file argument (to the component ``run`` or ``dev`` command), -must minimally contain: - -.. code:: json - - { - "type": "data_router", - "dmaap_info": { - "publish_url": "https://we-are-data-router.us/feed/xyz" - } - } - -Subscriber -~~~~~~~~~~ - -Here’s an example of what the generated ``Dmaap Connection Object`` for -a Data Router Subscriber looks like: (This is what will be passed in -application_configuration) - -.. code:: json - - { - "type": "data_router", - "dmaap_info": { - "location": "mtc00", - "delivery_url": "https://my-subscriber-app.dcae:8080/target-path", - "username": "some-user", - "password": "some-password", - "subscriber_id": "789012" - } - } - -At the top-level: - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| type | string | *Required as input*. Must | -| | | be ``data_router`` for | -| | | data router feeds | -+--------------------------------+---------+---------------------------+ -| dmaap_info | JSON | *Required as input*. | -| | object | Contains the feed | -| | | connection details | -+--------------------------------+---------+---------------------------+ - -The ``dmaap_info`` object contains: - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| location | string | DCAE location for the | -| | | subscriber, used to set | -| | | up routing | -+--------------------------------+---------+---------------------------+ -| delivery_url | string | URL to which the Data | -| | | Router should deliver | -| | | files | -+--------------------------------+---------+---------------------------+ -| username | string | Username Data Router uses | -| | | to authenticate to the | -| | | subscriber when | -| | | delivering files | -+--------------------------------+---------+---------------------------+ -| password | string | Password Data Router uses | -| | | to authenticate to the | -| | | subscriber when | -| | | delivering files | -+--------------------------------+---------+---------------------------+ -| subscriber_id | string | Subscriber id in Data | -| | | Router | -+--------------------------------+---------+---------------------------+ - -The –dmaap-file argument (to the component ``run`` or ``dev`` command), -must minimally contain: - -.. code:: json - - { - "type": "data_router", - "dmaap_info": { - } - } - -It is the recommended security practice that ``username`` and -``password`` are specified. You cannot provide the ``delivery_url`` in -your dmaap-file, because it’s not constructed until deployment time. -Therefore, after the test deployment, go back to the Data Router Feed -and provide the delivery_url (in order to start receiving the feeds). - -Data Router Example -~~~~~~~~~~~~~~~~~~~ - -(After the Data Router feed has been manually provisioned) - -:: - - $ dcae_cli component run --dmaap-file $dmaap_file $component-name - DCAE.Run | WARNING | Your component is a data router subscriber. Here are the delivery urls: - - some-sub-dr: http://135.205.226.128:32838/identity - -(Update the Data Router Feed to provide the delivery_url). diff --git a/docs/sections/components/component-specification/docker-specification.rst b/docs/sections/components/component-specification/docker-specification.rst deleted file mode 100755 index 128c006d..00000000 --- a/docs/sections/components/component-specification/docker-specification.rst +++ /dev/null @@ -1,351 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _docker-specification: - -Component specification (Docker) -================================ - -The Docker component specification contains the following groups of -information. Many of these are common to both Docker and CDAP components -and are therefore described in the common specification. - -- :any:`Metadata ` -- :any:`Interfaces ` including the - associated :any:`Data Formats ` -- :any:`Parameters ` -- :any:`Auxiliary Details ` -- :any:`List of Artifacts ` - -.. _docker-auxiliary-details: - -Auxiliary Details ------------------ - -``auxiliary`` contains Docker specific details like health check, port -mapping, volume mapping and policy reconfiguration script details. -(Policy reconfiguration is not yet supported). - -+--------------------------------+---------+---------------------------+ -| Name | Type | Description | -+================================+=========+===========================+ -| healthcheck | JSON | *Required*. Health check | -| | object | definition details | -+--------------------------------+---------+---------------------------+ -| ports | JSON | each array item maps a | -| | array | container port to the | -| | | host port. See example | -| | | below. | -+--------------------------------+---------+---------------------------+ -| volume | JSON | each array item contains | -| | array | a host and container | -| | | object. See example | -| | | below. | -+--------------------------------+---------+---------------------------+ -| *Planned for 1806* | | | -+--------------------------------+---------+---------------------------+ -| policy | JSON | *Required*. Policy | -| | array | reconfiguration script | -| | | details | -+--------------------------------+---------+---------------------------+ - -Health Check Definition -~~~~~~~~~~~~~~~~~~~~~~~ - -The platform uses Consul to perform periodic health check calls. Consul -provides different types of `check -definitions `__. The -platform currently supports http and docker health checks. - -When choosing a value for interval, consider that too frequent -healthchecks will put unnecessary load on Consul and DCAE. If there is a -problematic resource, then more frequent healthchecks are warranted (eg -15s or 60s), but as stability increases, so can these values, (eg -300s). - -When choosing a value for timeout, consider that too small a number will -result in increasing timeout failures, and too large a number will -result in a delay in the notification of the resource problem. A -suggestion is to start with 5s and work from there. - -http -^^^^ - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| type | string | *Required*. ``http`` | -+--------------------------------+---------+---------------------------+ -| interval | string | Interval duration in | -| | | seconds i.e. ``60s`` | -+--------------------------------+---------+---------------------------+ -| timeout | string | Timeout in seconds i.e. | -| | | ``5s`` | -+--------------------------------+---------+---------------------------+ -| endpoint | string | *Required*. GET endpoint | -| | | provided by the component | -| | | for Consul to call to | -| | | check health | -+--------------------------------+---------+---------------------------+ - -Example: - -.. code:: json - - "auxilary": { - "healthcheck": { - "type": "http", - "interval": "15s", - "timeout": "1s", - "endpoint": "/my-health" - } - } - -docker script example -^^^^^^^^^^^^^^^^^^^^^ - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| type | string | *Required*. ``docker`` | -+--------------------------------+---------+---------------------------+ -| interval | string | Interval duration in | -| | | seconds i.e. ``15s`` | -+--------------------------------+---------+---------------------------+ -| timeout | string | Timeout in seconds i.e. | -| | | ``1s`` | -+--------------------------------+---------+---------------------------+ -| script | string | *Required*. Full path of | -| | | script that exists in the | -| | | Docker container to be | -| | | executed | -+--------------------------------+---------+---------------------------+ - -Consul will use the `Docker exec -API `__ to -periodically call your script in your container. It will examine the -script result to identify whether your component is healthy. Your -component is considered healthy when the script returns ``0`` otherwise -your component is considered not healthy. - -Example: - -.. code:: json - - "auxilary": { - "healthcheck": { - "type": "docker", - "script": "/app/resources/check_health.py", - "timeout": "30s", - "interval": "180s" - } - } - -Ports -~~~~~ - -This method of exposing/mapping a local port to a host port is NOT -RECOMMENDED because of the possibility of port conflicts. If multiple -instances of a docker container will be running, there definitely will -be port conflicts. Use at your own risk. (The preferred way to expose a -port is to do so in the Dockerfile as described -:any:`here `). - -.. code:: json - - "auxilary": { - "ports": ["8080:8000"] - } - -In the example above, container port 8080 maps to host port 8000. - -Volume Mapping -~~~~~~~~~~~~~~ - -.. code:: json - - "auxilary": { - "volumes": [ - { - "container": { - "bind": "/tmp/docker.sock", - "mode": "ro" - }, - "host": { - "path": "/var/run/docker.sock" - } - } - ] - } - -At the top-level: - -+---------------+-------+-------------------------------------+ -| Property Name | Type | Description | -+===============+=======+=====================================+ -| volumes | array | Contains container and host objects | -+---------------+-------+-------------------------------------+ - -The ``container`` object contains: - - -+-----------------------+-----------------------+-------------------------------+ -| Property Name | Type | Description | -+=======================+=======================+===============================+ -| bind | string | path to the container | -| | | volume | -+-----------------------+-----------------------+-------------------------------+ -| mode | string | ro - indicates | -| | | read-only volume | -+-----------------------+-----------------------+-------------------------------+ -| | | w - indicates that | -| | | the contain can write | -| | | into the bind mount | -+-----------------------+-----------------------+-------------------------------+ - -The ``host`` object contains: - -+---------------+--------+-------------------------+ -| Property Name | Type | Description | -+===============+========+=========================+ -| path | string | path to the host volume | -+---------------+--------+-------------------------+ - -Here’s an example of the minimal JSON that must be provided as an input: - -.. code:: json - - "auxilary": { - "volumes": [ - { - "container": { - "bind": "/tmp/docker.sock" - }, - "host": { - "path": "/var/run/docker.sock" - } - } - ] - } - -In the example above, the container volume “/tmp/docker.sock” maps to -host volume “/var/run/docker.sock”. - - -Policy (not yet supported) -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Policy changes made in the Policy UI will be provided to the Docker -component by triggering a script that is defined here. - -+--------------------------------+---------+---------------------------+ -| Property Name | Type | Description | -+================================+=========+===========================+ -| reconfigure_type | string | *Required*. Current value | -| | | supported is ``policy`` | -+--------------------------------+---------+---------------------------+ -| script_path | string | *Required*. Current value | -| | | for ‘policy’ | -| | | reconfigure_type must be | -| | | “/opt/app/reconfigure.sh” | -+--------------------------------+---------+---------------------------+ - -Example: - -.. code:: json - - "auxilary": { - "policy": { - "reconfigure_type": "policy", - "script_path": "/opt/app/reconfigure.sh" - } - } - -The docker script interface is as follows: \`/opt/app/reconfigure.sh -$reconfigure_type {“updated policies”: , “application config”: } - -+---------------------+--------------+----------------------------------------+ -| Name | Type | Description | -+=====================+==============+========================================+ -| reconfigure_type | string | policy | -+---------------------+--------------+----------------------------------------+ -| updated_policies | json | TBD | -+---------------------+--------------+----------------------------------------+ -| updated_appl_config | json | complete generated app_config, not | -| | | fully-resolved, but ``policy-enabled`` | -| | | parameters have been updated. In order | -| | | to get the complete updated | -| | | app_config, the component would have | -| | | to call ``config-binding-service``. | -+---------------------+--------------+----------------------------------------+ - -Docker Component Spec - Complete Example ----------------------------------------- - -.. code:: json - - { - "self": { - "version": "1.0.0", - "name": "yourapp.component.kpi_anomaly", - "description": "Classifies VNF KPI data as anomalous", - "component_type": "docker" - }, - "streams": { - "subscribes": [{ - "format": "dcae.vnf.kpi", - "version": "1.0.0", - "route": "/data", - "type": "http" - }], - "publishes": [{ - "format": "yourapp.format.integerClassification", - "version": "1.0.0", - "config_key": "prediction", - "type": "http" - }] - }, - "services": { - "calls": [{ - "config_key": "vnf-db", - "request": { - "format": "dcae.vnf.meta", - "version": "1.0.0" - }, - "response": { - "format": "dcae.vnf.kpi", - "version": "1.0.0" - } - }], - "provides": [{ - "route": "/score-vnf", - "request": { - "format": "dcae.vnf.meta", - "version": "1.0.0" - }, - "response": { - "format": "yourapp.format.integerClassification", - "version": "1.0.0" - } - }] - }, - "parameters": [ - { - "name": "threshold", - "value": 0.75, - "description": "Probability threshold to exceed to be anomalous" - } - ], - "auxilary": { - "healthcheck": { - "type": "http", - "interval": "15s", - "timeout": "1s", - "endpoint": "/my-health" - } - }, - "artifacts": [{ - "uri": "fake.nexus.att.com/dcae/kpi_anomaly:1.0.0", - "type": "docker image" - }] - } diff --git a/docs/sections/components/component-specification/streams-grid.rst b/docs/sections/components/component-specification/streams-grid.rst deleted file mode 100755 index 6105e9e1..00000000 --- a/docs/sections/components/component-specification/streams-grid.rst +++ /dev/null @@ -1,149 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _streams-grid: - -Streams Formatting Quick Reference -================================== - -Each of the following tables represents an example of a publisher and -its subscriber, which are of course, different components. This focuses -on the fields that are ‘different’ for each of these TYPEs, to -illustrate the relationship between ``config_key``, dmaap connection -object, and the generated configuration. Some notes on specific -properties: - -- ``config_key`` is an arbitrary string, chosen by the component - developer. It is returned in the generated configuration where it - contains specific values for the target connection -- ``format``, ``version``, and ``type`` properties in the subscriber - would match these properties in the publisher -- ``aaf_username`` and ``aaf_password`` may be different between the - publisher and the subscriber - -Using http -~~~~~~~~~~ - -*Publishing Component* -^^^^^^^^^^^^^^^^^^^^^^ - -+-----------------------------+----------------------------------------+ -| component \ | runtime platform generated config | -| spec | | -+=============================+========================================+ -| "streams":{   | "streams_publishes":{  | -| "publishes":[{ | "prediction":"10.100.1.100:32567/data" | -| "config_key":"prediction", |  | -| "format":"some-format", | | -| "type":"http", | | -| "version":"0.1.0"   } | | -| ]} | | -+-----------------------------+----------------------------------------+ - -*Subscribing Component* -^^^^^^^^^^^^^^^^^^^^^^^ - -+-----------------------------+----------------------------------------+ -| component | runtime platform generated config | -| spec | | -+=============================+========================================+ -| “streams”:{    | "N/A" | -| "subscribes":[{ | | -| "route":"/data", | | -| "format":"some-format", | | -| "type":"http" | | -| "version":"0.1.0"   } | | -| ]} | | -+-----------------------------+----------------------------------------+ - -Using Message Router -~~~~~~~~~~~~~~~~~~~~ - -.. publishing-component-1: - -*Publishing Component* -^^^^^^^^^^^^^^^^^^^^^^ - -Note: When deploying, this component should be deployed first so satisfy -downstream dependencies. Refer to the –force option in component ‘run’ -command for more information. - -+---------------+------------------------+-----------------------------------------------------------------------------+ -| component \ | Dmaap Connection \ | runtime platform generated \ | -| spec | Object | config | -+===============+========================+=============================================================================+ -| “streams”:{  | {     “dmaap_info”: | “streams_publishes”:{    | -|     “config_k\| {} \ *Note: For \ | “aaf_username”:“pub-user”,   | -| ey”:“mr_out\ | message router, this \ |   “type”:“message_router”,   | -| put”,     “t\ | object is identical \ |      “topic_url”:"https://we-are-message-router.us:3905/events/some-topic"\ | -| ype”:“messag\ | for the publisher and \| "streams_subscribes":{...} | -| e_router”,   | the subscriber* | | -|  }]} | | | -+---------------+------------------------+-----------------------------------------------------------------------------+ - -*Subscribing Component* -^^^^^^^^^^^^^^^^^^^^^^^ - -+---------------+------------------------+-----------------------------------------------------------------------------+ -| component \ | Dmaap Connection \ | runtime platform generated \ | -| spec | Object | config | -+===============+========================+=============================================================================+ -| “streams”:{  | {     “dmaap_info”: | “streams_publishes”:{…}, | -|     “config_k\| {} \ *Note: For \ | “streams_subscribes”:{    | -| ey”:“mr_inp\ | message router, this \ | “aaf_username”:“sub-user”,   | -| ut”,     “ty\ | object is identical \ |   “type”:“message_router”,   | -| pe”:“message\ | for the publisher and \|      “topic_url”:“https://we-are-message-router.us:3905/events/some-topic" | -| _router”,    | the subscriber* | | -| }]} | | | -+---------------+------------------------+-----------------------------------------------------------------------------+ - -Using Data Router -~~~~~~~~~~~~~~~~~ - -.. publishing-component-2: - -*Publishing Component* -^^^^^^^^^^^^^^^^^^^^^^ - -+---------------+-----------------------------------------------+-----------------------------------------------+ -| component spec| Dmaap Connection Object | runtime platform generated config | -+===============+===============================================+===============================================+ -| “streams”:{  | {    “dmaap_info”: { | streams_publishes“:{    ”typ\ | -| “config_key: |      “location”: | e“:”data_router“,       "location":"mtc00" | -| “dr_output" | “mtc00”,      | , | -| , "type": | “publish_url”: | "publish_url“: | -| “data_r\ | "https://we-are-data-router.us/feed/xyz"\ | "http://we-are-data-router.us/feed/xyz" | -| outer”,   }] | , | , | -| } | “log_url”:\ | "log_url“:\ | -| | \ | ”https://we-are-data-router.us/feed/xyz/logs" | -| | "https://we-are-data-router.us/feed/xyz/logs"\| , | -| | , | ”username“:”pub-user“, | -| | “username”: | ”publisher_id“:”123456\ | -| | “pub-user”,      | “}}, | -| | “password”: |  ”streams_subscribes“:{ | -| | “pub-password”,      | … } | -| | “publisher_id”: | | -| | “123456”}} | | -+---------------+-----------------------------------------------+-----------------------------------------------+ - -.. subscribing-component-1: - -*Subscribing Component* -^^^^^^^^^^^^^^^^^^^^^^^ - -+---------------+---------------------------------------------------+---------------------------------------------------------------------------+ -| component \ | Dmaap Connection \ | runtime platform generated \ | -| spec | Object | config | -+===============+===================================================+===========================================================================+ -| “streams”:{  | {      “dmaap_info”: | “streams_publishes”:{ … }, | -|     “config_k\| {      “location”: | “streams_subscribes”:{       | -| ey”:“dr_inp\ | “mtc00”,      | “type”:“data_router”,        | -| ut”,     “ty\ | “delivery_url”: |   “location”:“mtc00”, | -| pe”:“data_ro\ | "https://my-subscriber-app.dcae:8080/target-path"\|          “delivery_url”:"https://my-subscriber-app.dcae:8080/target-path"\| -| uter”, | \ | \ | -|     “route”: | , | , | -| “/target-pat\ |      “password”: | \        | -| h”} | “sub-password”,      | “username”:“sub-user”, | -| | “subscriber_id”: |          | -| | “789012”}} | “subscriber_id”:“789012”}} | -+---------------+---------------------------------------------------+---------------------------------------------------------------------------+ diff --git a/docs/sections/components/component-type-cdap.rst b/docs/sections/components/component-type-cdap.rst deleted file mode 100755 index f2dce6c5..00000000 --- a/docs/sections/components/component-type-cdap.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _cdap-requirements: - -CDAP Component Requirements -=========================== - -This page contains information about CDAP app development in DCAE. - -Uploading your Jar File ------------------------ - -The DCAE component specification has you input your ``jar_url``, the URL -on Nexus to your Jar file. This DCAE controller deploys out of Nexus. -You can upload your jar(s) using the following command, replacing NAME: - -:: - - curl -v --user 'user:password' http://YOUR_NEXUS_RAW_REPO/NAME.jar --upload-file NAME.jar - -During the CLI Tool Usage, in your spec, supply -``http://YOUR_NEXUS_RAW_REPO/NAME.jar`` as the JAR artifact URL. - -Policy Reconfiguration ----------------------- - -We support reconfiguration of both AppConfig and AppPreferences. - -For AppConfig, we support CDAPs “update” API to `reconfigure an application `_. - -For AppPreferences, we: - -1. Stop your programs - -2. Set the new preferences - -3. Start your programs - -At the time of writing, there is no way to update a CDAP application’s -AppConfig or AppPreferences, without a restart, *and notify* the -application. The latter is a future promised feature by CASK—the ability -to update preferences and inform the application that something is -changed (so it repulls). As CDAP currently stands however, given the -above, if you are building a stateful application, you must persist your -state often (e.g., to a CDAP dataset), as you may be restarted at any -time with an updated configuration, or stopped&started at any time with -updated preferences. - -Metrics -------- - -Metrics are pulled from your CDAP application on a periodic basis and -(in the future: pushed to a central DCAE metric store, currently: just -dropped). For this to be useful, your application should provide `metrics `_. -While nothing in the DCAE runtime enforces that your CDAP application -tracks metrics, your metrics (or lack thereof) will be visible in the -DCAE dashboard and to operations. - -.. _dmaap-abstraction: - -Future DMaaP abstraction ------------------------- - -Shown below is our *vision* for how DMaaP is abstracted from component -developers: - -.. figure:: ./images/dmdvision.png - -Today, this is a vision; it is not in place. Today, each CDAP app is -built with built in assumptions about where they are getting their data -from. Some CDAP apps have the built in assumption of a UEB feed. Some -MR. Some DR. This becomes very difficult to orchestrate when each app in -the catalog has built in data assumptions. - -The goal of this vision is to *decouple* the data plane from the -analytics plane. Analytics should be agnostic to *how* they are -receiving their data beyond “filesystem” or “HTTP”. Analytics developers -shouldn’t have to worry about the data plane, that should be taken care -of by the platform. They should be spending their time on the problem at -hand—the analytic. - -This also allows each CDAP application to have a standard set of -interfaces: HTTP and HDFS: - -.. figure:: ./images/io.png - diff --git a/docs/sections/components/component-type-docker.rst b/docs/sections/components/component-type-docker.rst deleted file mode 100755 index 90742453..00000000 --- a/docs/sections/components/component-type-docker.rst +++ /dev/null @@ -1,465 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _docker-requirements: - -Docker Component Requirements -============================= - -Overview --------- - -Component developers are required to provide artifacts for the platform -to be able to deploy your component including: - -- `Component specification ` -- `One or more Data Formats ` \*unless they already exist -- `Docker image <#docker-on-the-platform>`__ - -In addition, components will have to be enhanced to be compliant with -the DCAE platform in order to correctly be deployed and be managed. This -page will discuss the changes which are grouped into the following -categories: - -- `Service Registration <#service-registration>`__ -- `Configuration management on the new platform <#configuration-management>`__ -- `Docker on the Platform <#docker-on-the-platform>`__ -- `Operational concerns <#operational-concerns>`__ - -Additional considerations are: - -- `Policy Reconfiguration <#policy-reconfiguration>`__ - -To help component developers to make and to test the changes needed to -have components run on the platform, a command-line tool called -:doc:`dcae-cli ` is provided by the platform team. -(Testing withing the dcae_cli tool is not yet available for Policy). - -Service Registration --------------------- - -Every :doc:`Docker component is registered <../components/architecture/service-discovery>` with the platform’s -service discovery layer. Docker components are not expected to make the -explicit make registration calls because that is done by through a -platform 3rd party registration service. A couple things are needed from -component developers in order for this registration to occur -successfully: - -1. Docker images must be created from a Dockerfile that has an - `EXPOSE `__ - instruction. This applies to components that listen on a port. -2. Component healthcheck details must be provided in the Docker - auxiliary component specification - -Expose port -~~~~~~~~~~~ - -Components that listen on a specific port must explicitly declare in -their Dockerfile that port using the ``EXPOSE`` instruction before -building the image. Warning! At the current time, you can not expose -multiple ports in your Dockerfile or registration *will not work* -correctly. Warning! Be sure to choose a port that is available. This may -vary by environment. - -Health check -~~~~~~~~~~~~ - -Component developers are required to provide a way for the platform to -periodically check the health of their running components. The platform -uses Consul to perform these periodic calls. Consul provides different -types of `check -definitions `__. The -details of the definition used by your component is to be provided -through the :any:`Docker auxiliary specification `. - -Configuration Management ------------------------- - -All configuration for a component is stored in CONSUL under the -components uniquely generated name which is provided by the environment -variable ``HOSTNAME`` as well as ``SERVICE_NAME``. It is then made -available to the component via a remote HTTP service call to CONFIG -BINDING SERVICE. - -The main entry in CONSUL for the component contains its -``generated application configuration``. This is based on the submitted -component specification, and consists of the ``interfaces`` (streams and -services/calls) and ``parameters`` sections. Other entries may exist as -well, under specific keys, such as :dmaap . Each key represents a -specific type of information and is also available to the component by -calling CONFIG BINDING SERVICE. More on this below. - -Components are required to pull their -``generated application configuration`` at application startup. The -component must provide an initialization script that retrieves the -application configuration and reference that script in its Dockerfile. -Other calls can be made to CONFIG BINDING SERVICE to retrieve DMaaP, -or Policy Reconfiguration (not yet supported). - -You can see more details on the generated application configuration -:any:`here ` - -.. _config_binding_service: - -Config Binding Service -~~~~~~~~~~~~~~~~~~~~~~ - -The config binding service is a platform HTTP service that is -responsible for providing clients with its fully resolve configuration -JSON at startup, and also other configurations objects -when requested. - -At runtime, components should make an HTTP GET on: - -:: - - :/service_component/NAME - -For Docker components, NAME should be set to ``HOSTNAME``, which is -provided as an ENV variable to the container. - -The binding service integrates with the streams and services section of -the component specification. For example, if you specify that you call a -service: - -:: - - "services": { - "calls": [{ - "config_key": "vnf-db", - "request": { - "format": "dcae.vnf.meta", - "version": "1.0.0" - }, - "response": { - "format": "dcae.vnf.kpi", - "version": "1.0.0" - } - }], - ... - } - -Then the config binding service will find all available IP addresses of -services meeting the containers needs, and provide them to the container -under your ``config_key``: - -:: - - // your configuration - { - "vbf-db" : // see above - [IP:Port1, IP:Port2,…] // all of these meet your needs, choose one. - } - -Regarding ``:``, there is DNS -work going on to make this resolvable in a convenient way inside of your -container. However, currently you will be given a name as an ENV -variable, ``CONFIG_BINDING_SERVICE``, and you will need to query -Consul’s service discovery to get -``:``. - -Generated Application Configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The DCAE platform uses the component specification to generate the -component’s application configuration provided at deployment time. The -component developer should expect to use this configuration JSON in the -component. - -Pro-tip: As you build the component specification, use the :any:`dcae-cli dev command ` -to see what the resulting application configuration will look like. - -For both Docker and CDAP, when the component is deployed, any -``streams`` and ``services/calls`` specified, will be injected into the -configuration under the following well known structure, along with all -``parameters``. (``services/provides`` is not passed in to the -application config). Your component is required to parse this -information if it has any DMaaP connections or interfaces with another -DCAE component. - -This is best served by an example. - -The following component spec snippet (from String Matching): - -:: - - "streams":{ - "subscribes": [{ - "format": "VES_specification", - "version": "4.27.2", - "type": "message_router", - "config_key" : "mr_input" - }], - "publishes": [{ - "format": "VES_specification", - "version": "4.27.2", - "config_key": "mr_output", - "type": "message_router" - }] - }, - "services":{ - "calls": [{ - "config_key" : "aai_broker_handle", - "verb": "GET", - "request": { - "format": "get_with_query_params", - "version": "1.0.0" - }, - "response": { - "format": "aai_broker_response", - "version": "3.0.0" - } - }], - "provides": [] - }, - -Will result in the following top level keys in the configuration (for -CDAP, this will be under AppConfig) - -:: - - "streams_publishes":{ - "mr_output":{ // notice the config key above - "aaf_password":"XXX", - "type":"message_router", - "dmaap_info":{ - "client_role": null, - "client_id": null, - "location": null, - "topic_url":"https://YOUR_HOST:3905/events/com.att.dcae.dmaap.FTL2.DCAE-CL-EVENT" // just an example - }, - "aaf_username":"XXX" - } - }, - "streams_subscribes":{ - "mr_input":{ // notice the config key above - "aaf_password":"XXX", - "type":"message_router", - "dmaap_info":{ - "client_role": null, - "client_id": null, - "location": null, - "topic_url":"https://YOUR_HOST:3905/events/com.att.dcae.dmaap.FTL2.TerrysStringMatchingTest" // just an example - }, - "aaf_username":"XXX" - } - }, - "services_calls":{ - "aai_broker_handle":[ // notice the config key above - "135.205.226.128:32768" // based on deployment time, just an example - ] - } - -These keys will always be populated whether they are empty or not. So -the minimum configuration you will get, (in the case of a component that -provides an HTTP service, doesn’t call any services, and has no streams, -is: - -:: - - "streams_publishes":{}, - "streams_subscribes":{}, - "services_calls":{} - -Thus your component should expect these well-known top level keys. - - -DMaaP -~~~~~ - -Components can be publishers or subscribers to either message router -topics or data router feeds. This is defined in the component -specification under the ``streams`` section where you can specify -whether your component is expected to subscribe or to publish to a -:any:`message router ` -topic or to a :any:`data router ` -feed. Given a composition with components that use DMaaP, the platform -will provision the topic or feed and provide the necessary :doc:`connection -details <./component-specification/dmaap-connection-objects>` -at runtime for each DMaaP dependent component. These connection details -are provided through your application’s generated configuration. - -In order to test DMaaP connections in onboarding, the developer -(currently) must provision all test topics and feeds manually and -provide the :any:`dcae-cli with the connection details ` when deploying your -application. - -Even thought the DMaaP connection information is included in the -generated application configuration, it may be obtained by doing a call -as in this example: - -:: - - curl http://:/dmaap/jm416b.d345ada1-cc31-4121-a741-9007b9f64808.1-0-1.dcae-collectors-cli-pm - -This would return the following: - -:: - - {"cli_gamma_cisco_pm_config_stat": - { - "publish_url": "https://YOUR_HOST/publish/1362", - "username": "mtl5-0", - "log_url": null, - "location": "mtl5-0", - "password": "i5qji048hdm2e38f0bg872tnqd", - "publisher_id": "1234" - } - } - -Policy Reconfiguration -~~~~~~~~~~~~~~~~~~~~~~ - -*(not yet supported)* - -Components must provide a way to receive policy reconfiguration, that -is, configuration parameters that have been updated via the Policy UI. -The component developer provides a docker script (defined in the :any:`Docker -auxiliary specification `) -that will be triggered when this occurs. - -Docker on the platform ----------------------- - -Images -~~~~~~ - -Docker images must be pushed to the environment specific Nexus -repository. This requires tagging your build with the full name of you -image which includes the Nexus repository name. - -Use the Docker command-line to -`tag `__ your -Docker image where the *target image* must contain the registry host -name and port. - -For example, an application called yourapp has been tagged for an example -Nexus registry: - -:: - - YOUR_NEXUS_DOCKER_REGISTRY/yourapp 0.4.0 154cc382df61 7 weeks ago 710.5 MB - yourapp 0.4.0 154cc382df61 7 weeks ago 710.5 MB - -The solutioning environment’s Nexus host for the Docker registry is -``YOUR_NEXSUS_HOST:18443``. You must run -``docker login YOUR_NEXSUS_HOST:18443`` to access the registry. -Please contact the DCAE platform team to provide you with the -credentials. - -:: - - docker login YOUR_NEXSUS_HOST:18443 - -Tag your image: - -:: - - docker tag yourapp:0.4.0 YOUR_NEXSUS_HOST:18443/dcae-platform/yourapp:0.4.0 - -Or build and tag: - -:: - - docker build -t YOUR_NEXSUS_HOST:18443/dcae-platform/yourapp:0.4.0 . - -After tagging, upload your image to the remote registry using the Docker -`push -command `__. -Note that the registry may require a login. Use the Docker `login -command `__ -before pushing in that case. - -:: - - docker push YOUR_NEXSUS_HOST:18443/dcae-platform/yourapp:0.4.0 - -*NOTE* Replace ``dcae-platform`` with the group directory that is -applicable to your image. Replace ``yourapp`` with your application’s -name. Replace the ``0.4.0`` version with your application’s version. - -Dockerfile -~~~~~~~~~~ - -The Dockerfile must contain the name of the container’s initialization -script. This will be called when the container is deployed, and must -call Config Binding Service as described in `Config Binding -Service <#config-binding-service>`__ - -.. _dcae-cli-docker-ports: - -Ports -~~~~~ - -On the DCAE platform, Docker components are run with the -``--publish-all`` or ``-P`` argument. This means the Docker container -for your component will be listening on a random port and that random -port will be mapped to the port `you exposed <#service-registration>`__. - -Envs -~~~~ - -The platform provides a set of environment variables into each Docker -container: - -+----------------------------+--------------+----------------------------------------+ -| Name | Type | Description | -+============================+==============+========================================+ -| ``HOSTNAME`` | string | Unique name of the component instance | -| | | that is generated | -+----------------------------+--------------+----------------------------------------+ -| ``CONSUL_HOST`` | string | Hostname of the platform's Consul | -| | | instance | -+----------------------------+--------------+----------------------------------------+ -| ``CONFIG_BINDING_SERVICE`` | string | Hostname of the platform's config | -| | | binding service instance | -| | | | -+----------------------------+--------------+----------------------------------------+ -| ``DOCKER_HOST`` | string | Host of the target platform Docker | -| | | host to run the container on | -+----------------------------+--------------+----------------------------------------+ - -Operational Concerns --------------------- - -Logging -~~~~~~~ - -Currently the platform uses the default ``json-file`` logging driver for -Docker. For onboarding testing, component developers can access their -logs from their Docker containers either by running their component -using the ``--attached`` flag or by using the ``docker logs`` command. -The requirement is that applications must write to stdout and/or stderr. - -To use the ``docker logs`` command for your deployed running Docker -container, - -- You must have Docker installed on your local machine -- Have the generated name of your component. This is generated for you - when you execute ``dcae_cli component dev`` or - ``dcae_cli component run``. -- Find the target Docker host using the ``dcae_cli profiles show`` - command: - -:: - - $ dcae_cli profiles show solutioning - { - "cdap_broker": "cdap_broker", - "config_binding_service": "config_binding_service", - "consul_host": "YOUR_CONSUL_HOST", - "docker_host": "YOUR_DOCKER_HOST:2376" - } - -- Set your Docker client to point to the target Docker host: - -:: - - $ export DOCKER_HOST="tcp://YOUR_DOCKER_HOST:2376" - -- Use the ``docker logs`` command: - -:: - - $ docker logs - diff --git a/docs/sections/components/data-formats.rst b/docs/sections/components/data-formats.rst deleted file mode 100755 index bf2ffbf2..00000000 --- a/docs/sections/components/data-formats.rst +++ /dev/null @@ -1,238 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _data-formats: - - -Data Formats -============ - -Data formats are descriptions of data; they are the data contract -between your component and other components. When the components are -‘composed’ into services in the SDC tool, they can only be matched with -components that have compatible data formats. Data formats will be -onboarded to SDC and assigned a UUID at that time. This UUID is then -used to ensure compatibility amoung components. (If component X outputs -data format ‘DF-Y’, and another component Z specifies ‘DF-Y’ as its -input data format, then X is said to be ``composable`` with component -Z). - -Since data formats will be shared across components, the onboarding -catalog should be checked first to see if the desired data format is -available before creating one. The vision is to have a repository of -shared data formats that developers and teams can re-use and also -provide them the means to extend and create new custom data formats. A -data format is referenced by its data format id and version number. - -JSON schema ------------ - - The data format specification is represented (and validated) against - this `Data Format json schema `__ - and described below: - -Meta Schema Definition -~~~~~~~~~~~~~~~~~~~~~~ - -The “Meta Schema” implementation defines how data format JSON schemas -can be written to define user input. It is itself a JSON schema (thus it -is a “meta schema”). It requires the name of the data format entry, the -data format entry version and allows a description under “self” object. -The meta schema version must be specified as the value of the -“dataformatversion” key. Then the input schema itself is described as -one of the four types listed below: - -+------------------+---------------------------------------------------+ -| Type | Description | -+==================+===================================================+ -| jsonschema | inline standard JSON Schema definitions of JSON | -| | inputs | -+------------------+---------------------------------------------------+ -| delimitedschema | delimited data input using a JSON description and | -| | defined delimiter | -+------------------+---------------------------------------------------+ -| unstructured | unstructured text, and reference that allows a | -| | pointer to another artifact for a schema. | -+------------------+---------------------------------------------------+ -| reference | allows for XML and Protocol Buffers schema, | -| | but can be used to reference other JSON, | -| | delimitedschema and unstructured schemas as well. | -+------------------+---------------------------------------------------+ - - -Example Schemas ---------------- - -By reference example - Common Event Format -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -First the full JSON schema description of the Common Event Format would -be loaded with a name of “Common Event Format” and the current version -of “25.0.0”. - -Then the data format description is loaded by this schema: - -:: - - { - "self": { - "name": "Common Event Format Definition", - "version": "25.0.0", - "description": "Common Event Format Definition" - - }, - "dataformatversion": "1.0.0", - "reference": { - "name": "Common Event Format", - "format": "JSON", - "version": "25.0.0" - } - } - - - -Simple JSON Example -~~~~~~~~~~~~~~~~~~~~~~~~ - - -:: - - { - "self": { - "name": "Simple JSON Example", - "version": "1.0.0", - "description": "An example of unnested JSON schema for Input and output" - - }, - "dataformatversion": "1.0.0", - "jsonschema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "raw-text": { - "type": "string" - } - }, - "required": ["raw-text"], - "additionalProperties": false - } - } - -Nested JSON Example -~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - { - "self": { - "name": "Nested JSON Example", - "version": "1.0.0", - "description": "An example of nested JSON schema for Input and output" - - }, - "dataformatversion": "1.0.0", - "jsonschema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "properties": { - "numFound": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "engagements": { - "type": "array", - "items": { - "properties": { - "engagementID": { - "type": "string", - "transcript": { - "type": "array", - "items": { - "type": { - "type": "string" - }, - "content": { - "type": "string" - }, - "senderName": { - "type": "string" - }, - "iso": { - "type": "string" - }, - "timestamp": { - "type": "integer" - }, - "senderId": { - "type": "string" - } - } - } - } - } - } - } - }, - "additionalProperties": false - } - } - -Unstructured Example -~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - { - "self": { - "name": "Unstructured Text Example", - "version": "25.0.0", - "description": "An example of a unstructured text used for both input and output for " - - }, - "dataformatversion": "1.0.0", - "unstructured": { - "encoding": "UTF-8" - } - } - - -An example of a delimited schema --------------------------------- - -:: - - { - "self": { - "name": "Delimited Format Example", - "version": "1.0.0", - "description": "Delimited format example just for testing" - - }, - "dataformatversion": "1.0.0", - "delimitedschema": { - "delimiter": "|", - "fields": [{ - "name": "field1", - "description": "test field1", - "fieldtype": "string" - }, { - "name": "field2", - "description": "test field2", - "fieldtype": "boolean" - }] - } - } - -Note: The referenced data format (in this case, a schema named “Common -Event Format” with version of “25.0.0”) must already exist in the -onboarding catalog. - -Working with Data Formats -------------------------- - -Data Formats can be added to the onboarding catalog (which first -validates them) by using the :doc:`dcae_cli Tool `. -Here you can also list all of your data formats, show the contents of a -data format, publish your data format, and even generate a data format -from a input sample file. For a list of these capabilities, see :any:`Data Format Commands `. diff --git a/docs/sections/components/dcae-cli/commands.rst b/docs/sections/components/dcae-cli/commands.rst deleted file mode 100644 index d63fb09c..00000000 --- a/docs/sections/components/dcae-cli/commands.rst +++ /dev/null @@ -1,590 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _dcae_cli_commands: - -dcae_cli Commands -================= - -The dcae_cli tool has four command groups. Each has several -sub-commands. - -``catalog`` ------------ - -The ``catalog`` command lists and shows resources (components and -data_formats) in the ‘onboarding’ catalog (regardless of the owner). A -resource can have a status of ``staged`` or ``published``. By default, -only ``published`` resources are displayed. To see ``staged`` resources, -add the –expanded argument. - -+------------------------------+----------------------------------------+ -| Catalog Status | Meaning | -+==============================+========================================+ -| staged | resource has be added | -| | (and validated), but | -| | is under development | -+------------------------------+----------------------------------------+ -| staged | data_formats can only be referenced in | -| | their owners component specs | -+------------------------------+----------------------------------------+ -| staged | components can only be deployed by | -| | their owners | -+------------------------------+----------------------------------------+ -| published | resource has been | -| | tested and can be | -| | shared | -+------------------------------+----------------------------------------+ -| | published data_formats can be used in | -| | anyone's component spec | -+------------------------------+----------------------------------------+ -| | published components and be deployed by| -| | anyone | -+------------------------------+----------------------------------------+ - -:: - - $ dcae_cli catalog --help - Usage: dcae_cli catalog [OPTIONS] COMMAND [ARGS]... - - Options: - --help Show this message and exit. - - Commands: - list Lists resources in the onboarding catalog - show Provides more information about resource - -List onboarding catalog contents -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - $ dcae_cli catalog list - Components: - +--------------------------------+---------+--------+---------------------------------------------------------------------+--------+-----------+------------+ - | Name | Version | Type | Description | Owner | Status | Published | - +--------------------------------+---------+--------+---------------------------------------------------------------------+--------+-----------+------------+ - | DcaeSyslogCollector | 2.0.0 | docker | DCAE Control Plane Syslog Collector | sh1986 | published | 2017-08-04 | - | cdap.dmaap.spec.example | 0.2.0 | cdap | dmaap spec example. Not a functioning application, only for showing | tc677g | published | 2017-07-24 | - | | | | how to pub/sub dmaap. Pretend this is like MAP with VES in and ou.. | | | | - | cdap.event.proc.enrich.app | 1.0.3 | cdap | CDAP Event Processing Enrich application | an4828 | published | 2017-09-20 | - | cdap.event.proc.map.app | 1.0.3 | cdap | CDAP Event Processing Map application | an4828 | published | 2017-09-20 | - - ... - - Data formats: - +--------------------------------------------+---------+-----------------------------------------------------------------------+--------+-----------+------------+ - | Name | Version | Description | Owner | Status | Published | - +--------------------------------------------+---------+-----------------------------------------------------------------------+--------+-----------+------------+ - | FOI_PM_VHSS_data_format | 1.0.0 | CSV pipe delimited data format for VHSS PM files | sr229c | published | 2017-09-05 | - | Map_input | 1.0.0 | The input format for Mapper, that in 1707 is the UCSNMP Collector | an4828 | published | 2017-07-18 | - | | | output format, but will support more formats later | | | | - | Syslog Collector Parsed Json Message | 1.0.0 | Post processed/parsed collected syslog message | sh1986 | published | 2017-08-04 | - | Syslog Collector Syslog Message Input | 1.0.0 | The input message for the DCAE syslog collector is free/unstructured | sh1986 | published | 2017-08-04 | - | | | text | | | | - | myapp Alert Definition | 1.0.0 | The format of the output event from myapp | an4828 | published | 2017-08-10 | - | VES_specification | 5.28.4 | VES spec for 5.4 | vv770d | published | 2017-09-19 | - - ... - -Show the contents of an item in the catalog -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - $ dcae_cli catalog show FOI_PM_VHSS_data_format - - Data format - ----------- - { - "dataformatversion": "1.0.0", - "delimitedschema": { - "delimiter": "|", - "fields": [ - { - "description": "System ID", - "fieldtype": "string", - "name": "SYSTEM" - }, - { - "description": "Date", - "fieldtype": "string", - "name": "DATE" - }, - { - "description": "Time", - "fieldtype": "string", - "name": "TIME" - }, - - ... - -.. _dcae_cli_component_commands: - --------------- - -``component`` -------------- - -The ``component`` command is for validating (adding), listing, showing, -verifying generated configuration, running, undeploying, and publishing -components that YOU own. - -:: - - $ dcae_cli component --help - Usage: dcae_cli component [OPTIONS] COMMAND [ARGS]... - - Options: - --help Show this message and exit. - - Commands: - add - dev Set up component in development for... - list Lists components in the onboarding catalog. - publish Pushes COMPONENT to the onboarding catalog - run Runs the latest version of COMPONENT. - show Provides more information about COMPONENT - undeploy Undeploys the latest version of COMPONENT. - --------------- - -.. _dcae_cli_add_a_component: - -Add a Component -~~~~~~~~~~~~~~~ - -A component must be added to the onboarding catalog in order to be -tested by the dcae_cli tool. The process of adding a component also -validates it’s component specification. In order to add a component, the -component docker image (or CDAP jar) must exist locally. - -Components in the onboarding catalog can be run by others, once they are -``published.`` ``Published`` components cannot be modified or deleted. -Rather a new version can be created instead. - -Validated component specs are used later to generate Tosca models and -Cloudify Blueprints for the component, which makes them available for -use in the SDC Tool for creating services. - -:: - - $ dcae_cli component add --help - Usage: dcae_cli component add [OPTIONS] COMPONENT-SPECIFICATION - - Options: - --update Updates a previously added component if it has not been already - published - --help Show this message and exit. - -:: - - $ dcae_cli component add component-spec.json - --------------- - -List Components -~~~~~~~~~~~~~~~ - -List components in the onboarding catalog that owned by YOUR userid.. - -:: - - $ dcae_cli component list - Active profile: solutioning - +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - | Name | Version | Type | Description | Status | Modified | #Deployed | - +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - | cdap.helloworld.endnode | 0.8.0 | cdap | cdap test component | staged | 2017-05-23 04:14:35.588075 | 0 | - | sandbox.platform.yourapp| 0.5.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-05-23 04:07:44.065610 | 0 | - +-------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - -The fields ``Name``, ``Version``, ``Type``, ``Description`` are -referenced from the component specification’s ``self`` JSON. Use the -“–deployed” argument to see more details on deployed components - --------------- - -.. _dcae_cli_run_a_component: - -Run a Component -~~~~~~~~~~~~~~~ - -The ``run`` operation is to be used for running your application in its -container remotely on the activated environment. Docker containers have -the additional option to run locally on your development machine. If the -component uses Dmaap, you can specify the Dmaap Connection Object as -well. Refer to :doc:`Dmaap Connection Object <../component-specification/dmaap-connection-objects>`. - -When you run a component via the dcae_cli Tool, remember the blueprint -has not been created and is not used for deployment. - -In order to run the component, the data formats and component must have -been added to the onboarding catalog. - -**DOCKER NOTE:** Make sure the Docker image has been uploaded to the -shared registry. - -A docker component can be run in either ``attached`` or ``unattached`` -mode. (Default is unattached). - -+------------------+---------------------------------------------------+ -| Mode | Description | -+==================+===================================================+ -| attached | component is run in the ‘foreground’, container | -| | logs are streamed to stdout. Ctrl-C is used to | -| | terminate the dcae_cli session. | -+------------------+---------------------------------------------------+ -| unattached | component is run in the ‘background’, container | -| | logs are viewed via ``docker logs`` command, | -| | container runs until undeployed with dcae_cli | -| | ``undeploy`` command. | -+------------------+---------------------------------------------------+ - -Run a component in attached mode: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - $ dcae_cli -v component run --attached sandbox.platform.yourapp:0.5.0 - - DCAE.Docker | INFO | Running image 'registry.proto.server.com/dcae-rework/yourapp:0.4.0' as 'user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp' - DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | Consul host: yourconsulhost.com - - DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | service name: mh677g.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp - - DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | Generated config: {'multiplier': 3} - - DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | * Running on http://0.0.0.0:8080/ (Press CTRL+C to quit) - - DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | 127.0.0.1 - - [24/May/2017 03:37:57] "GET /health HTTP/1.1" 200 - - - DCAE.Docker.user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp | INFO | 127.0.0.1 - - [24/May/2017 03:38:12] "GET /health HTTP/1.1" 200 - - -Hit Ctrl-C to terminate session. - -:: - - ^CDCAE.Docker | INFO | Stopping container 'user1.b7287639-37f5-4f25-8d54-8a2087f4c8da.0-5-0.sandbox-platform-yourapp' and cleaning up... - -Run a component in unattached mode: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - $ dcae_cli -v component run sandbox.platform.yourapp:0.5.0 - DCAE.Docker | INFO | Running image 'registry.proto.server.com/dcae-rework/yourapp:0.4.0' as 'user1.4811da0e-08d5-429f-93bf-bf6814924577.0-5-0.sandbox-platform-yourapp' - DCAE.Run | INFO | Deployed /user1.4811da0e-08d5-429f-93bf-bf6814924577.0-5-0.sandbox-platform-yourapp - -**NOTE** You must undeploy this component when finished testing. This is -important to conserve resources in the environment. - -Run a component that subscribes to Dmaap MR or DR -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - $ dcae_cli -v component run --attached --dmaap-file $dmaap-connection-file sandbox.platform.yourapp:0.5.0 - --------------- - -Undeploy a Component -~~~~~~~~~~~~~~~~~~~~ - -The ``undeploy`` command is used to undeploy any instance of a specified -component/version that you have deployed. This includes cleaning up the -configuration. - -:: - - $ dcae_cli component undeploy sandbox.platform.yourapp:0.5.0 - DCAE.Undeploy | WARNING | Undeploying components: 1 - DCAE.Undeploy | WARNING | Undeployed components: 1 - --------------- - -Publish a component -~~~~~~~~~~~~~~~~~~~ - -| Once a component has been tested, it should be published in the - onboarding catalog using the ``publish`` sub-command . -| Publishing will change the status of the component (from ``staged`` to - ``published``), indicating that it has been tested, and making it - accessible for other developers to use. - -**Note** Before a component can be published, all data_formats that it -references must be published. - -:: - - dcae_cli component publish sandbox.platform.yourapp:0.5.0 - --------------- - -Show a Component -~~~~~~~~~~~~~~~~ - -This will print out the contents of a component and is useful to copy a -component spec. - -:: - - $ dcae_cli component show - --------------- - -.. _dcae-cli-run-the-dev-command: - -Run the ``dev`` command -~~~~~~~~~~~~~~~~~~~~~~~ - -The ``dev`` command is used as part of a process to see the platform -generated configuration. It established the environment variables and is -best explained -:any:`here `. - -:: - - $ dcae_cli component dev component-spec.json - Ready for component development - --------------- - -.. _dcae_cli_data_format: - -``data_format`` ---------------- - -The ``data_format`` command is for validating (adding), listing, -showing, publishing data_formats that YOU own. data_formats can also be -generated with this command. - -:: - - $ dcae_cli data_format --help - Usage: dcae_cli data_format [OPTIONS] COMMAND [ARGS]... - - Options: - --help Show this message and exit. - - Commands: - add Tracks a data format file DATA_FORMAT-SPECIFICATION... - generate Create schema from a file or directory... - list Lists all your data formats - publish Publishes data format to make available to others... - show Provides more information about FORMAT - --------------- - -.. _dcae_cli_add_a_data_format: - -Add a Data Format -~~~~~~~~~~~~~~~~~ - -A data_format must be in the onboarding catalog in order to be -referenced in the component specification. The process of adding a -data_format also validates it. Data_formats in the onboarding catalog -can be run by others, once they are ``published.`` ``Published`` -data_formats cannot be modified or deleted. Rather a new version can be -created instead. - -:: - - $ dcae_cli data_format add --help - Usage: dcae_cli data_format add [OPTIONS] DATA_FORMAT-SPECIFICATION - - Options: - --update Updates a previously added data_format if it has not been already - published - --help Show this message and exit. - -:: - - dcae_cli data_format add health.json - --------------- - -List Data Formats -~~~~~~~~~~~~~~~~~ - -Only data_formats owned by YOUR userid will be shown. - -:: - - $ dcae_cli data_format list - - Data formats for user1 - +---------------------------------+---------+-------------------------------------------+--------+----------------------------+ - | Name | Version | Description | Status | Modified | - +---------------------------------+---------+-------------------------------------------+--------+----------------------------+ - | sandbox.platform.yourapp.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-05-23 04:02:38.952799 | - +---------------------------------+---------+-------------------------------------------+--------+----------------------------+ - -The fields ``name``, ``version``, ``description`` are referenced from -the data format specification’s ``self`` JSON. ``Status`` represents the -status of the data format in the catalog. See `Publish a Data -Format <#publish-a-data-format>`__ for more info. - --------------- - -Show a Data Format -~~~~~~~~~~~~~~~~~~ - -This will print out the contents of a data_format and is useful for -copying a data_format. - -:: - - $ dcae_cli data_format show - --------------- - -Publish a Data Format -~~~~~~~~~~~~~~~~~~~~~ - -| Once a data_format has been tested (by referencing it in a component - spec that has been tested), it should be published in the onboarding - catalog using the ``publish`` sub-command . -| Publishing will change the status of the data_format (from ``staged`` - to ``published``), indicating that it has been tested, and making it - accessible for other developers to use. - -:: - - $ dcae_cli data_format publish data_format.json - --------------- - -Generate a Data Format -~~~~~~~~~~~~~~~~~~~~~~ - -If you already have a valid input or output file, you can use the -generate command to create the it’s data format specification. - -:: - - $ dcae_cli data_format generate name:version file-or-dir-path - --------------- - -``profiles`` ------------- - -The\ ``profiles`` command is for creating, deleting, listing, showing, -activating, and updating (set) profiles. The profile contains -environment variables used to connect to different environments. This is -used in the running and deployment of a component using the -``dcae_cli component run`` or ``dev`` command. - -:: - - $ dcae_cli profiles --help - Usage: dcae_cli profiles [OPTIONS] COMMAND [ARGS]... - - Options: - --help Show this message and exit. - - Commands: - activate Sets profile NAME as the active profile - create Creates a new profile NAME initialized with... - delete Deletes profile NAME - list Lists available profiles - set Updates profile NAME such that KEY=VALUE - show Prints the profile dictionary - --------------- - -List the available profiles -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - $ dcae_cli profiles list - * solutioning - 1710 - 1802 - -The \* identifies the active profile. ``dcae-cli`` is currently -installed with profiles for the ``solutioning``, ``1710``, and ``1802`` -environments. They are intended for the following: - -+-----------------------------------+-----------------------------------+ -| Environment | Description | -+===================================+===================================+ -| solutioning | default environment; used for | -| | initial component developer | -| | testing with the dcae_cli tool. | -+-----------------------------------+-----------------------------------+ -| 1710 | FTL3 (Functional Testing Lab 3) | -| | environment, which represents the | -| | 1710 release. | -+-----------------------------------+-----------------------------------+ -| 1802 | FTL3a (Functional Testing Lab 3a) | -| | environment, which represents the | -| | 1802 release. | -+-----------------------------------+-----------------------------------+ - --------------- - -Show the details of a profile -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - $ dcae_cli profiles show solutioning - { - "cdap_broker": "cdap_broker", - "config_binding_service": "config_binding_service", - "consul_host": "yourconsulhost.com", - "docker_host": "yourdockerhost.com:2376" - } - --------------- - -.. _dcae_cli_activate_profile: - -Activate a profile -~~~~~~~~~~~~~~~~~~ - -To switch among profiles, use the activate sub-command. A subsequent -``list`` will reveal the change made. - -:: - - $ dcae_cli profiles activate test - --------------- - -Create a new profile -~~~~~~~~~~~~~~~~~~~~ - -If you want to work in a different environment using the dcae_cli tool, -you can make your own profile. (The environment must be a working DCAE -Platform environment). - -:: - - $ dcae_cli profiles create new-profile - -After creating you would assign the variables with the ``set`` -sub-command. Then activate it to use. - --------------- - -Set variables in a profile -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - $ dcae_cli profiles set $profile $key $value - --------------- - -Delete a profile -~~~~~~~~~~~~~~~~ - -:: - - $ dcae_cli profiles delete new-profile - - diff --git a/docs/sections/components/dcae-cli/dcae-cli-tool.rst b/docs/sections/components/dcae-cli/dcae-cli-tool.rst deleted file mode 100644 index 477fc341..00000000 --- a/docs/sections/components/dcae-cli/dcae-cli-tool.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -dcae-cli Tool -============= - -.. toctree:: - :maxdepth: 1 - - ./quickstart.rst - ./commands.rst - ./walkthrough.rst - diff --git a/docs/sections/components/dcae-cli/quickstart.rst b/docs/sections/components/dcae-cli/quickstart.rst deleted file mode 100755 index 47b9ac1b..00000000 --- a/docs/sections/components/dcae-cli/quickstart.rst +++ /dev/null @@ -1,156 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _quickstart: - -Overview -======== - -The ``dcae-cli`` is a Python command-line tool for component developers. -With it, the developer can : - -- validate the data formats and component specifications -- publish the validated data formats and component specifications into - the ``onboarding catalog`` -- access the ``onboarding catalog`` to search for existing data formats - (for possible reuse) and component specs -- deploy a component onto a local or remote DCAE platform for - functional and pair-wise testing (This is done without Cloudify) - -The git repository for the dcae_cli tool can be found -`here `__ - -Pre-requisites --------------- - -For Docker -~~~~~~~~~~ - -There are two options for development with Docker: - -For local development -^^^^^^^^^^^^^^^^^^^^^ - -- Install `Docker engine `__ locally on - your machine. -- Know the *external ip* of where the Docker engine is running. The - external ip is needed so that service discovery will connect to it. - - - *(For OSX users, this means making sure the VirtualBox VM that is - running your Docker engine has a bridged adapter and getting the - ip of that adapter).* - -For remote development -^^^^^^^^^^^^^^^^^^^^^^ - -- Have access to a remote host with Docker engine installed and with - remote API access. -- Have the associated connection information: - - - domain name or IP and port (port should be either 2375 or 2376). - Use this information to establish an active - :any:`profile `. - -For CDAP -~~~~~~~~ - -None at this time. - -Python, Pip, Virtualenv -~~~~~~~~~~~~~~~~~~~~~~~ - -Install python, pip (9.0.1 or higher), and virtualenv if they are not -installed. Do these when not in a VPN to avoid possible network issues. - -:: - - sudo apt-get -f install python - sudo apt-get -f install python-pip - sudo pip install virtualenv - -Set up a virtual environment and activate - -:: - - virtualenv cli_tool - source cli_tool/biin/activate - -Install dcae_cli ----------------- - -:: - - pip install onap-dcae-cli - -Check dcae_cli version ----------------------- - -You can verify the version of the dcae-cli with the following command. -To get the latest version of the dcae_cli tool, - -:: - - $ dcae_cli --version - -Upgrade dcae_cli ----------------- - -Periodically, upgrade the dcae_cli to get the latest version - -:: - - pip install --upgrade onap-dcae-cli - -Configuration -------------- - -When running the tool for the first time, a `configuration -directory `__ and -configuration file will be created. - -The configuration is first sourced from a remote server that is managed -by the platform team. You will be prompted to enter your ATTUID to -complete this process. - -Re-initializing Configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Configuration can be re-initialized or reset. There is a ``--reinit`` -flag that is to be used to re-initialize your configuration and your -environment profiles. You may be instructed to re-initialize after -certain updates are made to the dcae_cli tool. When you re-initialize -the configuration, your configuration will be added to or updated from -the platform configuration and profiles. No profiles will be deleted via -the reinit process. - -To re-initialize: - -:: - - $ dcae_cli --reinit - -Verify Installation -------------------- - -To Verify that the dcae_cli tool is installed, run the following command -and look for the output below. - -:: - - $ dcae_cli --help - Usage: dcae_cli [OPTIONS] COMMAND [ARGS]... - - Options: - -v, --verbose Prints INFO-level logs to screen. - --reinit Re-initialize dcae-cli configuration - --version Show the version and exit. - --help Show this message and exit. - - Commands: - catalog - component - data_format - profiles - -Refer to :doc:`dcae_cli Commands <./commands>`. - diff --git a/docs/sections/components/dcae-cli/walkthrough.rst b/docs/sections/components/dcae-cli/walkthrough.rst deleted file mode 100755 index fd1efb93..00000000 --- a/docs/sections/components/dcae-cli/walkthrough.rst +++ /dev/null @@ -1,385 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _walkthrough: - -Walk-through -============ - -This section demonstrates the flow and usage of the dcae_cli tool to -onboard a typical component to the DCAE platform. The commands are -explained in more detail in :doc:`dcae_cli Commands `. - -- `Add (and validate) a data format <#add-a-data-format>`__ -- `Add (and validate) the component <#add-the-component>`__ -- `View the platform generated - configuration <#view-the-platform-generated-configuration>`__ -- `If needed, Create the dmaap file for Dmaap Testing <#create-the-input-file-for-dmaap-testing>`__ -- `If needed, Create the input file for *Sourced at Deployment* Testing <#create-the-input-file-for-sourced-at-deployment-testing>`__ -- `Run the component <#run-the-component>`__ -- :any:`Undeploy the component ` -- :any:`Publish the component and data_format ` to let others - know its ready for reuse -- `List the Catalog Contents <#list-the-catalog-contents>`__ to see - your published resources - - --------------- - -Add a Data Format ------------------ - -:: - - $ dcae_cli data_format add $HOME/yourapp/data-formats/health.json - -Verify that the data_format was added - -:: - - $ dcae_cli data_format list | grep yourapp - | sandbox.platform.yourapp.health | 0.1.0 | Data format used for the /health endpoint | staged | 2017-11-07 21:48:47.736518 | - -(Note: Each of the data formats for your component need to be added, -unless already existing in the onboarding catalog ) - --------------- - -Add the Component ------------------ - -:: - - $ dcae_cli component add $HOME/yourapp/component-spec.json - -Verify that the component was added - -:: - - $ dcae_cli component list - Active profile: solutioning - - +-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - | Name | Version | Type | Description | Status | Modified | #Deployed | - +-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - | sandbox.platform.yourapp | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-11-08 20:27:34.168854 | 0 | - +-------------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - --------------- - -.. _dcae-cli-view-the-platform: - -View the platform generated configuration ------------------------------------------ - -The ``component dev`` command is useful during onboarding. Running this -command is part of a multi-step process that sets up a temporary test -environment, generates your application configuration, makes it -available in that environment, and allows you to view that configuration -to help with debugging. - -Here is a step-by-step example based on a component specification called -``component-spec.json``. - -Step 1 - Run the component dev command -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -(This creates a file called env_$ENV (in the current directory)- where -$ENV is the name of the active profile. Note: SERVICE_NAME and HOSTNAME -always resolve to the same value). - -:: - - $ dcae_cli component dev component-spec.json - Ready for component development - - Setup these environment variables. Run "source env_solutioning": - - export DOCKER_HOST=yourdockerhost.com:2376 - export SERVICE_CHECK_INTERVAL=15s - export CONFIG_BINDING_SERVICE=config_binding_service - export HOSTNAME=user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp - export CONSUL_HOST=yourconsulhost.com - export CDAP_BROKER=cdap_broker - export SERVICE_NAME=user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp - export SERVICE_CHECK_TIMEOUT=1s - export SERVICE_CHECK_HTTP=/health - - Press any key to stop and to clean up - -Step 2 - Setup the environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In another window, setup the temporary testing environment, by executing -the environment file created above. - -:: - - $ source env_solutioning - -(The application configuration is now available under the SERVICE_NAME -shown above - -``user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp``). - -Step 3 - Query CONSUL -~~~~~~~~~~~~~~~~~~~~~ - -Query CONSUL to get the IP/PORT of CONFIG BINDING SERVICE - -:: - - $ curl http://$CONSUL_HOST:8500/v1/catalog/service/$CONFIG_BINDING_SERVICE - [ - { - "ID": "bfbc220d-4603-7f90-ec2e-611d3c330f20", - "Node":"docker00", - "Address": "127.0.0.1", - "Datacenter":"dcae-central", - "TaggedAddresses": { - "lan":"10.226.1.15", - "wan":"10.226.1.15" - }, - "NodeMeta": {}, - "ServiceID": "472b116f9035:config_binding_service:10000", - "ServiceName": "config_binding_service", - "ServiceTags": [], - "ServiceAddress":"127.0.0.1", - "ServicePort":10000, - "ServiceEnableTagOverride": false, - "CreateIndex":1078990, - "ModifyIndex":1078990 - } - ] - -Fetch the generated configuration from CONFIG BINDING SERVICE using the -‘serviceaddress’ and ‘serviceport’ from above along with $SERVICE_NAME -from earlier. - -:: - - $ curl http://135.205.226.126:10000/service_component/user12.b599cf0e-75e8-484b-b8e2-557576d77036.0-7-0.sandbox-platform-yourapp - - {"streams_subscribes": {}, "services_calls": {}, "multiplier": 3, "streams_publishes": {}} - --------------- - -.. _dcae-cli-walkthrough-dmaap-testing: - -Create the input file for Dmaap Testing ---------------------------------------- - -Currently, the dcae-cli tool does not have the capability to provision -topics or feeds. Therefore, in order to test with ``message router`` or -``data router`` feeds, the developer must manually provision the topic -or feed and then provide the connection details in the form of a DMaap -JSON file for testing. This file is then passed in on the -``component run`` or ``component dev`` commands by using the argument -``--dmaap-file``. - -The structure of the DMaaP JSON is an object of config keys with the -topic or feed connection details. The config keys are the ``config_key`` -values specified in the component specification streams section where -the streams must be type ``message router`` or ``data router``. This -file corresponds to the ``Dmaap Connection Object`` which is generated -by the platform and provided to the component at runtime. The exception -is that ``delivery_url`` cannot be provided in the dmaap-file because it -is not created until the component is deployed. Refer to :any:`Dmaap Connection Object `, for details on creating the dmaap-file for testing. - --------------- - -Create the input file for *Sourced at Deployment* Testing ---------------------------------------------------------- - -Components may have configuration parameters whose values are to be -sourced at deployment time. This is established in the -:any:`component specification ` -by setting the property ``sourced_at_deployment`` to ``true`` for each -applicable parameter. - -Then, use the ``--inputs-file`` command-line argument when running the -component ``dev`` or ``run`` command for your component. This is to -simulate providing the dynamic, deployment time values for those -parameters marked as ``sourced_at_deployment``. - -For example, if your component specification has the following -configuration parameters: - -:: - - "parameters": [{ - "name": "vnf-ip", - "value": "", - "sourced_at_deployment": true - }, - { - "name": "static-param", - "value": 5 - }] - -Pass in an input file that looks like: - -:: - - { - "vnf-ip": "10.100.1.100" - } - -The application configuration would look like: - -:: - - { - "vnf-ip": "10.100.1.100", - "static-param": 5 - } - --------------- - -Run the component ------------------ - -The ``run`` operation is to be used for running your application in its -container remotely on the activated environment. Docker containers have -the additional option to run locally on your development machine. If the -component uses Dmaap, you can specify the Dmaap Connection Object as -well. Refer to :any:`Dmaap Connection Object `. - -In order to run the component, the data formats and component must have -been added to the onboarding catalog. - -To verify what’s in the catalog: - -:: - - $ dcae_cli catalog list --expanded - Active profile: solutioning - +---------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - | Name | Version | Type | Description | Status | Modified | #Deployed | - +---------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - | sandbox.platform.yourapp | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | staged | 2017-11-08 20:27:34.168854 | 0 | - +---------------------------+---------+--------+---------------------------------------------------------------+--------+----------------------------+-----------+ - -For Docker - -**NOTE** Make sure the Docker image has been uploaded to the shared -registry. - -A docker component can be run in either ``attached`` or ``unattached`` -mode. (Default is unattached). - -+------------------+-----------------------------------------------------------+ -| Mode | Description | -+==================+===========================================================+ -| attached | component is run in the foreground, container | -| | logs are streamed to stdout. Ctrl-C is used to | -| | terminate the dcae_cli session. | -+------------------+-----------------------------------------------------------+ -| unattached | component is run in the background container | -| | logs are viewed via ``docker logs`` command, | -| | container runs until undeployed with dcae_cli | -| | ``undeploy`` command. | -+------------------+-----------------------------------------------------------+ - -Run a component in attached mode: ---------------------------------- - -:: - - $ dcae_cli -v component run --attached sandbox.platform.yourapp:0.7.0 - DCAE.Docker | INFO | Running image 'nexus01.server.com:18443/repository/solutioning01-mte2-docker/dcae-platform/yourapp:0.7.0' as 'user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp' - DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | Consul host: yourconsulhost.com - - DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | service name: user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp - - DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | get_config returned the following configuration: {"streams_subscribes": {}, "multiplier": 3, "services_calls": {}, "streams_publishes": {}} - - DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | * Running on http://0.0.0.0:8080/ (Press CTRL+C to quit) - - DCAE.Docker.user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp | INFO | 127.0.0.1 - - [08/Nov/2017 23:27:30] "GET /health HTTP/1.1" 200 - - - - Hit Ctrl-C to terminate session. - - ^C - DCAE.Docker | INFO | Stopping container 'user12.dbb13a3c-d870-487e-b584-89929b856b5c.0-7-0.sandbox-platform-yourapp' and cleaning up... - -Run a component in unattached mode: ------------------------------------ - -:: - - $ dcae_cli -v component run sandbox.platform.yourapp:0.7.0 - DCAE.Docker | INFO | Running image 'nexus01.server.com:18443/repository/solutioning01-mte2-docker/dcae-platform/yourapp:0.7.0' as 'user12.22629ebd-417e-4e61-a9a0-f0cb16d4cef2.0-7-0.sandbox-platform-yourapp' - DCAE.Run | INFO | Deployed user12.22629ebd-417e-4e61-a9a0-f0cb16d4cef2.0-7-0.sandbox-platform-yourapp. Verifying.. - DCAE.Run | INFO | Container is up and healthy - -**NOTE** You must undeploy this component when finished testing. This is -important to conserve resources in the environment. - -Run a component that subscribes to Dmaap Message Router or Data Router ----------------------------------------------------------------------- - -:: - - $ dcae_cli -v component run $component-that-uses-dmamp --dmaap-file $dmaap-connection-object - -Run a component that expects input that is ``sourced at deployment`` --------------------------------------------------------------------- - -:: - - $ dcae_cli -v component run $component-that-expects-dti --inputs-file $input-file-to-simulate-dti - --------------- - - -.. _dcae_cli_undeploy_the_component: - -Undeploy the component ----------------------- - -The ``undeploy`` command is used to undeploy any instance of a specified component/version that you have deployed. This includes cleaning up the configuration. -Undeploy ``sandbox.platform.yourapp:0.7.0`` that was deployed above: -``$ dcae_cli -v component undeploy sandbox.platform.yourapp:0.7.0 DCAE.Undeploy | WARNING | Undeploying components: 1 DCAE.Undeploy | WARNING | Undeployed components: 1`` - -.. _dcae_cli_publish_the_component_and_data_format: - -Publish the component and data_format -------------------------------------- - -Once a component has been tested, it (and the data_format(s)) should be -published in the onboarding catalog using the ``publish`` sub-command -for both the ``data_format`` and ``component`` command. - -**Note** Before a component can be published, all data_formats that it -references must be published. - -Publishing will change the status of a component or data_format, -indicating that it has been tested, make accessible for other developers -to use. - -:: - - $ dcae_cli data_format publish sandbox.platform.yourapp:0.7.0 - Data format has been published - - $dcae_cli component publish sandbox.platform.yourapp:0.7.0 - Component has been published - --------------- - -List the catalog contents -------------------------- - -:: - - $dcae_cli catalog list - - $ dcae_cli data_format list | grep sandbox - | sandbox.platform.yourapp | 0.7.0 | docker | Web service used as a stand-alone test DCAE service compone.. | user12 | published | 2017-11-13 | - | sandbox.platform.yourapp.health | 0.1.0 | Data format used for the /health endpoint | published | 2017-11-13 17:48:10.121588 | - | sandbox.platform.any | 0.1.0 | Data format used when no data format is required. | published | 2017-11-13 17:47:51.622607 | - | sandbox.platform.yourapp.identity.response | 0.1.0 | Data format used for the /identity endpoint response which should | published | 2017-11-13 17:47:43.234715 | - | sandbox.platform.yourapp.identity.request | 0.1.0 | Data format used for the /identity endpoint request. This is | published | 2017-11-13 17:47:36.693643 | - | sandbox.platform.yourapp.rollcall.response | 0.1.0 | Data format used for the /rollcall endpoint respon.. | published | 2017-11-13 17:46:30.026846 | - diff --git a/docs/sections/components/glossary.rst b/docs/sections/components/glossary.rst deleted file mode 100644 index 51fb031b..00000000 --- a/docs/sections/components/glossary.rst +++ /dev/null @@ -1,210 +0,0 @@ - -.. http://creativecommons.org/licenses/by/4.0 - -.. _glossary: - - -Glossary -======== - -A&AI - Active and Available Inventory -------------------------------------- - -Inventory DB for all network components - -CLAMP ------ - -Non DCAE Platform Component - Controls the input and processing for -Closed Loop services. - -Closed Loop ------------ - -Services designed to monitor and report back to a controlling function -that automatically deals with the event reported without human -interaction. - -CDAP ----- - -Opensource Platform for development of Big Data platforms using Hadoop. -Some DCAE service components are written utilizing CDAP. - -Cloudify --------- - -Open Source application and network orchestration framework, based on -TOSCA used in DCAE to deploy platform and service components from -Cloudify Blueprints. Refer to :doc:`Architecture <./architecture/pieces>` -for more information. - -Cloudify Blueprints -------------------- - -YAML formatted file used by Cloudify to deploy platform and service -components. Contains all the information needed for installation. - -Consul ------- - -Opensource Platform Component that supports Service Discovery, -Configuration, and Healthcheck. Refer to -:doc:`Architecture <./architecture/pieces>` for more information. - -Component ---------- - -Refers to a DCAE service component which is a single micro-service that -is written to be run by the DCAE platform and to be composeable to form -a DCAE service. That composition occurs in the SDC. - -Config Binding Service ----------------------- - -DCAE Platform Component - Service Components use Config Binding Service -to access Consul and retrieve configuration variables. - -Component Specification ------------------------ - -JSON formatted file that fully describes a component and its interfaces - -Data Format / Data Format Specification ---------------------------------------- - -JSON formatted file that fully describes a components input or output - -dcae_cli Tool -------------- - -Tool used for development and testing. It validates the component and -data format specifications against their respective schemas and provides -the capability to view platform generated configuration for the -component. - -Deployment Handler ------------------- - -DCAE Platform Component - talks to -Cloudify to deploy components. - -Design-Time ------------ - -Refers to when the System Designer uses the SDC Tool to compose services -from components in the SDC catalog. The Designer can provide input to -assign/override defaults for configuration for any parameter with the -property ‘designer_editable’ set to ‘true’. - -Deploy-Time ------------ - -Refers to when a service is being deployed. This can be done -automatically via the SDC Tool, or manually via the DCAE Dashboard or -CLAMP UI. When manually deployed, DevOps can provide input to -assign/override defaults for configuration for any parameter with the -property ‘sourced_at_deployment’ set to ‘true’. - -Docker ------- - -Opensource Platform for development of containerized applications in the -cloud. Many DCAE service components and all DCAE collectors are written -utilizing Docker. - -Dmaap ------ - -A data transportation service platform that supports message-based -topics and file-based feeds. Runs locally at the Edge and Centrally. - -Inventory ---------- - -DCAE Platform Component - Postgres DB containing Cloudify Blueprints for -platform and service components. - -Onboarding catalog ------------------- - -Catalog used exclusively by the dcae_cli tool during development and -testing. Contains validated components and data_formats to be used among -developers during development and testing. - -Policy (not yet implemented) ----------------------------- - -Refers to the setting of configuration parameters for a component, by -Operations via the Policy UI. - -Policy Handler (not yet implemented) ------------------------------------- - -DCAE Platform Component that received Policy updates from Policy UI - -Policy UI (not yet implemented) -------------------------------- - -Non DCAE Component - Policy User Interface where Operations assigns -values to configuration specified for this. - -Run-Time --------- - -Refers to the when a service is running on the platform. - -SCH - Service Change Handler ----------------------------- - -DCAE Platform Component - Receives updates from SDC and updates -Inventory - -SDC - Service Design and Creation - (formerly ASDC) ---------------------------------------------------- - -Tool used by Service Designers to compose services from SDC catalog -artifacts. Once services are created, Cloudify Blueprints can be -generated to deployment and installation. - -SDC Catalog ------------ - -Catalog of composable Components and Data Formats to be used in the SDC -Tool to create services. Currently, there is no access to the SDC -Catalog from the dcae_cli tool. Artifacts are manually placed there -after testing. Every catalog artifact has a ``UUID``, a globally unique -identifier that identifies that artifact. - -Self-Service ------------- - -Refers to services that are supported by SDC, and that are automatically -installed as a result of a Service Designer’s composition and submission -of a service. Only a handful of services are ‘self-service’ currently. -Most require manual effort to generate the Tosca Model files and -Cloudify Blueprints. - -Service Component ------------------ - -Microservice that provides network monitoring or analytic function on -the DCAE platform. - -Service -------- - -Generally composed of multiple service components, which is deployed to -the DCAE platform. - -Tosca Model ------------ - -Model generated from validately component specification, (stored in SDC -catalog for Self-Service components), and used as input to generate -Cloudify Blueprints - -VNF - Virtualized Network Function ----------------------------------- - -A network function that runs on one or more virtualized machines. diff --git a/docs/sections/components/images/IO.graffle b/docs/sections/components/images/IO.graffle deleted file mode 100644 index 47c18b2a..00000000 Binary files a/docs/sections/components/images/IO.graffle and /dev/null differ diff --git a/docs/sections/components/images/dmd vision.graffle b/docs/sections/components/images/dmd vision.graffle deleted file mode 100644 index c6ad0d51..00000000 Binary files a/docs/sections/components/images/dmd vision.graffle and /dev/null differ diff --git a/docs/sections/components/images/dmdvision.png b/docs/sections/components/images/dmdvision.png deleted file mode 100644 index cc6f195f..00000000 Binary files a/docs/sections/components/images/dmdvision.png and /dev/null differ diff --git a/docs/sections/components/images/io.png b/docs/sections/components/images/io.png deleted file mode 100644 index 26c5eba9..00000000 Binary files a/docs/sections/components/images/io.png and /dev/null differ diff --git a/docs/sections/components/intro.rst b/docs/sections/components/intro.rst deleted file mode 100755 index db7175c1..00000000 --- a/docs/sections/components/intro.rst +++ /dev/null @@ -1,105 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -.. _intro: - - -Overview -======== - -DCAE components are services that provide a specific functionality and -are generally written to be composable with other DCAE components, -although a component can run independently as well. The DCAE platform is -responsible for running and managing DCAE service components reliably. - -Currently, the DCAE platform supports two types of components, CDAP -applications and Docker containers. For each, there are requirements -that must be met for the component to integrate into the DCAE platform -(see :doc:`CDAP ` and -:doc:`Docker `). - -A Component requires one or more data formats. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A component is a software application that performs a function. It -doesn’t run independently; it depends upon other components. A -component’s function could require connecting to other components to -fulfill that function. A component could also be providing its function -as a service through an interface for other components to use. - -A component cannot connect to or be connected with any other component. -The upstream and downstream components must *speak* the same vocabulary -or *data format*. The output of an one component must match another -component’s input. This is necessary for components to function -correctly and without errors. - -The platform requires data formats to ensure that a component will be -run with other *compatible* components. - -Data formats can and should be shared by multiple components. - -Each Component requires a component specification. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The component specification is a JSON artifact that fully specifies the -component, it’s interfaces, and configuration. It’s standardized for -CDAP and Docker applications and is validated using a :doc:`JSON schema <./component-json-schema>`. - -The component specification fully specifies all the configuration -parameters of the component. This is used by the designer and by policy -(future) to configure the runtime behavior of the component. - -The component specification is used to *generate* application -configuration in a standardized JSON that the platform will make -available to the component. This application configuration JSON will -include: - -- Parameters that have been assigned values from the component - specification, policy, and/or the designer -- Connection details of downstream components - -The component specification is transformed by DCAE tooling (explained -later) into TOSCA models (one for the component, and in the future, one -for Policy). The TOSCA models then get transformed into Cloudify -blueprints. - -The component specification is used by: - -- dcae_cli tool - to validate it -- Design Tools - TOSCA models are generated from the component - specification so that the component can be used by designers to - compose new DCAE services in SDC. -- Policy (future) - TOSCA models are generated from the component - specification so that operations can create policy models used to - dynamically configure the component. -- the runtime platform - The component’s application configuration - (JSON) is generated from the component specification and will be - provided to the component at runtime. - -Onboarding ----------- - -Onboarding is a process that ensures that the component is compliant -with the DCAE platform rules. A command-line tool called -:doc:`dcae-cli <./dcae-cli/quickstart>` is provided to -help with onboarding. The high level summary of the onboarding process -is: - -1. Defining the :doc:`data formats ` if they don’t already - exist. -2. Defining the :doc:`component specification <./component-specification/common-specification>`. - See :doc:`docker <./component-specification/docker-specification>` and - :doc:`CDAP <./component-specification/cdap-specification>`. -3. Use the dcae_cli tool to :any:`add the data formats ` and - :any:`add the component ` to the - onboarding catalog. This process will validate them as well. -4. Use the dcae_cli tool to - :any:`deploy ` the - component. (The component is deployed to the environment indicated in - the :any:`profile ` - section). -5. Test the component. Also do pairwise-test the component with any - other components it connects with. -6. Publish the component and data formats into the Service Design and - Creation (SDC) ‘catalog’. (Currently, this is a manual step, not done - via the dcae_cli tool). diff --git a/docs/sections/components/requirements-guidelines.rst b/docs/sections/components/requirements-guidelines.rst deleted file mode 100644 index 1f1870eb..00000000 --- a/docs/sections/components/requirements-guidelines.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 - -Requirements/Guidelines -======================= - -.. toctree:: - :maxdepth: 1 - - ./component-type-cdap.rst - ./component-type-docker.rst - diff --git a/docs/sections/design-components/DCAE-MOD/DCAE-MOD-Architecture.rst b/docs/sections/design-components/DCAE-MOD/DCAE-MOD-Architecture.rst new file mode 100644 index 00000000..64bd4dc2 --- /dev/null +++ b/docs/sections/design-components/DCAE-MOD/DCAE-MOD-Architecture.rst @@ -0,0 +1,178 @@ +===================== +DCAE MOD Architecture +===================== + + +DCAE MOD is composed of a mix of components developed in ONAP and other +components taken from the Apache Nifi project and modified for +appropriate use. The MOD architecture and design was intended to simplify the onboarding +and design experience in ONAP addressing below goals. + +.. toctree:: + :maxdepth: 1 + + ./DCAE-MOD-goals.rst + + +MOD aims to streamline the construction, management, +and evolution of DCAE flows from role to role, from environment to +environment, and from release to release. MOD is composed of three functional areas: onboarding, design, and +distribution and caters to different `user group <./Roles>`__ + + +The below illustrations describe the architecture of DCAE-MOD and show the +usage flow in DCAE MOD + +|image0| + +|image1| + +Onboarding API +-------------- + +It is a component developed to onboard +models/components/microservices (spec files) into DCAE MOD. + +Genprocessor +------------ + +It has been developed in Java. This project is a tool to +experiment with generating a Nifi Processor POJO from a DCAE component +spec. + +Nifi Web UI +----------- + +It is a component taken from the Apache Nifi Project but modified for +use in the MOD project. + +Apache NiFi is a dataflow system based on the concepts of flow-based +programming. It supports powerful and scalable directed graphs of data +routing, transformation, and system mediation logic. NiFi has a +web-based user interface for design, control, feedback, and monitoring +of dataflows. It is highly configurable along several dimensions of +quality of service, such as loss-tolerant versus guaranteed delivery, +low latency versus high throughput, and priority-based queuing. NiFi +provides fine-grained data provenance for all data received, forked, +joined cloned, modified, sent, and ultimately dropped upon reaching its +configured end-state. + +The NiFi UI provides mechanisms for creating automated dataflows, as +well as visualizing, editing, monitoring, and administering those +dataflows. The UI can be broken down into several segments, each +responsible for different functionality of the application. This section +provides screenshots of the application and highlights the different +segments of the UI. Each segment is discussed in further detail later in +the document. + +The users of Apache Nifi will find that it is used very differently than +intended to serve our purpose in the DCAE-MOD project. + + +Registry API +------------ + +This component taken from the Apache Nifi project, is a REST API that +provides an interface to a registry with operations for saving, +versioning, reading NiFi flows and components. + +Distributor API +--------------- + +It is a component developed using the Flask framework in Python. +It is a HTTP API to manage distribution targets for DCAE design. +Distribution targets are DCAE runtime environments that have been +registered and are enabled to accept flow design changes that are to be +orchestrated in that environment. + +Flow Based Programming (FBP) +---------------------------- + +NiFi’s fundamental design concepts closely relate to the main ideas of +Flow Based +Programming `[fbp] `__. + +For more information on how some of the main NiFi concepts map to FBP, +check out https://nifi.apache.org/docs/nifi-docs/html/overview.html + +Runtime API +----------- + +It is developed in Java’s Spring Boot framework. It +is a HTTP API to support runtime environment for DCAE-MOD. It has two +major functionalities: + +1. It accepts changes on the flow-graph via fbp protocol + +2. It generates and distributes blueprints based on the change made on + the flow-graph + +Blueprint Generator +------------------- + +This tool allows the user to create a blueprint from a component spec json file. +This tool is used by the runtime api. + +Inventory API +------------- + +DCAE Inventory is a web +service that provides the following: + +1. Real-time data on all DCAE services and their components + +2. Comprehensive details on available DCAE service types + +DCAE Inventory is a composite API that relies on other APIs to obtain +resources on underlying components and uses these resources to compose a +DCAE service resource. In addition, DCAE Inventory will store data that +is unique to the DCAE service level including: + +1. DCAE service metadata + +2. DCAE service type description and composition details + +3. Relationships between DCAE service and DCAE service types and their + respective VNF and VNF types + +DCAE Inventory has a REST interface to service client requests. It has a +well-defined query interface that would filter result sets based upon +resource attributes. + +Here, think of it as a back end API for the DCAE dashboard. The runtime +posts Cloudify Blueprints to this API so they show up in the DCAE +dashboard. + +DCAE Dashboard +-------------- + +The DCAE dashboard provides visibility into running DCAE services for +operational purposes. It queries the DCAE Inventory for aggregate +details on all the running DCAE services and for getting up-to-date +status information on DCAE services and their components. + +End-to-End Flow +--------------- + +A model/component/microservice can be onboarded by a ms Developer by +posting a spec file on the onboarding API. Alternatively, an Acumos +model can be onboarded using the Acumos Adapter. Once successfully +onboarded, the genprocessor converts converts them to jars and onboards +them into Nifi i.e DCAE MOD. These artifacts are now available to use +from the Modified Nifi Web UI i.e DCAE Designer. + +The registry api offers version control and retrieval for flows. The +distributor api can be used to set distribution targets. Once a flow is +designed and distributed, it goes to the distributor api which is +supposed to post graph changes (in accordance with fbp) to the runtime +api. The runtime api generates and distributes blueprints based on the +change made on the flow-graph. These blueprints received by the DCAE +inventory can then be viewed and deployed from the DCAE dashboard. + + + +.. |image0| image:: ../images/DCAE-Mod-Architecture.png + +.. |image1| image:: ../images/Onboarding-with-DCAE-MOD.png + +.. |image2| image:: ../images/nifi-toolbar-components.png diff --git a/docs/sections/design-components/DCAE-MOD/DCAE-MOD-User-Guide.rst b/docs/sections/design-components/DCAE-MOD/DCAE-MOD-User-Guide.rst new file mode 100644 index 00000000..25bf0957 --- /dev/null +++ b/docs/sections/design-components/DCAE-MOD/DCAE-MOD-User-Guide.rst @@ -0,0 +1,443 @@ +=================== +DCAE MOD User Guide +=================== + + + +- `Types of Users and Usage + Instructions: <#DCAEMODUserGuide(draft)-TypesofUsersand>`__ + +- `1.    Deployment of DCAE MOD components via Helm + charts <#DCAEMODUserGuide(draft)-1.DeploymentofD>`__ + + - `Using DCAE MOD without an Ingress + Controller <#DCAEMODUserGuide(draft)-UsingDCAEMODwit>`__ + +- `2.    Configuring DCAE + mod <#DCAEMODUserGuide(draft)-2.ConfiguringDC>`__ + +- `3.    Design & Distribution + Flow <#DCAEMODUserGuide(draft)-3.Design&Distri>`__ + + +Types of Users and Usage Instructions: +====================================== + ++-------+-----------------------------+-----------------------------+ +| Sr.No | User | Usage Instructions | ++=======+=============================+=============================+ +| 1. | Developers who are looking | -        Access the Nifi  | +| | to onboard their mS | Web UI url provided to you | +| | | | +| | | -        Follow steps  2.c | +| | | to 2.f | +| | | | +| | | -        You should be able | +| | | to see your microservices | +| | | in the Nifi Web UI by | +| | | clicking and dragging | +| | | ‘Processor’ on the canvas, | +| | | and searching for the name | +| | | of the | +| | | micros | +| | | ervice/component/processor. | ++-------+-----------------------------+-----------------------------+ +| 2. | Designers who are building | -        Access the Nifi  | +| | the flows through UI and | Web UI url provided to you | +| | triggering distribution | | +| | | -        Follow steps 3 to | +| | | the end of the document | ++-------+-----------------------------+-----------------------------+ +| 3. | Infrastructure/ Admins who | -        Follow start to | +| | want to stand up DCAE Mod | the end | +| | and validate it | | ++-------+-----------------------------+-----------------------------+ + + +1.    Deployment of DCAE MOD components via Helm charts +======================================================= + +The DCAE MOD components are deployed using the standard ONAP OOM +deployment process.   When deploying ONAP using the helm deploy command, +DCAE MOD components are deployed when the dcaemod.enabled flag is set to +true, either via a --set option on the command line or by an entry in an +overrides file.  In this respect, DCAE MOD is no different from any +other ONAP subsystem. + +The default DCAE MOD deployment relies on an nginx ingress controller +being available in the Kubernetes cluster where DCAE MOD is being +deployed.   The Rancher RKE installation process sets up a suitable +ingress controller.   In order to enable the use of the ingress +controller, it is necessary to override the OOM default global settings +for ingress configuration.   Specifically, the installation needs to set +the following configuration in an override file:: + + global: + ingress: + enabled: true + virtualhost: + enabled: false + +When DCAE MOD is deployed with an ingress controller, several endpoints +are exposed outside the cluster at the ingress controller's external IP +address and port.   (In the case of a Rancher RKE installation, there is +an ingress controller on every worker node, listening at the the +standard HTTP port (80).)  These exposed endpoints are needed by users +using machines outside the Kubernetes cluster. + ++--------------+--------------------------------------------------+--------------------------+ +| **Endpoint** | ** Routes to (cluster | **Description** | +| | internal address)** | | ++==============+==================================================+==========================+ +| /nifi | http://dcaemod-designtool:8080/nifi | Design tool Web UI | +| | | | ++--------------+--------------------------------------------------+--------------------------+ +| /nifi-api | http://dcaemod-designtool:8080/nifi-api | Design tool API | +| | | | ++--------------+--------------------------------------------------+--------------------------+ +| /nifi-jars | http://dcaemod-nifi-registry:18080/nifi-jars | Flow registry listing of | +| | | JAR files built from | +| | | component specs | ++--------------+--------------------------------------------------+--------------------------+ +| /onboarding | http://dcaemod-onboarding-api:8080/onboarding | Onboarding API | +| | | | ++--------------+--------------------------------------------------+--------------------------+ +| /distributor | http://dcaemod-distributor-api:8080/distributor | Distributor API | +| | | | ++--------------+--------------------------------------------------+--------------------------+ + +| To access the design Web UI, for example, a user would use the URL : + http://*ingress_controller_address:ingress_controller_port*/nifi. +| *ingress_controller_address* is the the IP address or DNS FQDN of the + ingress controller and +| *ingress_controller_port* is the port on which the ingress controller + is listening for HTTP requests.  (If the port is 80, the HTTP default, + then there is no need to specify a port.) + +There are two additional *internal* endpoints that users need to know, +in order to configure a registry client and a distribution target in the +design tool's controller settings. + ++------------------------+--------------------------------------------+ +| **Configuration Item** | **Endpoint URL** | ++========================+============================================+ +| Registry client | http://dcaemod-nifi-registry:18080 | ++------------------------+--------------------------------------------+ +| Distribution target | http://dcaemod-runtime-api:9090 | ++------------------------+--------------------------------------------+ + +Using DCAE MOD without an Ingress Controller + + +Not currently supported + +2.    Configuring DCAE mod +========================== + +**a. Configure Nifi Registry url** + +Next check Nifi settings by selecting the Hamburger button in the Nifi +UI. It should lead you to the Nifi Settings screen + +|image16| + +|image3| + +Add a registry client. The Registry client url will be +http://dcaemod-nifi-registry:18080 + +|image4| + + +**b. Add distribution target which will be the runtime api url** + +Set the distribution target in the controller settings + +|image17| + +Distribution target URL will be +`http://dcaemod-runtime-api:9090 `__ + + + +Now let’s access the Nifi (DCAE designer) UI - http:///nifi + +IPAddress is the host address or the DNS FQDN, if there is one, for one of the Kubernetes nodes. + +|image0| + + +**c. Get the artifacts to test and onboard.** + +Let's fetch the artifacts/ spec files + +**Sample Component DCAE-VES-Collector :** https://git.onap.org/dcaegen2/collectors/ves/tree/dpo/spec/vescollector-componentspec.json + +**Sample Data Format :** https://git.onap.org/dcaegen2/collectors/ves/tree/dpo/data-formats/VES-5.28.4-dataformat.json + +For the purpose of onboarding, a Sample Request body should be of the type -:: + + { "owner": "", "spec": } + +where the json object inside the spec field can be a component spec json. + +Request bodies of this type will be used in the onboarding requests you make using curl or the onboarding swagger interface. + +**The prepared Sample Request body for a component dcae-ves-collector looks like +so –** + +See :download:`VES Collector Spec <./Sample-Input-Files/Request-body-of-Sample-Component.json>` + +**The prepared Sample request body for a sample data format  looks like so -** + +See :download:`VES data Format <./Sample-Input-Files/Request-body-of-Sample-Data-Format.json>` + + + +**d. To onboard a data format and a component** + +Each component has a description that tells what it does. + +These requests would be of the type + +curl -X POST http:///onboarding/dataformats     -H "Content-Type: application/json" -d +@ + +curl -X POST http:///onboarding/components     -H "Content-Type: application/json" -d +@ + +In our case, + +curl -X POST http:///onboarding/dataformats     -H "Content-Type: application/json" -d @ + +curl -X POST http:///onboarding/components    -H "Content-Type: application/json" -d @ + + + + +**e. Verify the resources were created using** + +curl -X GET http:///onboarding/dataformats + +curl -X GET http:///onboarding/components + +**f. Verify the genprocessor (which polls onboarding periodically to convert component specs to nifi processor), converted the component** + +Open http:///nifi-jars in a browser. + +These jars should now be available for you to use in the nifi UI as +processors + +|image1| + +3.    Design & Distribution Flow +================================ + + +**a**. To start creating flows, we need to create a process group first. The +name of the process group will be the name of the flow. Drag and Drop on +the canvas, the ‘Processor Group’ icon from the DCAE Designer bar on the +top. + +|image2| + + +Now enter the process group by double clicking it, + +You can now drag and drop on the canvas ‘Processor’ icon from the top +DCAE Designer tab. You can search for a particular component in the +search box that appears when you attempt to drag the ‘Processor’ icon to +the canvas. + +|image5| + +If the Nifi registry linking worked, you should see the “Import” button +when you try to add a Processor or Process group to the Nifi canvas, +like so- + +|image6| + +By clicking on the import button, we can import already created saved +and version controlled flows from the Nifi registry, if they are +present. + +|image7| + +We can save created flows by version controlling them like so starting +with a 'right click' anywhere on the canvas- + +|image8| + +Ideally you would name the flow and process group the same, because +functionally they are similar. + +|image9| + +When the flow is checked in, the bar at the bottom shows a green +checkmark + +|image10| + +Note: Even if you move a component around on the canvas, and its +position on the canvas changes, it is recognized as a change, and it +will have to recommitted. + +You can add additional components in your flow and connect them. + +DcaeVesCollector connects to DockerTcagen2. + +|image11| + +|image12| + +|image13| + +Along the way you need to also provide topic names in the settings +section. These can be arbitrary names. + +|image14| + +To recap, see how DcaeVesCollector connects to DockerTcagen2. Look at +the connection relationships. Currently there is no way to validate +these relationships. Notice how it is required to name the topics by +going to Settings. + +The complete flow after joining our components looks like so + +|image15| + + +**b. Submit/ Distribute the flow:** + +Once your flow is complete and saved in the Nifi registry, you can +choose to submit it for distribution. + +|image18| + +If the flow was submitted successfully to the runtime api, you should +get a pop up a success message like so - + +|image19| + +At this step, the design was packaged and sent to Runtime api. + +The runtime is supposed to generate the blueprint out of the packaged +design/flow and push it to the DCAE inventory and the DCAE Dasboard. + +**c. Checking the components in the DCAE Dashboard** + +You should see the generated artifact/ blueprint in the DCAE Dashboard +dashboard at https://:30418/ccsdk-app/login_external.htm in +our deployment. The name for each component will be appended by the flow +name followed by underscore followed by the component’s name. + +The credentials to access the DCAE Dashboard are- + +:: + +Login: su1234 + +Password: fusion + + +|image20| + +|image21| + +|image22| + +The generated Blueprint can be viewed. + +|image23| + +Finally, the generated Blueprint can be deployed. + +|image24| + +You can use/import the attached input configurations files to deploy. Drag and Drop these sample JSON files to fill in the configuration values. +See :download:`VES Collector Input Configuration <./Sample-Input-Files/ves-deploy.input.json>` +See :download:`Tcagen2 Input Configuration <./Sample-Input-Files/tca-deploy.input.json>` + +|image25| + +|image26| + +.. |image0| image:: ../images/1.png + :width: 6.5in + :height: 1.08333in +.. |image1| image:: ../images/2.png + :width: 6.5in + :height: 1.58333in +.. |image2| image:: ../images/3.png + :width: 5.83333in + :height: 3.58333in +.. |image3| image:: ../images/4.png + :width: 4.91667in + :height: 2.16667in +.. |image4| image:: ../images/5.png + :width: 6.5in + :height: 2.66667in +.. |image5| image:: ../images/6.png + :width: 6.5in + :height: 3.33333in +.. |image6| image:: ../images/7.png + :width: 4.91667in + :height: 2.25in +.. |image7| image:: ../images/8.png + :width: 4.91667in + :height: 2.58333in +.. |image8| image:: ../images/9.png + :width: 6.5in + :height: 4.58333in +.. |image9| image:: ../images/10.png + :width: 6.5in + :height: 4in +.. |image10| image:: ../images/11.png + :width: 4.91667in + :height: 0.41667in +.. |image11| image:: ../images/12.png + :width: 6.33333in + :height: 3.16667in +.. |image12| image:: ../images/13.png + :width: 6in + :height: 2.66667in +.. |image13| image:: ../images/14.png + :width: 6.5in + :height: 3.41667in +.. |image14| image:: ../images/15.png + :width: 6.5in + :height: 3.58333in +.. |image15| image:: ../images/16.png + :width: 6.5in + :height: 2.25in +.. |image16| image:: ../images/17.png + :width: 6.5in + :height: 2.83333in +.. |image17| image:: ../images/18.png + :width: 6.5in + :height: 3.08333in +.. |image18| image:: ../images/19.png + :width: 4.91667in + :height: 1.91667in +.. |image19| image:: ../images/20.png + :width: 4.91667in + :height: 2.41667in +.. |image20| image:: ../images/21.png + :width: 6.5in + :height: 2.41667in +.. |image21| image:: ../images/22.png + :width: 6.5in + :height: 3in +.. |image22| image:: ../images/23.png + :width: 6.5in + :height: 2.16667in +.. |image23| image:: ../images/24.png + :width: 6.5in + :height: 2.83333in +.. |image24| image:: ../images/25.png + :width: 6.5in + :height: 3in +.. |image25| image:: ../images/26.png +.. |image26| image:: ../images/27.png + + diff --git a/docs/sections/design-components/DCAE-MOD/DCAE-MOD-goals.rst b/docs/sections/design-components/DCAE-MOD/DCAE-MOD-goals.rst new file mode 100644 index 00000000..23d393b1 --- /dev/null +++ b/docs/sections/design-components/DCAE-MOD/DCAE-MOD-goals.rst @@ -0,0 +1,43 @@ +============== +MOD Objectives +============== + +MOD stands for "micro-service onboarding and design" and the project is +an effort to reboot the onboarding and design experience in DCAE. + + +**Goals and Stretch Goals** +--------------------------- + + +- Due to resource constraints, there are mismatched capabilities between SDC/DCAE-DS and DCAE mS deployment. + +- Due to #1, mS developers upload handcrafted blueprint, and stay involved throughout the deployment process. This also ties mS development to specific Cloudify implementation. + +- There is no Service Assurance flow design in SDC/DCAE-DS, and so there are no reusable flow designs for the Service Designer. + +- There is extensive reliance on developers’ involvement in providing [Inputs.json] as runtime configurations for mS deployment. + +- There is no E2E tracking of the microservice lifecycle. + + +**To address these problems, the new DCAE MOD, replacing the mS onboarding & DCAE-DS in SDC, aims to -** + + + +- Move DCAE mS onboarding & design from SDC project to DCAE Project. + +- Provide simplified mS Onboarding, Service Assurance flow design, & mS microservice design time & runtime configurations to support developers, service designers, and operations. + +- Auto-generate blueprint at the end of the design process, not onboarded before the design process. + +- Support Policy onboarding & artifact distribution to Policy/CLAMP to support Self Service Control Loop. + +- Streamline the process of constructing to deploying flows, Provide the ability to track flows - capture and store the progress and evolution of flows and Provide clear coordination and accountability i.e Provide catalog & data for microservice lifecycle tracking. It fits the ECOMP's release process and must provide clear visibility along the entire process and across different environments. + +- Support automated adaptation of ML model from Acumos to DCAE design & runtime environment through the Acumos Adapter. + +- DCAE-MOD is developed by the DCAE team to ensure consistency across all DCAE implementation, with the long term objective to integrate with SDC as part of the Design Platform. + +- Integrate with ONAP User Experience portals (initially ONAP portal, later SDC portal). + diff --git a/docs/sections/design-components/DCAE-MOD/Roles.rst b/docs/sections/design-components/DCAE-MOD/Roles.rst new file mode 100644 index 00000000..c3460841 --- /dev/null +++ b/docs/sections/design-components/DCAE-MOD/Roles.rst @@ -0,0 +1,168 @@ +===== +Roles +===== + + +Here is master list of all roles involved in ECOMP with DCAE: + +- System engineer + +- Component developer/expert - components are also referred to as + micro-services but include collectors, analytics + +- Designer + +- Tester + +- Operations + +- Platform developer + +- Manager + + +System engineer +--------------- + +Person who knows the high-level technical requirements for DCAE's +upcoming release cycle and dictates the development needs.  This person +is responsible for the service assurance flows.  This person expresses +the nodes and connections of a flow at a high level in a new graph or an +existing graph and assigns nodes to component developer/experts to be +implemented. + +This person must know: + +- What newly added flows should look like at a high level + +- What changes that are needed to existing flows + +- Target environments/sites/locations that need the flows at what SLA + +- Data requirements e.g. volume, rate, format, retention + +This person creates a top-level representation of the flow and assigns +the pieces to developers or experts for implementation. + + +Component developer/expert +-------------------------- + +Person who is responsible for defining an assigned node's subgraph.  +This person can be: + +- A developer who might be onboarding a new component or a new version + of an existing component to fulfill the system engineer's + requirements + +- A domain expert who selects a suitable existing component, wires and + configures.  This expert knows the intricacies of a class of + components (e.g. Acumos machine learning). + +Developers +---------- + +They must know: + +- The target DCAE runtime and can develop a component to successfully + run on the runtime + +- The DCAE onboarding process for components including the development + testing procedure + +- Best practices of data flow management (data provenance?) + +- Lifecycle of DCAE components specifically impact of changes to + existing running instances globally + +- The resource requirement of the developed component + +Experts +------- + +They must know: + +- The target DCAE runtime + +- The technical capabilities of a set of components in order to best + select + +- The technical needs of the set of components in order to properly + configure and connect + +Designer +-------- + + +Person who is responsible for connecting remote nodes to flows and +configuring all nodes in a flow in the context of the flow and in the +context of the greater graph.  An example of the former is connecting a +flow with a collector to a black boxed vMME.  An example of the latter +is assigning the threshold to a threshold-crossing-analytics component +when it is connected to a specific VES collector who is connected to a +specific vMME. + +This person knows: + +- The VNFs to monitor and the technical details to correctly connect + with them + +- Enough about the capability of a component and understands the + characteristics and requirements of a flow to properly + assign designer_editable configuration parameters + +This person has the ability to promote flows through the development +process (i.e. FTL to IST to ETE) and will coordinate with testers to +make sure the progression happens. + +Tester +------ + +Person who is responsible for testing a promoted new flow or newly +edited flow.  Once the designer has promoted a flow to a tester's +environment, the tester will have ready access to the deployment +artifacts necessary to apply the runtime changes that will reflect the +flow design and verify the resulting functionality matches to the system +engineer's expectations. + +Person knows and owns a DCAE runtime. + +Read access to the design tool would be useful for troubleshooting. + + +Operations +---------- + +Person who is responsible that DCAE both platform and service assurance +flows are all operational in production environments and locations.  +Once a flow has been fully certified, the required deployment artifact +is provided to operations and operations is responsible for applying the +runtime changes to reflect the flow design. + +Person knows and owns a DCAE runtime. + +Read access to the design tool would be useful for troubleshooting. + + + +Manager +------- + + +Person who are accountable to the business of the successful delivery of +a set of service assurance flows.  Read access to the design tool +specifically high level reports are useful to understand if goals are +being met and to better measure project success. + +Platform developer +------------------ + + +Person who is responsible for the development of the DCAE platform which +ranges from onboarding, design, and runtime.  In the scope of onboarding +and design, they are also responsible (unless there's an internal +operational team) for the operational concerns which means the tooling +built in this effort will need to be continually supported.  +Their **users** are all the above.  The design tool is intended to span +across multiple environments thus must run where all the required +parties have access. diff --git a/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json b/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json new file mode 100644 index 00000000..68667836 --- /dev/null +++ b/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/Request-body-of-Sample-Component.json @@ -0,0 +1,328 @@ +{ + "owner": "mS-Developer", + "spec": { + "self": { + "version": "1.5.4", + "name": "dcae-ves-collector", + "description": "Collector for receiving VES events through restful interface", + "component_type": "docker" + }, + "streams": { + "subscribes": [], + "publishes": [ + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-fault" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-measurement" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-syslog" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-heartbeat" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-other" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-mobileflow" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-statechange" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-thresholdCrossingAlert" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-voicequality" + }, + { + "format": "VES_specification", + "version": "5.28.4", + "type": "message router", + "config_key": "ves-sipsignaling" + }, + { + "format": "VES_specification", + "version": "7.30.1", + "type": "message router", + "config_key": "ves-pnfRegistration" + }, + { + "format": "VES_specification", + "version": "7.30.1", + "type": "message router", + "config_key": "ves-notification" + }, + { + "format": "VES_specification", + "version": "7.30.1", + "type": "message router", + "config_key": "ves-perf3gpp" + } + ] + }, + "services": { + "calls": [], + "provides": [ + { + "route": "/eventListener/v1", + "verb": "POST", + "request": { + "format": "VES_specification", + "version": "4.27.2" + }, + "response": { + "format": "ves.coll.response", + "version": "1.0.0" + } + }, + { + "route": "/eventListener/v2", + "verb": "POST", + "request": { + "format": "VES_specification", + "version": "4.27.2" + }, + "response": { + "format": "ves.coll.response", + "version": "1.0.0" + } + }, + { + "route": "/eventListener/v3", + "verb": "POST", + "request": { + "format": "VES_specification", + "version": "4.27.2" + }, + "response": { + "format": "ves.coll.response", + "version": "1.0.0" + } + }, + { + "route": "/eventListener/v4", + "verb": "POST", + "request": { + "format": "VES_specification", + "version": "4.27.2" + }, + "response": { + "format": "ves.coll.response", + "version": "1.0.0" + } + }, + { + "route": "/eventListener/v5", + "verb": "POST", + "request": { + "format": "VES_specification", + "version": "5.28.4" + }, + "response": { + "format": "ves.coll.response", + "version": "1.0.0" + } + }, + { + "route": "/eventListener/v7", + "verb": "POST", + "request": { + "format": "VES_specification", + "version": "7.30.1" + }, + "response": { + "format": "ves.coll.response", + "version": "1.0.0" + } + } + ] + }, + "parameters": [ + { + "name": "collector.service.port", + "value": 8080, + "description": "standard http port collector will open for listening;", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "collector.service.secure.port", + "value": 8443, + "description": "secure http port collector will open for listening ", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": true + }, + { + "name": "collector.keystore.file.location", + "value": "/opt/app/dcae-certificate/cert.jks", + "description": "fs location of keystore file in vm", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "collector.keystore.passwordfile", + "value": "/opt/app/dcae-certificate/jks.pass", + "description": "location of keystore password file in vm", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "collector.truststore.file.location", + "value": "/opt/app/dcae-certificate/trust.jks", + "description": "fs location of truststore file in vm", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "collector.truststore.passwordfile", + "value": "/opt/app/dcae-certificate/trust.pass", + "description": "location of truststore password file in vm", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "collector.dmaap.streamid", + "value": "fault=ves-fault|syslog=ves-syslog|heartbeat=ves-heartbeat|measurementsForVfScaling=ves-measurement|measurement=ves-measurement|mobileFlow=ves-mobileflow|other=ves-other|stateChange=ves-statechange|thresholdCrossingAlert=ves-thresholdCrossingAlert|voiceQuality=ves-voicequality|sipSignaling=ves-sipsignaling|notification=ves-notification|pnfRegistration=ves-pnfRegistration|perf3gpp=ves-perf3gpp", + "description": "domain-to-streamid mapping used by VESCollector to distributes events based on domain. Both primary and secondary config_key are included for resilency (multiple streamid can be included commma separated). The streamids MUST match to topic config_keys. For single site without resiliency deployment - configkeys with -secondary suffix can be removed", + "sourced_at_deployment": true, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "auth.method", + "value": "noAuth", + "description": "Property to manage application mode, possible configurations: noAuth - default option - no security (http) , certOnly - auth by certificate (https), basicAuth - auth by basic auth username and password (https),certBasicAuth - auth by certificate and basic auth username / password (https),", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "header.authlist", + "value": "sample1,$2a$10$pgjaxDzSuc6XVFEeqvxQ5u90DKJnM/u7TJTcinAlFJVaavXMWf/Zi|userid1,$2a$10$61gNubgJJl9lh3nvQvY9X.x4e5ETWJJ7ao7ZhJEvmfJigov26Z6uq|userid2,$2a$10$G52y/3uhuhWAMy.bx9Se8uzWinmbJa.dlm1LW6bYPdPkkywLDPLiy", + "description": "List of id and base 64 encoded password.For each onboarding VNF - unique userid and password should be assigned and communicated to VNF owner. Password value should be base64 encoded in config here", + "policy_editable": false, + "sourced_at_deployment": true, + "designer_editable": true + }, + { + "name": "collector.schema.checkflag", + "value": 1, + "description": "Schema check validation flag. When enabled, collector will validate input VES events against VES Schema defined on collector.schema.file ", + "sourced_at_deployment": false, + "policy_editable": false, + "designer_editable": false + }, + { + "name": "collector.schema.file", + "value": "{\"v1\":\"./etc/CommonEventFormat_27.2.json\",\"v2\":\"./etc/CommonEventFormat_27.2.json\",\"v3\":\"./etc/CommonEventFormat_27.2.json\",\"v4\":\"./etc/CommonEventFormat_27.2.json\",\"v5\":\"./etc/CommonEventFormat_28.4.1.json\",\"v7\":\"./etc/CommonEventFormat_30.1.1.json\"}", + "description": "VES schema file name per version used for validation", + "designer_editable": true, + "sourced_at_deployment": false, + "policy_editable": false + }, + { + "name": "event.transform.flag", + "value": 1, + "description": "flag to enable tranformation rules defined under eventTransform.json; this is applicable when event tranformation rules preset should be activated for transforming elements, starting at 1", + "type": "string" + }, + "url": { + "description": "Hyperlink to a detailed error resource e.g., an HTML page for browser user agents", + "type": "string" + }, + "variables": { + "description": "List of zero or more strings that represent the contents of the variables used by the message text", + "type": "string" + } + }, + "required": [ + "messageId", + "text" + ] + }, + "sipSignalingFields": { + "description": "sip signaling fields", + "type": "object", + "properties": { + "additionalInformation": { + "description": "additional sip signaling fields if needed", + "type": "array", + "items": { + "$ref": "#/definitions/field" + } + }, + "compressedSip": { + "description": "the full SIP request/response including headers and bodies", + "type": "string" + }, + "correlator": { + "description": "this is the same for all events on this call", + "type": "string" + }, + "localIpAddress": { + "description": "IP address on VNF", + "type": "string" + }, + "localPort": { + "description": "port on VNF", + "type": "string" + }, + "remoteIpAddress": { + "description": "IP address of peer endpoint", + "type": "string" + }, + "remotePort": { + "description": "port of peer endpoint", + "type": "string" + }, + "sipSignalingFieldsVersion": { + "description": "version of the sipSignalingFields block", + "type": "number" + }, + "summarySip": { + "description": "the SIP Method or Response (‘INVITE’, ‘200 OK’, ‘BYE’, etc)", + "type": "string" + }, + "vendorVnfNameFields": { + "$ref": "#/definitions/vendorVnfNameFields" + } + }, + "required": [ + "correlator", + "localIpAddress", + "localPort", + "remoteIpAddress", + "remotePort", + "sipSignalingFieldsVersion", + "vendorVnfNameFields" + ] + }, + "stateChangeFields": { + "description": "stateChange fields", + "type": "object", + "properties": { + "additionalFields": { + "description": "additional stateChange fields if needed", + "type": "array", + "items": { + "$ref": "#/definitions/field" + } + }, + "newState": { + "description": "new state of the entity", + "type": "string", + "enum": [ + "inService", + "maintenance", + "outOfService" + ] + }, + "oldState": { + "description": "previous state of the entity", + "type": "string", + "enum": [ + "inService", + "maintenance", + "outOfService" + ] + }, + "stateChangeFieldsVersion": { + "description": "version of the stateChangeFields block", + "type": "number" + }, + "stateInterface": { + "description": "card or port name of the entity that changed state", + "type": "string" + } + }, + "required": [ + "newState", + "oldState", + "stateChangeFieldsVersion", + "stateInterface" + ] + }, + "suppressedNvPairs": { + "description": "List of specific NvPairsNames to suppress within a given Name-Value Field for event Throttling", + "type": "object", + "properties": { + "nvPairFieldName": { + "description": "Name of the field within which are the nvpair names to suppress", + "type": "string" + }, + "suppressedNvPairNames": { + "description": "Array of nvpair names to suppress within the nvpairFieldName", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "nvPairFieldName", + "suppressedNvPairNames" + ] + }, + "syslogFields": { + "description": "sysLog fields", + "type": "object", + "properties": { + "additionalFields": { + "description": "additional syslog fields if needed provided as name=value delimited by a pipe ‘|’ symbol, for example: 'name1=value1|name2=value2|…'", + "type": "string" + }, + "eventSourceHost": { + "description": "hostname of the device", + "type": "string" + }, + "eventSourceType": { + "description": "type of event source; examples: other, router, switch, host, card, port, slotThreshold, portThreshold, virtualMachine, virtualNetworkFunction", + "type": "string" + }, + "syslogFacility": { + "description": "numeric code from 0 to 23 for facility--see table in documentation", + "type": "integer" + }, + "syslogFieldsVersion": { + "description": "version of the syslogFields block", + "type": "number" + }, + "syslogMsg": { + "description": "syslog message", + "type": "string" + }, + "syslogPri": { + "description": "0-192 combined severity and facility", + "type": "integer" + }, + "syslogProc": { + "description": "identifies the application that originated the message", + "type": "string" + }, + "syslogProcId": { + "description": "a change in the value of this field indicates a discontinuity in syslog reporting", + "type": "number" + }, + "syslogSData": { + "description": "syslog structured data consisting of a structured data Id followed by a set of key value pairs", + "type": "string" + }, + "syslogSdId": { + "description": "0-32 char in format name@number for example ourSDID@32473", + "type": "string" + }, + "syslogSev": { + "description": "numerical Code for severity derived from syslogPri as remaider of syslogPri / 8", + "type": "string", + "enum": [ + "Alert", + "Critical", + "Debug", + "Emergency", + "Error", + "Info", + "Notice", + "Warning" + ] + }, + "syslogTag": { + "description": "msgId indicating the type of message such as TCPOUT or TCPIN; NILVALUE should be used when no other value can be provided", + "type": "string" + }, + "syslogVer": { + "description": "IANA assigned version of the syslog protocol specification - typically 1", + "type": "number" + } + }, + "required": [ + "eventSourceType", + "syslogFieldsVersion", + "syslogMsg", + "syslogTag" + ] + }, + "thresholdCrossingAlertFields": { + "description": "fields specific to threshold crossing alert events", + "type": "object", + "properties": { + "additionalFields": { + "description": "additional threshold crossing alert fields if needed", + "type": "array", + "items": { + "$ref": "#/definitions/field" + } + }, + "additionalParameters": { + "description": "performance counters", + "type": "array", + "items": { + "$ref": "#/definitions/counter" + } + }, + "alertAction": { + "description": "Event action", + "type": "string", + "enum": [ + "CLEAR", + "CONT", + "SET" + ] + }, + "alertDescription": { + "description": "Unique short alert description such as IF-SHUB-ERRDROP", + "type": "string" + }, + "alertType": { + "description": "Event type", + "type": "string", + "enum": [ + "CARD-ANOMALY", + "ELEMENT-ANOMALY", + "INTERFACE-ANOMALY", + "SERVICE-ANOMALY" + ] + }, + "alertValue": { + "description": "Calculated API value (if applicable)", + "type": "string" + }, + "associatedAlertIdList": { + "description": "List of eventIds associated with the event being reported", + "type": "array", + "items": { + "type": "string" + } + }, + "collectionTimestamp": { + "description": "Time when the performance collector picked up the data; with RFC 2822 compliant format: Sat, 13 Mar 2010 11:29:05 -0800", + "type": "string" + }, + "dataCollector": { + "description": "Specific performance collector instance used", + "type": "string" + }, + "elementType": { + "description": "type of network element - internal ATT field", + "type": "string" + }, + "eventSeverity": { + "description": "event severity or priority", + "type": "string", + "enum": [ + "CRITICAL", + "MAJOR", + "MINOR", + "WARNING", + "NORMAL" + ] + }, + "eventStartTimestamp": { + "description": "Time closest to when the measurement was made; with RFC 2822 compliant format: Sat, 13 Mar 2010 11:29:05 -0800", + "type": "string" + }, + "interfaceName": { + "description": "Physical or logical port or card (if applicable)", + "type": "string" + }, + "networkService": { + "description": "network name - internal ATT field", + "type": "string" + }, + "possibleRootCause": { + "description": "Reserved for future use", + "type": "string" + }, + "thresholdCrossingFieldsVersion": { + "description": "version of the thresholdCrossingAlertFields block", + "type": "number" + } + }, + "required": [ + "additionalParameters", + "alertAction", + "alertDescription", + "alertType", + "collectionTimestamp", + "eventSeverity", + "eventStartTimestamp", + "thresholdCrossingFieldsVersion" + ] + }, + "vendorVnfNameFields": { + "description": "provides vendor, vnf and vfModule identifying information", + "type": "object", + "properties": { + "vendorName": { + "description": "VNF vendor name", + "type": "string" + }, + "vfModuleName": { + "description": "ASDC vfModuleName for the vfModule generating the event", + "type": "string" + }, + "vnfName": { + "description": "ASDC modelName for the VNF generating the event", + "type": "string" + } + }, + "required": [ + "vendorName" + ] + }, + "vNicPerformance": { + "description": "describes the performance and errors of an identified virtual network interface card", + "type": "object", + "properties": { + "receivedBroadcastPacketsAccumulated": { + "description": "Cumulative count of broadcast packets received as read at the end of the measurement interval", + "type": "number" + }, + "receivedBroadcastPacketsDelta": { + "description": "Count of broadcast packets received within the measurement interval", + "type": "number" + }, + "receivedDiscardedPacketsAccumulated": { + "description": "Cumulative count of discarded packets received as read at the end of the measurement interval", + "type": "number" + }, + "receivedDiscardedPacketsDelta": { + "description": "Count of discarded packets received within the measurement interval", + "type": "number" + }, + "receivedErrorPacketsAccumulated": { + "description": "Cumulative count of error packets received as read at the end of the measurement interval", + "type": "number" + }, + "receivedErrorPacketsDelta": { + "description": "Count of error packets received within the measurement interval", + "type": "number" + }, + "receivedMulticastPacketsAccumulated": { + "description": "Cumulative count of multicast packets received as read at the end of the measurement interval", + "type": "number" + }, + "receivedMulticastPacketsDelta": { + "description": "Count of multicast packets received within the measurement interval", + "type": "number" + }, + "receivedOctetsAccumulated": { + "description": "Cumulative count of octets received as read at the end of the measurement interval", + "type": "number" + }, + "receivedOctetsDelta": { + "description": "Count of octets received within the measurement interval", + "type": "number" + }, + "receivedTotalPacketsAccumulated": { + "description": "Cumulative count of all packets received as read at the end of the measurement interval", + "type": "number" + }, + "receivedTotalPacketsDelta": { + "description": "Count of all packets received within the measurement interval", + "type": "number" + }, + "receivedUnicastPacketsAccumulated": { + "description": "Cumulative count of unicast packets received as read at the end of the measurement interval", + "type": "number" + }, + "receivedUnicastPacketsDelta": { + "description": "Count of unicast packets received within the measurement interval", + "type": "number" + }, + "transmittedBroadcastPacketsAccumulated": { + "description": "Cumulative count of broadcast packets transmitted as read at the end of the measurement interval", + "type": "number" + }, + "transmittedBroadcastPacketsDelta": { + "description": "Count of broadcast packets transmitted within the measurement interval", + "type": "number" + }, + "transmittedDiscardedPacketsAccumulated": { + "description": "Cumulative count of discarded packets transmitted as read at the end of the measurement interval", + "type": "number" + }, + "transmittedDiscardedPacketsDelta": { + "description": "Count of discarded packets transmitted within the measurement interval", + "type": "number" + }, + "transmittedErrorPacketsAccumulated": { + "description": "Cumulative count of error packets transmitted as read at the end of the measurement interval", + "type": "number" + }, + "transmittedErrorPacketsDelta": { + "description": "Count of error packets transmitted within the measurement interval", + "type": "number" + }, + "transmittedMulticastPacketsAccumulated": { + "description": "Cumulative count of multicast packets transmitted as read at the end of the measurement interval", + "type": "number" + }, + "transmittedMulticastPacketsDelta": { + "description": "Count of multicast packets transmitted within the measurement interval", + "type": "number" + }, + "transmittedOctetsAccumulated": { + "description": "Cumulative count of octets transmitted as read at the end of the measurement interval", + "type": "number" + }, + "transmittedOctetsDelta": { + "description": "Count of octets transmitted within the measurement interval", + "type": "number" + }, + "transmittedTotalPacketsAccumulated": { + "description": "Cumulative count of all packets transmitted as read at the end of the measurement interval", + "type": "number" + }, + "transmittedTotalPacketsDelta": { + "description": "Count of all packets transmitted within the measurement interval", + "type": "number" + }, + "transmittedUnicastPacketsAccumulated": { + "description": "Cumulative count of unicast packets transmitted as read at the end of the measurement interval", + "type": "number" + }, + "transmittedUnicastPacketsDelta": { + "description": "Count of unicast packets transmitted within the measurement interval", + "type": "number" + }, + "valuesAreSuspect": { + "description": "Indicates whether vNicPerformance values are likely inaccurate due to counter overflow or other condtions", + "type": "string", + "enum": [ + "true", + "false" + ] + }, + "vNicIdentifier": { + "description": "vNic identification", + "type": "string" + } + }, + "required": [ + "valuesAreSuspect", + "vNicIdentifier" + ] + }, + "voiceQualityFields": { + "description": "provides statistics related to customer facing voice products", + "type": "object", + "properties": { + "additionalInformation": { + "description": "additional voice quality fields if needed", + "type": "array", + "items": { + "$ref": "#/definitions/field" + } + }, + "calleeSideCodec": { + "description": "callee codec for the call", + "type": "string" + }, + "callerSideCodec": { + "description": "caller codec for the call", + "type": "string" + }, + "correlator": { + "description": "this is the same for all events on this call", + "type": "string" + }, + "endOfCallVqmSummaries": { + "$ref": "#/definitions/endOfCallVqmSummaries" + }, + "phoneNumber": { + "description": "phone number associated with the correlator", + "type": "string" + }, + "midCallRtcp": { + "description": "Base64 encoding of the binary RTCP data excluding Eth/IP/UDP headers", + "type": "string" + }, + "vendorVnfNameFields": { + "$ref": "#/definitions/vendorVnfNameFields" + }, + "voiceQualityFieldsVersion": { + "description": "version of the voiceQualityFields block", + "type": "number" + } + }, + "required": [ + "calleeSideCodec", + "callerSideCodec", + "correlator", + "midCallRtcp", + "vendorVnfNameFields", + "voiceQualityFieldsVersion" + ] + } + } + } + } +} diff --git a/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json b/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json new file mode 100644 index 00000000..c3faf800 --- /dev/null +++ b/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/tca-deploy.input.json @@ -0,0 +1,21 @@ +{ + "spring.data.mongodb.uri": "mongodb://dcae-mongohost/dcae-tcagen2", + "tca.aai.enable_enrichment": true, + "docker-tcagen2_memory_request": "512Mi", + "replicas": 1, + "docker-tcagen2_cpu_limit": "250m", + "tca.aai.username": "DCAE", + "image": "nexus3.onap.org:10001/onap/org.onap.dcaegen2.analytics.tca-gen2.dcae-analytics-tca-web:1.0.1", + "service_component_name_override": "dcaegen2-tcagen2", + "use_tls": true, + "always_pull_image": true, + "tca.aai.url": "http://aai.onap.svc.cluster.local", + "location_id": "", + "envs": {}, + "tca_handle_in_subscribe_url": "http://message-router:3904/events/unauthenticated.VES_MEASUREMENT_OUTPUT/", + "tca_handle_out_publish_url": "http://message-router:3904/events/unauthenticated.TCAGEN2_OUTPUT/", + "external_port_0": 0, + "docker-tcagen2_memory_limit": "512Mi", + "docker-tcagen2_cpu_request": "250m", + "tca.aai.password": "DCAE" +} \ No newline at end of file diff --git a/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json b/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json new file mode 100644 index 00000000..d86ff0bc --- /dev/null +++ b/docs/sections/design-components/DCAE-MOD/Sample-Input-Files/ves-deploy.input.json @@ -0,0 +1,30 @@ +{ + "always_pull_image": true, + "collector.dmaap.streamid": "fault=ves-fault|syslog=ves-syslog|heartbeat=ves-heartbeat|measurementsForVfScaling=ves-measurement|measurement=ves-measurement|mobileFlow=ves-mobileflow|other=ves-other|stateChange=ves-statechange|thresholdCrossingAlert=ves-thresholdCrossingAlert|voiceQuality=ves-voicequality|sipSignaling=ves-sipsignaling|notification=ves-notification|pnfRegistration=ves-pnfRegistration|perf3gpp=ves-perf3gpp", + "dcae-ves-collector_cpu_limit": "250m", + "dcae-ves-collector_cpu_request": "250m", + "dcae-ves-collector_memory_limit": "512Mi", + "dcae-ves-collector_memory_request": "512Mi", + "envs": {}, + "external_port_0": 30235, + "external_port_1": 0, + "header.authlist": "sample1,$2a$10$pgjaxDzSuc6XVFEeqvxQ5u90DKJnM/u7TJTcinAlFJVaavXMWf/Zi|userid1,$2a$10$61gNubgJJl9lh3nvQvY9X.x4e5ETWJJ7ao7ZhJEvmfJigov26Z6uq|userid2,$2a$10$G52y/3uhuhWAMy.bx9Se8uzWinmbJa.dlm1LW6bYPdPkkywLDPLiy", + "image": "nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.ves.vescollector:1.5.4", + "location_id": "", + "replicas": 1, + "service_component_name_override": "dcae-ves-collector-http", + "use_tls": true, + "ves_fault_publish_url": "http://message-router:3904/events/unauthenticated.SEC_FAULT_OUTPUT/", + "ves_heartbeat_publish_url": "http://message-router:3904/events/unauthenticated.SEC_HEARTBEAT_OUTPUT/", + "ves_measurement_publish_url": "http://message-router:3904/events/unauthenticated.VES_MEASUREMENT_OUTPUT/", + "ves_mobileflow_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", + "ves_notification_publish_url": "http://message-router:3904/events/unauthenticated.VES_NOTIFICATION_OUTPUT/", + "ves_other_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", + "ves_perf3gpp_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", + "ves_pnfRegistration_publish_url": "http://message-router:3904/events/unauthenticated.VES_PNFREG_OUTPUT/", + "ves_sipsignaling_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", + "ves_statechange_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", + "ves_syslog_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", + "ves_thresholdCrossingAlert_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/", + "ves_voicequality_publish_url": "http://message-router:3904/events/unauthenticated.SEC_OTHER_OUTPUT/" +} \ No newline at end of file diff --git a/docs/sections/design-components/blueprint_generator.rst b/docs/sections/design-components/blueprint_generator.rst new file mode 100644 index 00000000..7670bb2b --- /dev/null +++ b/docs/sections/design-components/blueprint_generator.rst @@ -0,0 +1,84 @@ + +Blueprint Generator +=================== + +What is Blueprint Generator? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The blueprint generator is java-based tool to take a component spec +for a given micro-service and translate that component spec into a +cloudify blueprint yaml file that can be used during deployment in DCAE +Runtime plaform. + +Service components to be deployed as stand-alone +(i.e not part of DCAE service composition flow) can use the blueprint-generator +utility to create deployment yaml. The generated blueprint can be uploaded +to inventory and deployed from Dashboard directly. + + +Steps to run the blueprint generator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. Download the `blueprint generator jar `__ file from Nexus + +2. To execute the application, run the following command + + ``java -jar blueprint-generator-1.3.1-executable.jar blueprint`` + +3. This execution will provide the help, as you have not provided the required flags. + +4. When ready you can run the program again except with the required flags. + +5. OPTIONS + + - -p: The path to where the final blueprint yaml file will be created (required) + - -i: The path to the JSON spec file (required) + - -n: Name of the blueprint (optional) + - -t: the path to the import yaml file (optional) + - -d: If this flag is present the bp generator will be created with dmaap plugin (optional) + - -o: This flag will create a service component override for your deployment equal to the value you put (optional) + +6. An example running this program is shown below + + ``java -jar blueprint-generator-1.3.1-executable.jar -p blueprint_output -i ComponentSpecs/TestComponentSpec.json -n TestAppBlueprint`` + + +Extra information +----------------- + +1. The component spec must be compliant with `Component Spec json schema `__ + +2. If the flag is marked required then the corresponding values must be provided for blueprint-generator execution + +3. If the flag is identified as optional then it is not mandatory for blueprint-generator execution + +4. If you do not add a -n flag the blueprint name will default to what it is in the component spec + +5. If the directory you specified in the -p flag does not already exist the directory will be created for you + +6. The -t flag will override the default imports set for the blueprints. To see an example of how the import yaml file should be structured see the testImports.yaml file under the folder TestCases + + +How to create policy models: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. Policy model creation can be done with the same jar as downloaded for the blueprint generation. + +2. Run the same command as the blueprint generator except replace the ``blueprint`` positional with ``policy`` + +3. Example command + + ``java -jar blueprint-generator-1.3.1-executable.jar policy`` + +4. Options + + - -i: The path to the JSON spec file (required) + - -p: The Output path for all of the models (required) + + +Extra information +----------------- + +1. Not all component specs will be able to create policy models + +2. Multiple policy model files may be created from a single component spec \ No newline at end of file diff --git a/docs/sections/design-components/component-specification/component-json-schema.rst b/docs/sections/design-components/component-specification/component-json-schema.rst new file mode 100644 index 00000000..18139598 --- /dev/null +++ b/docs/sections/design-components/component-specification/component-json-schema.rst @@ -0,0 +1,933 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +.. _dcae-component-schema: + +Component JSON Schema Definition +================================ + +The schema file used for DCAE onboarding is maintained in `gerrit `__ +The same is provided below for documentation reference. + + +:: + + { + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Component specification schema", + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "version": { + "$ref": "#/definitions/version" + }, + "description": { + "type": "string" + }, + "component_type": { + "type": "string", + "enum": [ + "docker", + "cdap" + ] + }, + "name": { + "$ref": "#/definitions/name" + } + }, + "required": [ + "version", + "name", + "description", + "component_type" + ] + }, + "streams": { + "type": "object", + "properties": { + "publishes": { + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + { "$ref": "#/definitions/publisher_http" }, + { "$ref": "#/definitions/publisher_message_router" }, + { "$ref": "#/definitions/publisher_data_router" } + ] + } + }, + "subscribes": { + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + { "$ref": "#/definitions/subscriber_http" }, + { "$ref": "#/definitions/subscriber_message_router" }, + { "$ref": "#/definitions/subscriber_data_router" } + ] + } + } + }, + "required": [ + "publishes", + "subscribes" + ] + }, + "services": { + "type": "object", + "properties": { + "calls": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/caller" + } + }, + "provides": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/provider" + } + } + }, + "required": [ + "calls", + "provides" + ] + }, + "parameters" : { + "anyOf" : [ + {"$ref": "#/definitions/docker-parameters"}, + {"$ref": "#/definitions/cdap-parameters"} + ] + }, + "auxilary": { + "oneOf" : [ + {"$ref": "#/definitions/auxilary_cdap"}, + {"$ref": "#/definitions/auxilary_docker"} + ] + }, + "artifacts": { + "type": "array", + "description": "List of component artifacts", + "items": { + "$ref": "#/definitions/artifact" + } + }, + "policy_info": { + "type": "object", + "properties": { + "policy": + { + "type": "array", + "items": + { + "type": "object", + "properties": + { + "node_label": + { + "type": "string" + }, + "policy_id": + { + "type": "string" + }, + "policy_model_id": + { + "type": "string" + } + }, + "required": ["node_label", "policy_model_id"] + } + } + }, + "additionalProperties": false + } + }, + "required": [ + "self", + "streams", + "services", + "parameters", + "auxilary", + "artifacts" + ], + "additionalProperties": false, + "definitions": { + "cdap-parameters": { + "description" : "There are three seperate ways to pass parameters to CDAP: app config, app preferences, program preferences. These are all treated as optional.", + "type": "object", + "properties" : { + "program_preferences": { + "description" : "A list of {program_id, program_type, program_preference} objects where program_preference is an object passed into program_id of type program_type", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/program_preference" + } + }, + "app_preferences" : { + "description" : "Parameters Passed down to the CDAP preference API", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + }, + "app_config" : { + "description" : "Parameters Passed down to the CDAP App Config", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + } + } + }, + "program_preference": { + "type": "object", + "properties": { + "program_type": { + "$ref": "#/definitions/program_type" + }, + "program_id": { + "type": "string" + }, + "program_pref":{ + "description" : "Parameters that the CDAP developer wants pushed to this program's preferences API. Optional", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + } + }, + "required": ["program_type", "program_id", "program_pref"] + }, + "program_type": { + "type": "string", + "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] + }, + "docker-parameters": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + }, + "parameter": { + "oneOf": [ + {"$ref": "#/definitions/parameter-list"}, + {"$ref": "#/definitions/parameter-other"} + ] + }, + "parameter-list": { + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "Only valid type is list, the entry_schema is required - which contains the type of the list element. All properties set for the parameter apply to all elements in the list at this time", + "type": "string", + "enum": ["list"] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "entry_schema": { + "description": "The optional property used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is list. This is the only type it is currently supported for.", + "type": "object", + "uniqueItems": true, + "items": {"$ref": "#/definitions/list-parameter"} + }, + "designer_editable": { + "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", + "type": "boolean" + }, + "sourced_at_deployment": { + "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", + "type": "boolean" + }, + "policy_editable": { + "description": "A required property that declares a parameter as editable by DevOps in Policy UI (true) or not (false).", + "type": "boolean" + }, + "policy_group": { + "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", + "type": "string" + }, + "policy_schema" :{ + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/policy_schema_parameter"} + } + }, + "required": [ + "name", + "value", + "description", + "designer_editable", + "policy_editable", + "sourced_at_deployment", + "entry_schema" + ], + "additionalProperties": false, + "dependencies": { + "policy_schema": ["policy_editable"] + } + }, + "parameter-other": { + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "The required data type for the parameter.", + "type": "string", + "enum": [ "string", "number", "boolean", "datetime" ] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "designer_editable": { + "description": "A required property that declares a parameter as editable by designer in SDC Tool (true) or not (false).", + "type": "boolean" + }, + "sourced_at_deployment": { + "description": "A required property that declares that a parameter is assigned at deployment time (true) or not (false).", + "type": "boolean" + }, + "policy_editable": { + "description": "A required property that declares a parameter as editable in Policy UI (true) or not (false).", + "type": "boolean" + }, + "policy_group": { + "description": "An optional property used to group policy_editable parameters into groups. Each group will become it's own policy model. Any parameters without this property will be grouped together to form their own policy model", + "type": "string" + }, + "policy_schema" :{ + "description": "An optional property used to define policy_editable parameters as lists or maps", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/policy_schema_parameter"} + } + }, + "required": [ + "name", + "value", + "description", + "designer_editable", + "sourced_at_deployment", + "policy_editable" + ], + "additionalProperties": false, + "dependencies": { + "policy_schema": ["policy_editable"] + } + }, + "list-parameter": { + "type": "object", + "properties": { + "type": { + "description": "The required data type for each parameter in the list.", + "type": "string", + "enum": ["string", "number"] + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "policy_schema_parameter": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "description": "Default value for the parameter" + }, + "description": { + "description": "Description for the parameter.", + "type": "string" + }, + "type": { + "description": "The required data type for the parameter.", + "type": "string", + "enum": [ "string", "number", "boolean", "datetime", "list", "map" ] + }, + "required": { + "description": "An optional key that declares a parameter as required (true) or not (false). Default is true.", + "type": "boolean", + "default": true + }, + "constraints": { + "description": "The optional list of sequenced constraint clauses for the parameter.", + "type": "array", + "items": { + "$ref": "#/definitions/parameter-constraints" + } + }, + "entry_schema": { + "description": "The optional key that is used to declare the name of the Datatype definition for entries of certain types. entry_schema must be defined when the type is either list or map. If the type is list and the entry type is a simple type (string, number, boolean, datetime), follow with a simple string to describe the entry type. If the type is list and the entry type is a map, follow with an array to describe the keys for the entry map. If the type is list and the entry type is also list, this is not currently supported here. If the type is map, then follow with an array to describe the keys for this map. ", + "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/policy_schema_parameter"} + } + }, + "required": [ + "name", + "type" + ], + "additionalProperties": false + }, + "parameter-constraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "equal": { + "description": "Constrains a property or parameter to a value equal to (‘=’) the value declared." + }, + "greater_than": { + "description": "Constrains a property or parameter to a value greater than (‘>’) the value declared.", + "type": "number" + }, + "greater_or_equal": { + "description": "Constrains a property or parameter to a value greater than or equal to (‘>=’) the value declared.", + "type": "number" + }, + "less_than": { + "description": "Constrains a property or parameter to a value less than (‘<’) the value declared.", + "type": "number" + }, + "less_or_equal": { + "description": "Constrains a property or parameter to a value less than or equal to (‘<=’) the value declared.", + "type": "number" + }, + "valid_values": { + "description": "Constrains a property or parameter to a value that is in the list of declared values.", + "type": "array" + }, + "length": { + "description": "Constrains the property or parameter to a value of a given length.", + "type": "number" + }, + "min_length": { + "description": "Constrains the property or parameter to a value to a minimum length.", + "type": "number" + }, + "max_length": { + "description": "Constrains the property or parameter to a value to a maximum length.", + "type": "number" + } + } + }, + "stream_message_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "message router", "message_router" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "publisher_http": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "http", + "https" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "publisher_message_router": { + "$ref": "#/definitions/stream_message_router" + }, + "publisher_data_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "config_key": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "data router", "data_router" + ] + } + }, + "required": [ + "format", + "version", + "config_key", + "type" + ] + }, + "subscriber_http": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "route": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "http", + "https" + ] + } + }, + "required": [ + "format", + "version", + "route", + "type" + ] + }, + "subscriber_message_router": { + "$ref": "#/definitions/stream_message_router" + }, + "subscriber_data_router": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + }, + "route": { + "type": "string" + }, + "type": { + "description": "Type of stream to be used", + "type": "string", + "enum": [ + "data router", "data_router" + ] + }, + "config_key": { + "description": "Data router subscribers require config info to setup their endpoints to handle requests. For example, needs username and password", + "type": "string" + } + }, + "required": [ + "format", + "version", + "route", + "type", + "config_key" + ] + }, + "provider" : { + "oneOf" : [ + {"$ref": "#/definitions/docker-provider"}, + {"$ref": "#/definitions/cdap-provider"} + ] + }, + "cdap-provider" : { + "type": "object", + "properties" : { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "service_name" : { + "type" : "string" + }, + "service_endpoint" : { + "type" : "string" + }, + "verb" : { + "type": "string", + "enum": ["GET", "PUT", "POST", "DELETE"] + } + }, + "required" : [ + "request", + "response", + "service_name", + "service_endpoint", + "verb" + ] + }, + "docker-provider": { + "type": "object", + "properties": { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "route": { + "type": "string" + }, + "verb": { + "type": "string", + "enum": ["GET", "PUT", "POST", "DELETE"] + } + }, + "required": [ + "request", + "response", + "route" + ] + }, + "caller": { + "type": "object", + "properties": { + "request": { + "$ref": "#/definitions/formatPair" + }, + "response": { + "$ref": "#/definitions/formatPair" + }, + "config_key": { + "type": "string" + } + }, + "required": [ + "request", + "response", + "config_key" + ] + }, + "formatPair": { + "type": "object", + "properties": { + "format": { + "$ref": "#/definitions/name" + }, + "version": { + "$ref": "#/definitions/version" + } + } + }, + "name": { + "type": "string" + }, + "version": { + "type": "string", + "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + }, + "artifact": { + "type": "object", + "description": "Component artifact object", + "properties": { + "uri": { + "type": "string", + "description": "Uri to artifact" + }, + "type": { + "type": "string", + "enum": ["jar", "docker image"] + } + }, + "required": ["uri", "type"] + }, + + "auxilary_cdap": { + "title": "cdap component specification schema", + "type": "object", + "properties": { + "streamname": { + "type": "string" + }, + "artifact_name" : { + "type": "string" + }, + "artifact_version" : { + "type": "string", + "pattern": "^(\\d+\\.)(\\d+\\.)(\\*|\\d+)$" + }, + "namespace":{ + "type": "string", + "description" : "optional" + }, + "programs": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/cdap_program" + } + } + }, + "required": [ + "streamname", + "programs", + "artifact_name", + "artifact_version" + ] + }, + "cdap_program_type": { + "type": "string", + "enum": ["flows","mapreduce","schedules","spark","workflows","workers","services"] + }, + "cdap_program": { + "type": "object", + "properties": { + "program_type": { + "$ref": "#/definitions/cdap_program_type" + }, + "program_id": { + "type": "string" + } + }, + "required": ["program_type", "program_id"] + }, + + "auxilary_docker": { + "title": "Docker component specification schema", + "type": "object", + "properties": { + "healthcheck": { + "description": "Define the health check that Consul should perfom for this component", + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/docker_healthcheck_http" }, + { "$ref": "#/definitions/docker_healthcheck_script" } + ] + }, + "ports": { + "description": "Port mapping to be used for Docker containers. Each entry is of the format :.", + "type": "array", + "items": { + "type": "string" + } + }, + "log_info": { + "description": "Component specific details for logging", + "type": "object", + "properties": { + "log_directory": { + "description": "The path in the container where the component writes its logs. If the component is following the EELF requirements, this would be the directory where the four EELF files are being written. (Other logs can be placed in the directory--if their names in '.log', they'll also be sent into ELK.)", + "type": "string" + }, + "alternate_fb_path": { + "description": "By default, the log volume is mounted at /var/log/onap/ in the sidecar container's file system. 'alternate_fb_path' allows overriding the default. Will affect how the log data can be found in the ELK system.", + "type": "string" + } + }, + "additionalProperties": false + }, + "tls_info": { + "description": "Component information to use tls certificates", + "type": "object", + "properties": { + "cert_directory": { + "description": "The path in the container where the component certificates will be placed by the init container", + "type": "string" + }, + "use_tls": { + "description": "Boolean flag to determine if the application is using tls certificates", + "type": "boolean" + } + }, + "required": [ + "cert_directory","use_tls" + ], + "additionalProperties": false + }, + "databases": { + "description": "The databases the application is connecting to using the pgaas", + "type": "object", + "additionalProperties": { + "type": "string", + "enum": [ + "postgres" + ] + } + }, + "policy": { + "properties": { + "trigger_type": { + "description": "Only value of docker is supported at this time.", + "type": "string", + "enum": ["docker"] + }, + "script_path": { + "description": "Script command that will be executed for policy reconfiguration", + "type": "string" + } + }, + "required": [ + "trigger_type","script_path" + ], + "additionalProperties": false + }, + "volumes": { + "description": "Volume mapping to be used for Docker containers. Each entry is of the format below", + "type": "array", + "items": { + "type": "object", + "properties": { + "host":{ + "type":"object", + "path": {"type": "string"} + }, + "container":{ + "type":"object", + "bind": { "type": "string"}, + "mode": { "type": "string"} + } + } + } + } + }, + "required": [ + "healthcheck" + ], + "additionalProperties": false + }, + "docker_healthcheck_http": { + "properties": { + "type": { + "description": "Consul health check type", + "type": "string", + "enum": [ + "http", + "https" + ] + }, + "interval": { + "description": "Interval duration in seconds i.e. 10s", + "default": "15s", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds i.e. 10s", + "default": "1s", + "type": "string" + }, + "endpoint": { + "description": "Relative endpoint used by Consul to check health by making periodic HTTP GET calls", + "type": "string" + } + }, + "required": [ + "type", + "endpoint" + ] + }, + "docker_healthcheck_script": { + "properties": { + "type": { + "description": "Consul health check type", + "type": "string", + "enum": [ + "script", + "docker" + ] + }, + "interval": { + "description": "Interval duration in seconds i.e. 10s", + "default": "15s", + "type": "string" + }, + "timeout": { + "description": "Timeout in seconds i.e. 10s", + "default": "1s", + "type": "string" + }, + "script": { + "description": "Script command that will be executed by Consul to check health", + "type": "string" + } + }, + "required": [ + "type", + "script" + ] + } + } + } + diff --git a/docs/sections/design-components/component-specification/component-type-docker.rst b/docs/sections/design-components/component-specification/component-type-docker.rst new file mode 100755 index 00000000..70412c0f --- /dev/null +++ b/docs/sections/design-components/component-specification/component-type-docker.rst @@ -0,0 +1,1437 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + + +.. _component_specification: + +What is Component Specification? +================================ + +This page will discuss categories defined in :any:`component specification schema ` and their usage. + + +Meta Schema Definition +---------------------- + + +The “Meta Schema” implementation defines how component specification +JSON schemas can be written to define user input. It is itself a JSON +schema (thus it is a “meta schema”). It requires the name of the +component entry, component type (either ‘cdap’ or ‘docker’) and a +description under “self” object. The meta schema version must be +specified as the value of the “version” key. Then the input schema +itself is described. + +There are four types of schema descriptions objects - jsonschema for +inline standard JSON Schema definitions of JSON inputs, delimitedschema +for delimited data input using a JSON description defined by AT&T, +unstructured for unstructured text, and reference that allows a pointer +to another artifact for a schema. The reference allows for XML and Protocol Buffer schema, +but can be used as a pointer to JSON, Delimited Format, and Unstructured +schemas as well. + +.. _metadata: + +Component Metadata +------------------ + +Metadata refers to the properties found under the ``self`` JSON. This +group of properties is used to uniquely identify this component +specification and identify the component that this specification is used +to capture. + +Example: + +:: + + "self": { + "version": "1.0.0", + "name": "yourapp.component.kpi_anomaly", + "description": "Classifies VNF KPI data as anomalous", + "component_type": "docker" + }, + +``self`` Schema: + ++-------------+--------+----------------+ +| Property | Type | Description | +| Name | | | ++=============+========+================+ +| version | string | *Required*. | +| | | Semantic | +| | | version | +| | | for this | +| | | specification | ++-------------+--------+----------------+ +| name | string | *Required*. | +| | | Full | +| | | name of | +| | | this | +| | | component | +| | | which is | +| | | also | +| | | used as | +| | | this | +| | | component's | +| | | catalog | +| | | id. | ++-------------+--------+----------------+ +| description | string | *Required* | +| | | Human-readable | +| | | text | +| | | describing | +| | | the | +| | | component | +| | | and the | +| | | components | +| | | functional | +| | | purpose. | ++-------------+--------+----------------+ +| component_t\| string | *Required* | +| ype | | Identify | +| | | what | +| | | containe\ | +| | | rization | +| | | technolo\ | +| | | gy | +| | | this | +| | | componen\ | +| | | t | +| | | uses: | +| | | *docker* | +| | | or | +| | | *cdap*. | +| | | | ++-------------+--------+----------------+ + +.. _interfaces: + +Interfaces +---------- + +Interfaces are the JSON objects found under the ``streams`` key and the +``services`` key. These are used to describe the interfaces that the +component uses and the interfaces that the component provides. The +description of each interface includes the associated :any:`data +format `. + +Streams +~~~~~~~ + +- The ``streams`` JSON is for specifying data produced for consumption + by other components, and the streams expected to subscribe to that is + produced by other components. These are “fire and forget” type + interfaces where the publisher of a stream does not expect or parse a + response from the subscriber. +- The term ``stream`` here is abstract and neither refers to “CDAP + streams” or “DMaaP feeds”. While a stream is very likely a DMaaP + feed, it could be a direct stream of data being routed via HTTP too. + It abstractly refers to a sequence of data leaving a publisher. +- Streams have anonymous publish/subscribe semantics, which decouples + the production of information from its consumption. Like the component + specification, the data format specification is represented/validated against this + `Data Format json schema `__ +- In general, components are not aware of who they are communicating + with. +- Instead, components that are interested in data, subscribe to the + relevant stream; components that generate data publish to the + relevant stream. +- There can be multiple publishers and subscribers to a stream. Streams + are intended for unidirectional, streaming communication. + +Streams interfaces that implement an HTTP endpoint must support POST. + +Streams are split into: + ++-------------+----+----------+ +| Property | Ty\| Descript\| +| Name | pe | ion | ++=============+====+==========+ +| subscribes | JS\| *Require\| +| | ON | d*. | +| | li\| List of | +| | st | all | +| | | availabl\| +| | | e | +| | | stream | +| | | interfac\| +| | | es | +| | | that | +| | | this | +| | | componen\| +| | | t | +| | | has that | +| | | can be | +| | | used for | +| | | subscrib\| +| | | ing | ++-------------+----+----------+ +| publishes | JS\| *Require\| +| | ON | d*. | +| | li\| List of | +| | st | all | +| | | stream | +| | | interfac\| +| | | es | +| | | that | +| | | this | +| | | componen\| +| | | t | +| | | will | +| | | publish | +| | | onto | ++-------------+----+----------+ + +Subscribes +^^^^^^^^^^ + +Example: + +.. code:: json + + "streams": { + "subscribes": [{ + "format": "dcae.vnf.kpi", + "version": "1.0.0", + "route": "/data", // for CDAP this value is not used + "type": "http" + }], + ... + } + +This describes that ``yourapp.component.kpi_anomaly`` exposes an HTTP +endpoint called ``/data`` which accepts requests that have the data +format of ``dcae.vnf.kpi`` version ``1.0.0``. + +``subscribes`` Schema: + ++-------------+----+--------------------+ +| Property | Ty\| Descript\ | +| Name | pe | ion | ++=============+====+====================+ +| format | st\| *Require\ | +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | id of | +| | | the data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\ | +| | | e | ++-------------+----+--------------------+ +| version | st\| *Require\ | +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | version | +| | | of the | +| | | data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\ | +| | | e | ++-------------+----+--------------------+ +| route | st\| *Require\ | +| | ri\| d | +| | ng | for HTTP | +| | | and data | +| | | router*. | +| | | The HTTP | +| | | route | +| | | that | +| | | this | +| | | interfac\ | +| | | e | +| | | listens | +| | | on | ++-------------+----+--------------------+ +| config_key | st\| *Require\ | +| | ri\| d \ | +| | ng | for \ | +| | | message_router\ | +| | | and data \ | +| | | router*. | +| | | The HTTP | +| | | route | +| | | that | +| | | this | +| | | interfac\ | +| | | e | +| | | listens | +| | | on | ++-------------+----+--------------------+ +| type | st\| *Require\ | +| | ri\| d*. | +| | ng | Type of | +| | | stream: | +| | | ``http`` | +| | | , | +| | | ``message_router`` | +| | | , | +| | | ``data_router`` | ++-------------+----+--------------------+ + +.. _message-router: + +Message router +'''''''''''''' + +Message router subscribers are http clients rather than http services +and performs a http a ``GET`` call. Thus, message router subscribers +description is structured like message router publishers and requires +``config_key``: + +.. code:: json + + "streams": { + "subscribes": [{ + "format": "dcae.some-format", + "version": "1.0.0", + "config_key": "some_format_handle", + "type": "message router" + }], + ... + } + + +.. _data-router: + +Data router +''''''''''' + +Data router subscribers are http or https services that handle ``PUT`` +requests from data router. Developers must provide the ``route`` or url +path/endpoint that is expected to handle data router requests. This will +be used to construct the delivery url needed to register the subscriber +to the provisioned feed. Developers must also provide a ``config_key`` +because there is dynamic configuration information associated with the +feed that the application will need e.g. username and password. See the +page on :any:`DMaaP connection objects ` for more details on +the configuration information. + +Example (not tied to the larger example): + +.. code:: json + + "streams": { + "subscribes": [{ + "config_key": "some-sub-dr", + "format": "sandbox.platform.any", + "route": "/identity", + "type": "data_router", + "version": "0.1.0" + }], + ... + } + +Publishes +^^^^^^^^^ + +Example: + +.. code:: json + + "streams": { + ... + "publishes": [{ + "format": "yourapp.format.integerClassification", + "version": "1.0.0", + "config_key": "prediction", + "type": "http" + }] + }, + +This describes that ``yourapp.component.kpi_anomaly`` publishes by making +POST requests to streams that support the data format +``yourapp.format.integerClassification`` version ``1.0.0``. + +``publishes`` Schema: + ++-------------+----+--------------------+ +| Property | Ty\| Descript\ | +| Name | pe | ion | ++=============+====+====================+ +| format | st\| *Require\ | +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | id of | +| | | the data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\ | +| | | e | ++-------------+----+--------------------+ +| version | st\| *Require\ | +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | version | +| | | of the | +| | | data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\ | +| | | e | ++-------------+----+--------------------+ +| config_key | st\| *Require\ | +| | ri\| d*. | +| | ng | The JSON | +| | | key in | +| | | the | +| | | generate\ | +| | | d | +| | | applicat | +| | | ion | +| | | configur\ | +| | | ation | +| | | that | +| | | will be | +| | | used to | +| | | pass the | +| | | downstre\ | +| | | am | +| | | componen\ | +| | | t’s | +| | | (the | +| | | subscrib\ | +| | | er’s) | +| | | connecti\ | +| | | on | +| | | informat\ | +| | | ion. | ++-------------+----+--------------------+ +| type | st\| *Require\ | +| | ri\| d*. | +| | ng | Type of | +| | | stream: | +| | | ``http`` | +| | | , | +| | | ``message_router`` | +| | | , | +| | | ``data_router`` | ++-------------+----+--------------------+ + +.. message-router-1: + +Message router +'''''''''''''' + +Message router publishers are http clients of DMaap message_router. +Developers must provide a ``config_key`` because there is dynamic +configuration information associated with the feed that the application +needs to receive e.g. topic url, username, password. See the page on +:any:`DMaaP connection objects ` for more details on +the configuration information. + +Example (not tied to the larger example): + +.. code:: json + + "streams": { + ... + "publishes": [{ + "config_key": "some-pub-mr", + "format": "sandbox.platform.any", + "type": "message_router", + "version": "0.1.0" + }] + } + +.. data-router-1: + +Data router +''''''''''' + +Data router publishers are http clients that make ``PUT`` requests to +data router. Developers must also provide a ``config_key`` because there +is dynamic configuration information associated with the feed that the +application will need to receive e.g. publish url, username, password. +See the page on :any:`DMaaP connection objects ` for more details on +the configuration information. + +Example (not tied to the larger example): + +.. code:: json + + "streams": { + ... + "publishes": [{ + "config_key": "some-pub-dr", + "format": "sandbox.platform.any", + "type": "data_router", + "version": "0.1.0" + }] + } + +Quick Reference +^^^^^^^^^^^^^^^ + +Refer to this :doc:`Quick Reference ` for a +comparison of the Streams ‘Publishes’ and ‘Subscribes’ sections. + +Services +~~~~~~~~ + +- The publish / subscribe model is a very flexible communication + paradigm, but its many-to-many one-way transport is not appropriate + for RPC request / reply interactions, which are often required in a + distributed system. +- Request / reply is done via a Service, which is defined by a pair of + messages: one for the request and one for the reply. + +Services are split into: + ++-------------+----+----------+ +| Property | Ty\| Descript\| +| Name | pe | ion | ++=============+====+==========+ +| calls | JS\| *Require\| +| | ON | d*. | +| | li\| List of | +| | st | all | +| | | service | +| | | interfac\| +| | | es | +| | | that | +| | | this | +| | | componen\| +| | | t | +| | | will | +| | | call | ++-------------+----+----------+ +| provides | JS\| *Require\| +| | ON | d*. | +| | li\| List of | +| | st | all | +| | | service | +| | | interfac\| +| | | es | +| | | that | +| | | this | +| | | componen\| +| | | t | +| | | exposes | +| | | and | +| | | provides | ++-------------+----+----------+ + +Calls +^^^^^ + +The JSON ``services/calls`` is for specifying that the component relies +on an HTTP(S) service—the component sends that service an HTTP request, +and that service responds with an HTTP reply. An example of this is how +string matching (SM) depends on the AAI Broker. SM performs a +synchronous REST call to the AAI broker, providing it the VMNAME of the +VNF, and the AAI Broker responds with additional details about the VNF. +This dependency is expressed via ``services/calls``. In contrast, the +output of string matching (the alerts it computes) is sent directly to +policy as a fire-and-forget interface, so that is an example of a +``stream``. + +Example: + +.. code:: json + + "services": { + "calls": [{ + "config_key": "vnf-db", + "request": { + "format": "dcae.vnf.meta", + "version": "1.0.0" + }, + "response": { + "format": "dcae.vnf.kpi", + "version": "1.0.0" + } + }], + ... + } + +This describes that ``yourapp.component.kpi_anomaly`` will make HTTP +calls to a downstream component that accepts requests of data format +``dcae.vnf.meta`` version ``1.0.0`` and is expecting the response to be +``dcae.vnf.kpi`` version ``1.0.0``. + +``calls`` Schema: + ++-------------+----+----------+ +| Property | Ty\| Descript\| +| Name | pe | ion | ++=============+====+==========+ +| request | JS\| *Require\| +| | ON | d*. | +| | ob\| Descript\| +| | je\| ion | +| | ct | of the | +| | | expected | +| | | request | +| | | for this | +| | | downstre\| +| | | am | +| | | interfac\| +| | | e | ++-------------+----+----------+ +| response | JS\| *Require\| +| | ON | d*. | +| | ob\| Descript\| +| | je\| ion | +| | ct | of the | +| | | expected | +| | | response | +| | | for this | +| | | downstre\| +| | | am | +| | | interfac\| +| | | e | ++-------------+----+----------+ +| config_key | st\| *Require\| +| | ri\| d*. | +| | ng | The JSON | +| | | key in | +| | | the | +| | | generate\| +| | | d | +| | | applicat | +| | | ion | +| | | configur\| +| | | ation | +| | | that | +| | | will be | +| | | used to | +| | | pass the | +| | | downstre\| +| | | am | +| | | componen | +| | | t | +| | | connecti\| +| | | on | +| | | informat\| +| | | ion. | ++-------------+----+----------+ + +The JSON object schema for both ``request`` and ``response``: + ++-------------+----+----------+ +| Property | Ty\| Descript\| +| Name | pe | ion | ++=============+====+==========+ +| format | st\| *Require\| +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | id of | +| | | the data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\| +| | | e | ++-------------+----+----------+ +| version | st\| *Require\| +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | version | +| | | of the | +| | | data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\| +| | | e | ++-------------+----+----------+ + +Provides +^^^^^^^^ + +Example: + +.. code:: json + + "services": { + ... + "provides": [{ + "route": "/score-vnf", + "request": { + "format": "dcae.vnf.meta", + "version": "1.0.0" + }, + "response": { + "format": "yourapp.format.integerClassification", + "version": "1.0.0" + } + }] + }, + +This describes that ``yourapp.component.kpi_anomaly`` provides a service +interface and it is exposed on the ``/score-vnf`` HTTP endpoint. The +endpoint accepts requests that have the data format ``dcae.vnf.meta`` +version ``1.0.0`` and gives back a response of +``yourapp.format.integerClassification`` version ``1.0.0``. + +``provides`` Schema for a Docker component: + ++-------------+----+----------+ +| Property | Ty\| Descript\| +| Name | pe | ion | ++=============+====+==========+ +| request | JS\| *Require\| +| | ON | d*. | +| | ob\| Descript\| +| | je\| ion | +| | ct | of the | +| | | expected | +| | | request | +| | | for this | +| | | interfac\| +| | | e | ++-------------+----+----------+ +| response | JS\| *Require\| +| | ON | d*. | +| | ob\| Descript\| +| | je\| ion | +| | ct | of the | +| | | expected | +| | | response | +| | | for this | +| | | interfac\| +| | | e | ++-------------+----+----------+ +| route | st\| *Require\| +| | ri\| d*. | +| | ng | The HTTP | +| | | route | +| | | that | +| | | this | +| | | interfac\| +| | | e | +| | | listens | +| | | on | ++-------------+----+----------+ + +The JSON object schema for both ``request`` and ``response``: + ++-------------+----+----------+ +| Property | Ty\| Descript\| +| Name | pe | ion | ++=============+====+==========+ +| format | st\| *Require\| +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | id of | +| | | the data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\| +| | | e | ++-------------+----+----------+ +| version | st\| *Require\| +| | ri\| d*. | +| | ng | Data | +| | | format | +| | | version | +| | | of the | +| | | data | +| | | format | +| | | that is | +| | | used by | +| | | this | +| | | interfac\| +| | | e | ++-------------+----+----------+ + +Note, for CDAP, there is a slight variation due to the way CDAP exposes +services: + +:: + + "provides":[ // note this is a list of JSON + { + "request":{ ...}, + "response":{ ...}, + "service_name":"name CDAP service", + "service_endpoint":"greet", // E.g the URL is /services/service_name/methods/service_endpoint + "verb":"GET" // GET, PUT, or POST + } + ] + +``provides`` Schema for a CDAP component: + ++-------------+----+-----------+ +| Property | Ty\| Descript\ | +| Name | pe | ion | ++=============+====+===========+ +| request | JS\| *Require\ | +| | ON | d*. | +| | ob\| Descript\ | +| | je\| ion | +| | ct | of the | +| | | expected | +| | | request | +| | | data | +| | | format | +| | | for this | +| | | interfac\ | +| | | e | ++-------------+----+-----------+ +| response | JS\| *Require\ | +| | ON | d*. | +| | ob\| Descript\ | +| | je\| ion | +| | ct | of the | +| | | expected | +| | | response | +| | | for this | +| | | interfac\ | +| | | e | ++-------------+----+-----------+ +| service_nam\| st\| *Require\ | +| e | ri\| d*. | +| | ng | The CDAP | +| | | service | +| | | name (eg | +| | | “Greetin\ | +| | | g”) | ++-------------+----+-----------+ +| service_end | st\| *Require\ | +| point | ri\| d*. | +| | ng | The CDAP | +| | | service | +| | | endpoint | +| | | for this | +| | | service_n\| +| | | ame | +| | | (eg | +| | | “/greet” | +| | | ) | ++-------------+----+-----------+ +| verb | st\| *Require\ | +| | ri\| d*. | +| | ng | ‘GET’, | +| | | ‘PUT’ or | +| | | ‘POST’ | ++-------------+----+-----------+ + +.. _common-specification-parameters: + +Parameters +---------- + +``parameters`` is where to specify the component’s application +configuration parameters that are not connection information. + ++---------------+------------+----------------------------------+ +| Property Name | Type | Description | ++===============+============+==================================+ +| parameters | JSON array | Each entry is a parameter object | ++---------------+------------+----------------------------------+ + +Parameter object has the following available properties: + ++--------------+----+----------+------+ +| Property | Ty\| Descript\| Defa\| +| Name | pe | ion | ult | ++==============+====+==========+======+ +| name | st\| *Require\| | +| | ri\| d*. | | +| | ng | The | | +| | | property | | +| | | name | | +| | | that | | +| | | will be | | +| | | used as | | +| | | the key | | +| | | in the | | +| | | generate\| | +| | | d | | +| | | config | | ++--------------+----+----------+------+ +| value | an\| *Require\| | +| | y | d*. | | +| | | The | | +| | | default | | +| | | value | | +| | | for the | | +| | | given | | +| | | paramete\| | +| | | r | | ++--------------+----+----------+------+ +| description | st\| *Require\| | +| | ri\| d*. | | +| | ng | Human-re\| | +| | | adable | | +| | | text | | +| | | describi\| | +| | | ng | | +| | | the | | +| | | paramete\| | +| | | r | | +| | | like | | +| | | what its | | +| | | for | | ++--------------+----+----------+------+ +| type | st\| The | | +| | ri\| required | | +| | ng | data | | +| | | type for | | +| | | the | | +| | | paramete\| | +| | | r | | ++--------------+----+----------+------+ +| required | bo\| An | true | +| | ol\| optional | | +| | ea\| key that | | +| | n | declares | | +| | | a | | +| | | paramete\| | +| | | r | | +| | | as | | +| | | required | | +| | | (true) | | +| | | or not | | +| | | (false) | | ++--------------+----+----------+------+ +| constraints | ar\| The | | +| | ra\| optional | | +| | y | list of | | +| | | sequence | | +| | | d | | +| | | constrai\| | +| | | nt | | +| | | clauses | | +| | | for the | | +| | | paramete\| | +| | | r. | | +| | | See | | +| | | below | | ++--------------+----+----------+------+ +| entry_schem\ | st\| The | | +| a | ri\| optional | | +| | ng | key that | | +| | | is used | | +| | | to | | +| | | declare | | +| | | the name | | +| | | of the | | +| | | Datatype | | +| | | definiti\| | +| | | on | | +| | | for | | +| | | entries | | +| | | of set | | +| | | types | | +| | | such as | | +| | | the | | +| | | TOSCA | | +| | | ‘list’ | | +| | | or | | +| | | ‘map’. | | +| | | Only 1 | | +| | | level is | | +| | | supporte\| | +| | | d | | +| | | at this | | +| | | time | | ++--------------+----+----------+------+ +| designer_ed\ | bo\| An | | +| itable | ol\| optional | | +| | ea\| key that | | +| | n | declares | | +| | | a | | +| | | paramete\| | +| | | r | | +| | | to be | | +| | | editable | | +| | | by | | +| | | designer | | +| | | (true) | | +| | | or not | | +| | | (false) | | ++--------------+----+----------+------+ +| sourced_at_d\| bo\| An | | +| eployment | ol\| optional | | +| | ea\| key that | | +| | n | declares | | +| | | a | | +| | | paramete\| | +| | | r’s | | +| | | value to | | +| | | be | | +| | | assigned | | +| | | at | | +| | | deployme\| | +| | | nt | | +| | | time | | +| | | (true) | | ++--------------+----+----------+------+ +| policy_edit\ | bo\| An | | +| able | ol\| optional | | +| | ea\| key that | | +| | n | declares | | +| | | a | | +| | | paramete\| | +| | | r | | +| | | to be | | +| | | editable | | +| | | by | | +| | | policy | | +| | | (true) | | +| | | or not | | +| | | (false) | | ++--------------+----+----------+------+ +| policy_sche\ | ar\| The | | +| ma | ra\| optional | | +| | y | list of | | +| | | schema | | +| | | definiti\| | +| | | ons | | +| | | used for | | +| | | policy. | | +| | | See | | +| | | below | | ++--------------+----+----------+------+ + +Example: + +.. code:: json + + "parameters": [ + { + "name": "threshold", + "value": 0.75, + "description": "Probability threshold to exceed to be anomalous" + } + ] + +Many of the parameter properties have been copied from TOSCA model +property definitions and are to be used for service design composition +and policy creation. See `section 3.5.8 *Property +definition* `__. + +The property ``constraints`` is a list of objects where each constraint +object: + ++--------------+----+----------+ +| Property | Ty\| Descript\| +| Name | pe | ion | ++==============+====+==========+ +| equal | | Constrai\| +| | | ns | +| | | a | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value | +| | | equal to | +| | | (‘=’) | +| | | the | +| | | value | +| | | declared | ++--------------+----+----------+ +| greater_tha\ | nu\| Constrai\| +| n | mb\| ns | +| | er | a | +| | | property | +| | | or | +| | | paramete | +| | | r | +| | | to a | +| | | value | +| | | greater | +| | | than | +| | | (‘>’) | +| | | the | +| | | value | +| | | declared | ++--------------+----+----------+ +| greater_or_e\| nu\| Constrai\| +| qual | mb\| ns | +| | er | a | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value | +| | | greater | +| | | than or | +| | | equal to | +| | | (‘>=’) | +| | | the | +| | | value | +| | | declared | ++--------------+----+----------+ +| less_than | nu\| Constrai\| +| | mb\| ns | +| | er | a | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value | +| | | less | +| | | than | +| | | (‘<’) | +| | | the | +| | | value | +| | | declared | ++--------------+----+----------+ +| less_or_equ\ | nu\| Constrai\| +| al | mb\| ns | +| | er | a | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value | +| | | less | +| | | than or | +| | | equal to | +| | | (‘<=’) | +| | | the | +| | | value | +| | | declared | ++--------------+----+----------+ +| valid_value\ | ar\| Constrai\| +| s | ra\| ns | +| | y | a | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value | +| | | that is | +| | | in the | +| | | list of | +| | | declared | +| | | values | ++--------------+----+----------+ +| length | nu\| Constrai\| +| | mb\| ns | +| | er | the | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value of | +| | | a given | +| | | length | ++--------------+----+----------+ +| min_length | nu\| Constrai\| +| | mb\| ns | +| | er | the | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value to | +| | | a | +| | | minimum | +| | | length | ++--------------+----+----------+ +| max_length | nu\| Constrai\| +| | mb\| ns | +| | er | the | +| | | property | +| | | or | +| | | paramete\| +| | | r | +| | | to a | +| | | value to | +| | | a | +| | | maximum | +| | | length | ++--------------+----+----------+ + +``threshold`` is the configuration parameter and will get set to 0.75 +when the configuration gets generated. + +The property ``policy_schema`` is a list of objects where each +policy_schema object: + ++-------------+----+----------+------+ +| Property | Ty\| Descript\| Defa\| +| Name | pe | ion | ult | ++=============+====+==========+======+ +| name | st\| *Require\| | +| | ri\| d*. | | +| | ng | paramete\| | +| | | r | | +| | | name | | ++-------------+----+----------+------+ +| value | st\| default | | +| | ri\| value | | +| | ng | for the | | +| | | paramete\| | +| | | r | | ++-------------+----+----------+------+ +| description | st\| paramete\| | +| | ri\| r | | +| | ng | descript\| | +| | | ion | | ++-------------+----+----------+------+ +| type | en\| *Require\| | +| | um | d*. | | +| | | data | | +| | | type of | | +| | | the | | +| | | paramete\| | +| | | r, | | +| | | ‘string’ | | +| | | , | | +| | | ‘number’ | | +| | | , | | +| | | ‘boolean | | +| | | ’, | | +| | | ‘datetim\| | +| | | e’, | | +| | | ‘list’, | | +| | | or ‘map’ | | ++-------------+----+----------+------+ +| required | bo\| is | true | +| | ol\| paramete\| | +| | ea\| r | | +| | n | required | | +| | | or not? | | ++-------------+----+----------+------+ +| constraints | ar\| The | | +| | ra\| optional | | +| | y | list of | | +| | | sequence\| | +| | | d | | +| | | constrai\| | +| | | nt | | +| | | clauses | | +| | | for the | | +| | | paramete\| | +| | | r. | | +| | | See | | +| | | above | | ++-------------+----+----------+------+ +| entry_schem\| st\| The | | +| a | ri\| optional | | +| | ng | key that | | +| | | is used | | +| | | to | | +| | | declare | | +| | | the name | | +| | | of the | | +| | | Datatype | | +| | | definiti\| | +| | | on | | +| | | for | | +| | | certain | | +| | | types. | | +| | | entry_sc\| | +| | | hema | | +| | | must be | | +| | | defined | | +| | | when the | | +| | | type is | | +| | | either | | +| | | list or | | +| | | map. If | | +| | | the type | | +| | | is list | | +| | | and the | | +| | | entry | | +| | | type is | | +| | | a simple | | +| | | type | | +| | | (string, | | +| | | number, | | +| | | bookean, | | +| | | datetime | | +| | | ), | | +| | | follow | | +| | | with an | | +| | | string | | +| | | to | | +| | | describe | | +| | | the | | +| | | entry | | ++-------------+----+----------+------+ +| | If | | | +| | th\| | | +| | e | | | +| | ty\| | | +| | pe | | | +| | is | | | +| | li\| | | +| | st | | | +| | an\| | | +| | d | | | +| | th\| | | +| | e | | | +| | en\| | | +| | tr\| | | +| | y | | | +| | ty\| | | +| | pe | | | +| | is | | | +| | a | | | +| | ma\| | | +| | p, | | | +| | fo\| | | +| | ll\| | | +| | ow | | | +| | wi\| | | +| | th | | | +| | an | | | +| | ar\| | | +| | ra\| | | +| | y | | | +| | to | | | +| | de\| | | +| | sc\| | | +| | ri\| | | +| | be | | | +| | th\| | | +| | e | | | +| | ke\| | | +| | ys | | | +| | fo\| | | +| | r | | | +| | th\| | | +| | e | | | +| | en\| | | +| | tr\| | | +| | y | | | +| | ma\| | | +| | p | | | ++-------------+----+----------+------+ +| | If | | | +| | th\| | | +| | e | | | +| | ty\| | | +| | pe | | | +| | is | | | +| | li\| | | +| | st | | | +| | an\| | | +| | d | | | +| | th\| | | +| | e | | | +| | en\| | | +| | tr\| | | +| | y | | | +| | ty\| | | +| | pe | | | +| | is | | | +| | a | | | +| | li\| | | +| | st | | | +| | , | | | +| | th\| | | +| | at | | | +| | is | | | +| | no\| | | +| | t | | | +| | cu\| | | +| | rr\| | | +| | en\| | | +| | tl\| | | +| | y | | | +| | su\| | | +| | pp\| | | +| | or\| | | +| | te\| | | +| | d \| | | ++-------------+----+----------+------+ +| | If | | | +| | th\| | | +| | e | | | +| | ty\| | | +| | pe | | | +| | is | | | +| | ma\| | | +| | p, | | | +| | fo\| | | +| | ll\| | | +| | ow | | | +| | wi\| | | +| | th | | | +| | an | | | +| | ar\| | | +| | ay | | | +| | to | | | +| | de\| | | +| | sc\| | | +| | ri\| | | +| | be | | | +| | th\| | | +| | e | | | +| | ke\| | | +| | ys | | | +| | fo\| | | +| | r | | | +| | th\| | | +| | e | | | +| | ma\| | | +| | p | | | ++-------------+----+----------+------+ + + +.. _artifacts: + +Artifacts +--------- + +``artifacts`` contains a list of artifacts associated with this +component. For Docker, this is the full path (including the registry) to +the Docker image. For CDAP, this is the full path to the CDAP jar. + ++---------------+------------+---------------------------------+ +| Property Name | Type | Description | ++===============+============+=================================+ +| artifacts | JSON array | Each entry is a artifact object | ++---------------+------------+---------------------------------+ + +``artifact`` Schema: + ++---------------+--------+--------------------------------------------+ +| Property Name | Type | Description | ++===============+========+============================================+ +| uri | string | *Required*. Uri to the artifact, full path | ++---------------+--------+--------------------------------------------+ +| type | string | *Required*. ``docker image`` or ``jar`` | ++---------------+--------+--------------------------------------------+ + +.. _auxilary: + +Auxilary +-------- + + +Health check +~~~~~~~~~~~~ + +Component developers are required to provide a way for the platform to +periodically check the health of their running components. The +details of the definition used by your component is to be provided +through the :any:`Docker auxiliary specification `. + + diff --git a/docs/sections/design-components/component-specification/configuration-grid.rst b/docs/sections/design-components/component-specification/configuration-grid.rst new file mode 100755 index 00000000..3efb9850 --- /dev/null +++ b/docs/sections/design-components/component-specification/configuration-grid.rst @@ -0,0 +1,118 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Configuration Quick Reference +============================= + +Default Values +^^^^^^^^^^^^^^ + +The component developer can provide default values for any ``parameter`` +in the component specification. These defaults will be passed to the +component in its generated configuration. + +Overridden/Entered Values +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Depending on the other properties set for the parameter, the default +value can be overridden at ‘design-time’, ‘deploy-time’ or once the +microservice is running (‘run-time’). + ++--------+--------+--------+--------+------------+ +| | Design\| CLAMP | Policy | Deploy-Time| +| | -Time | Input | Input | Input | +| | Input | | | | +| | | | | | ++========+========+========+========+============+ +| Descri\| Applie\| Applie\| (not | Applies to | +| ption | s | s | yet | manually | +| | to | to | suppor\| deployed | +| | self-s\| compon\| ted) | services | +| | ervice | ents | | | +| | compon\| deploy\| | | +| | ents | ed | | | +| | | by | | | +| | | CLAMP | | | ++--------+--------+--------+--------+------------+ +| Input | Servic\| CLAMP | Operat\| DevOps | +| provid\| e | | ions | | +| ed | Designe| | | | +| by | r | | | | ++--------+--------+--------+--------+------------+ +| How it | In the | In the | In the | In the DCAE| +| is | SDC/MOD| CLAMP | POLICY | Dashboard | +| provid\| UI | UI | GUI | (or Jenkins| +| ed | | | | job) | ++--------+--------+--------+--------+------------+ +| Compon\| ‘desig\| None. | ‘polic\| ‘sourced\ | +| ent | ner-ed\| Develo\| y_edit\| _at_deploy\| +| Specif\| itable\| per | able’ | ment’ must | +| icatio\| ’ | provid\| must | be set to | +| n | set to | es | be set | ‘true’ | +| Detail\| ‘true’ | CLAMP | to | | +| s | | an | ‘true’ | | +| | | email | and | | +| | | with | ‘polic\| | +| | | parame\| y_sche\| | +| | | ters | ma’ | | +| | | to be | must | | +| | | suppor\| be | | +| | | ted | provid\| | +| | | | ed | | +| | | | | | +| | | | | | ++--------+--------+--------+--------+------------+ +| Additi\| | | For | | +| onal | | | Docker | | +| Info | | | only: | | +| for | | | In the | | +| Compon\| | | auxili\| | +| ent | | | ary | | +| Develo\| | | sectio\| | +| per | | | n: | | +| | | | {“poli\| | +| | | | cy”: | | +| | | | {“trig\| | +| | | | ger_ty\| | +| | | | pe”: | | +| | | | “polic\| | +| | | | y”,“sc\| | +| | | | ript_p\| | +| | | | ath”: | | +| | | | “/opt/\| | +| | | | app/re\| | +| | | | config\| | +| | | | ure.sh | | +| | | | ”} | | +| | | | } | | +| | | | Script | | +| | | | interf\| | +| | | | ace | | +| | | | would | | +| | | | then | | +| | | | be | | +| | | | “/opt/\| | +| | | | app/re\| | +| | | | config\| | +| | | | ure.sh | | +| | | | ” | | +| | | | $trigg\| | +| | | | er_typ\| | +| | | | e | | +| | | | $updat\| | +| | | | ed_pol\| | +| | | | icy" | | +| | | | where | | +| | | | $updat\| | +| | | | ed_pol\| | +| | | | icy | | +| | | | is | | +| | | | json | | +| | | | provid\| | +| | | | ed | | +| | | | by the | | +| | | | Policy | | +| | | | Handle\| | +| | | | r. | | ++--------+--------+--------+--------+------------+ + diff --git a/docs/sections/design-components/component-specification/data-formats.rst b/docs/sections/design-components/component-specification/data-formats.rst new file mode 100755 index 00000000..42194fa3 --- /dev/null +++ b/docs/sections/design-components/component-specification/data-formats.rst @@ -0,0 +1,235 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +.. _data-formats: + + +Data Formats +============ + +Data formats are descriptions of data; they are the data contract +between your component and other components. When the components are +‘composed’ into services in the Design tool, they can only be matched with +components that have compatible data formats. Data formats will be +onboarded to Design tool and assigned a UUID at that time. This UUID is then +used to ensure compatibility amoung components. (If component X outputs +data format ‘DF-Y’, and another component Z specifies ‘DF-Y’ as its +input data format, then X is said to be ``composable`` with component +Z). + +Since data formats will be shared across components, the onboarding +catalog should be checked first to see if the desired data format is +available before creating one. The vision is to have a repository of +shared data formats that developers and teams can re-use and also +provide them the means to extend and create new custom data formats. A +data format is referenced by its data format id and version number. + +JSON schema +----------- + + The data format specification is represented (and validated) against + this `Data Format json schema `__ + and described below: + +Meta Schema Definition +~~~~~~~~~~~~~~~~~~~~~~ + +The “Meta Schema” implementation defines how data format JSON schemas +can be written to define user input. It is itself a JSON schema (thus it +is a “meta schema”). It requires the name of the data format entry, the +data format entry version and allows a description under “self” object. +The meta schema version must be specified as the value of the +“dataformatversion” key. Then the input schema itself is described as +one of the four types listed below: + ++------------------+---------------------------------------------------+ +| Type | Description | ++==================+===================================================+ +| jsonschema | inline standard JSON Schema definitions of JSON | +| | inputs | ++------------------+---------------------------------------------------+ +| delimitedschema | delimited data input using a JSON description and | +| | defined delimiter | ++------------------+---------------------------------------------------+ +| unstructured | unstructured text, and reference that allows a | +| | pointer to another artifact for a schema. | ++------------------+---------------------------------------------------+ +| reference | allows for XML and Protocol Buffers schema, | +| | but can be used to reference other JSON, | +| | delimitedschema and unstructured schemas as well. | ++------------------+---------------------------------------------------+ + + +Example Schemas +--------------- + +By reference example - Common Event Format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +First the full JSON schema description of the Common Event Format would +be loaded with a name of “Common Event Format” and the current version +of “25.0.0”. + +Then the data format description is loaded by this schema: + +:: + + { + "self": { + "name": "Common Event Format Definition", + "version": "25.0.0", + "description": "Common Event Format Definition" + + }, + "dataformatversion": "1.0.0", + "reference": { + "name": "Common Event Format", + "format": "JSON", + "version": "25.0.0" + } + } + + + +Simple JSON Example +~~~~~~~~~~~~~~~~~~~~~~~~ + + +:: + + { + "self": { + "name": "Simple JSON Example", + "version": "1.0.0", + "description": "An example of unnested JSON schema for Input and output" + + }, + "dataformatversion": "1.0.0", + "jsonschema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "raw-text": { + "type": "string" + } + }, + "required": ["raw-text"], + "additionalProperties": false + } + } + +Nested JSON Example +~~~~~~~~~~~~~~~~~~~ + +:: + + { + "self": { + "name": "Nested JSON Example", + "version": "1.0.0", + "description": "An example of nested JSON schema for Input and output" + + }, + "dataformatversion": "1.0.0", + "jsonschema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "numFound": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "engagements": { + "type": "array", + "items": { + "properties": { + "engagementID": { + "type": "string", + "transcript": { + "type": "array", + "items": { + "type": { + "type": "string" + }, + "content": { + "type": "string" + }, + "senderName": { + "type": "string" + }, + "iso": { + "type": "string" + }, + "timestamp": { + "type": "integer" + }, + "senderId": { + "type": "string" + } + } + } + } + } + } + } + }, + "additionalProperties": false + } + } + +Unstructured Example +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + { + "self": { + "name": "Unstructured Text Example", + "version": "25.0.0", + "description": "An example of a unstructured text used for both input and output for " + + }, + "dataformatversion": "1.0.0", + "unstructured": { + "encoding": "UTF-8" + } + } + + +An example of a delimited schema +-------------------------------- + +:: + + { + "self": { + "name": "Delimited Format Example", + "version": "1.0.0", + "description": "Delimited format example just for testing" + + }, + "dataformatversion": "1.0.0", + "delimitedschema": { + "delimiter": "|", + "fields": [{ + "name": "field1", + "description": "test field1", + "fieldtype": "string" + }, { + "name": "field2", + "description": "test field2", + "fieldtype": "boolean" + }] + } + } + +Note: The referenced data format (in this case, a schema named “Common +Event Format” with version of “25.0.0”) must already exist in the +onboarding catalog. + +Working with Data Formats +------------------------- + +Data Formats can be validated using `schema `__ +Once validated, the dataformat can be onboarded using :doc:`DCAE-MOD <../DCAE-MOD/DCAE-MOD-User-Guide>` \ No newline at end of file diff --git a/docs/sections/design-components/component-specification/dmaap-connection-objects.rst b/docs/sections/design-components/component-specification/dmaap-connection-objects.rst new file mode 100755 index 00000000..da368abf --- /dev/null +++ b/docs/sections/design-components/component-specification/dmaap-connection-objects.rst @@ -0,0 +1,218 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +.. _dmaap-connection-objects: + +DMaaP connection objects +======================== + +DMaaP Connection objects are generated by the DCAE Platform at runtime +and passed to the component in its application_configuration + +.. _dmaap-message-router: + +Message Router +-------------- + +Publishers and subscribers have the same generated +``Dmaap Connection Object`` structure. Here’s an example for any given +config-key: (This is what will be in application_configuration) + +.. code:: json + + { + "type": "message_router", + "aaf_username": "some-user", + "aaf_password": "some-password", + "dmaap_info": { + "client_role": "com.dcae.member", + "client_id": "1500462518108", + "location": "mtc00", + "topic_url": "https://we-are-message-router.us:3905/events/some-topic" + } + } + +At the top-level: + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| type | string | *Required as input*. Must | +| | | be ``message_router`` for | +| | | message router topics | ++--------------------------------+---------+---------------------------+ +| aaf_username | string | AAF username message | +| | | router clients use to | +| | | authenticate with secure | +| | | topics | ++--------------------------------+---------+---------------------------+ +| aaf_password | string | AAF password message | +| | | router clients use to | +| | | authenticate with secure | +| | | topics | ++--------------------------------+---------+---------------------------+ +| dmaap_info | JSON | *Required as input*. | +| | object | Contains the topic | +| | | connection details | ++--------------------------------+---------+---------------------------+ + +The ``dmaap_info`` object contains: + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| client_role | string | AAF client role that’s | +| | | requesting publish or | +| | | subscribe access to the | +| | | topic | ++--------------------------------+---------+---------------------------+ +| client_id | string | Client id for given AAF | +| | | client | ++--------------------------------+---------+---------------------------+ +| location | string | DCAE location for the | +| | | publisher or subscriber, | +| | | used to set up routing | ++--------------------------------+---------+---------------------------+ +| topic_url | string | *Required as input*. URL | +| | | for accessing the topic | +| | | to publish or receive | +| | | events | ++--------------------------------+---------+---------------------------+ + + + +.. _dmaap-data-router: + +Data Router +----------- + +Publisher +~~~~~~~~~ + +Here’s an example of what the generated ``Dmaap Connection Object`` for +Data Router Publisher looks like: (This is what will be in +application_configuration) + +.. code:: json + + { + "type": "data_router", + "dmaap_info": { + "location": "mtc00", + "publish_url": "https://we-are-data-router.us/feed/xyz", + "log_url": "https://we-are-data-router.us/feed/xyz/logs", + "username": "some-user", + "password": "some-password", + "publisher_id": "123456" + } + } + +At the top-level: + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| type | string | *Required as input*. Must | +| | | be ``data_router`` for | +| | | data router feeds | ++--------------------------------+---------+---------------------------+ +| dmaap_info | JSON | *Required as input*. | +| | object | Contains the feed | +| | | connection details | ++--------------------------------+---------+---------------------------+ + +The ``dmaap_info`` object contains: + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| location | string | DCAE location for the | +| | | publisher, used to set up | +| | | routing | ++--------------------------------+---------+---------------------------+ +| publish_url | string | *Required as input*. URL | +| | | to which the publisher | +| | | makes Data Router publish | +| | | requests | ++--------------------------------+---------+---------------------------+ +| log_url | string | URL from which log data | +| | | for the feed can be | +| | | obtained | ++--------------------------------+---------+---------------------------+ +| username | string | Username the publisher | +| | | uses to authenticate to | +| | | Data Router | ++--------------------------------+---------+---------------------------+ +| password | string | Password the publisher | +| | | uses to authenticate to | +| | | Data Router | ++--------------------------------+---------+---------------------------+ +| publisher_id | string | Publisher id in Data | +| | | Router | ++--------------------------------+---------+---------------------------+ + + +Subscriber +~~~~~~~~~~ + +Here’s an example of what the generated ``Dmaap Connection Object`` for +a Data Router Subscriber looks like: (This is what will be passed in +application_configuration) + +.. code:: json + + { + "type": "data_router", + "dmaap_info": { + "location": "mtc00", + "delivery_url": "https://my-subscriber-app.dcae:8080/target-path", + "username": "some-user", + "password": "some-password", + "subscriber_id": "789012" + } + } + +At the top-level: + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| type | string | *Required as input*. Must | +| | | be ``data_router`` for | +| | | data router feeds | ++--------------------------------+---------+---------------------------+ +| dmaap_info | JSON | *Required as input*. | +| | object | Contains the feed | +| | | connection details | ++--------------------------------+---------+---------------------------+ + +The ``dmaap_info`` object contains: + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| location | string | DCAE location for the | +| | | subscriber, used to set | +| | | up routing | ++--------------------------------+---------+---------------------------+ +| delivery_url | string | URL to which the Data | +| | | Router should deliver | +| | | files | ++--------------------------------+---------+---------------------------+ +| username | string | Username Data Router uses | +| | | to authenticate to the | +| | | subscriber when | +| | | delivering files | ++--------------------------------+---------+---------------------------+ +| password | string | Password Data Router uses | +| | | to authenticate to the | +| | | subscriber when | +| | | delivering files | ++--------------------------------+---------+---------------------------+ +| subscriber_id | string | Subscriber id in Data | +| | | Router | ++--------------------------------+---------+---------------------------+ + + + + diff --git a/docs/sections/design-components/component-specification/docker-specification.rst b/docs/sections/design-components/component-specification/docker-specification.rst new file mode 100755 index 00000000..6dd4f927 --- /dev/null +++ b/docs/sections/design-components/component-specification/docker-specification.rst @@ -0,0 +1,347 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +.. _docker-requirements: + +Component Spec Requirements +=========================== + +The component specification contains the following groups of +information. + +- :any:`Metadata ` +- :any:`Interfaces ` including the + associated :any:`Data Formats ` +- :any:`Parameters ` +- :any:`Auxiliary Details ` +- :any:`List of Artifacts ` + +.. _docker-auxiliary-details: + +Auxiliary Details +----------------- + +``auxiliary`` contains Docker specific details like health check, port +mapping, volume mapping and policy reconfiguration script details. + + ++--------------------------------+---------+---------------------------+ +| Name | Type | Description | ++================================+=========+===========================+ +| healthcheck | JSON | *Required*. Health check | +| | object | definition details | ++--------------------------------+---------+---------------------------+ +| ports | JSON | each array item maps a | +| | array | container port to the | +| | | host port. See example | +| | | below. | ++--------------------------------+---------+---------------------------+ +| volume | JSON | each array item contains | +| | array | a host and container | +| | | object. See example | +| | | below. | ++--------------------------------+---------+---------------------------+ +| policy | JSON | *Required*. Policy | +| | array | reconfiguration script | +| | | details | ++--------------------------------+---------+---------------------------+ + +Health Check Definition +~~~~~~~~~~~~~~~~~~~~~~~ + +The platform currently supports http and docker script based health checks. + +When choosing a value for interval, consider that too frequent +healthchecks will put unnecessary load on the platform. If there is a +problematic resource, then more frequent healthchecks are warranted (eg +15s or 60s), but as stability increases, so can these values, (eg +300s). + +When choosing a value for timeout, consider that too small a number will +result in increasing timeout failures, and too large a number will +result in a delay in the notification of the resource problem. A +suggestion is to start with 5s and work from there. + +http +^^^^ + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| type | string | *Required*. ``http`` | ++--------------------------------+---------+---------------------------+ +| interval | string | Interval duration in | +| | | seconds i.e. ``60s`` | ++--------------------------------+---------+---------------------------+ +| timeout | string | Timeout in seconds i.e. | +| | | ``5s`` | ++--------------------------------+---------+---------------------------+ +| endpoint | string | *Required*. GET endpoint | +| | | provided by the component | +| | | for checking health | ++--------------------------------+---------+---------------------------+ + +Example: + +.. code:: json + + "auxilary": { + "healthcheck": { + "type": "http", + "interval": "15s", + "timeout": "1s", + "endpoint": "/my-health" + } + } + +docker script example +^^^^^^^^^^^^^^^^^^^^^ + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| type | string | *Required*. ``docker`` | ++--------------------------------+---------+---------------------------+ +| interval | string | Interval duration in | +| | | seconds i.e. ``15s`` | ++--------------------------------+---------+---------------------------+ +| timeout | string | Timeout in seconds i.e. | +| | | ``1s`` | ++--------------------------------+---------+---------------------------+ +| script | string | *Required*. Full path of | +| | | script that exists in the | +| | | Docker container to be | +| | | executed | ++--------------------------------+---------+---------------------------+ + +During deployment, the K8S plugin maps the healthcheck defined into +into a Kubernetes readiness probe. + +Kubernetes execs the script in the container (using the `docker exec API + `__ ). +It will examine the +script result to identify whether your component is healthy. Your +component is considered healthy when the script returns ``0`` otherwise +your component is considered not healthy. + +Example: + +.. code:: json + + "auxilary": { + "healthcheck": { + "type": "docker", + "script": "/app/resources/check_health.py", + "timeout": "30s", + "interval": "180s" + } + } + +Ports +~~~~~ + +This method of exposing/mapping a local port to a host port is NOT +RECOMMENDED because of the possibility of port conflicts. If multiple +instances of a docker container will be running, there definitely will +be port conflicts. Use at your own risk. (The preferred way to expose a +port is to do so in the Dockerfile as described +:any:`here `). + +.. code:: json + + "auxilary": { + "ports": ["8080:8000"] + } + +In the example above, container port 8080 maps to host port 8000. + +Volume Mapping +~~~~~~~~~~~~~~ + +.. code:: json + + "auxilary": { + "volumes": [ + { + "container": { + "bind": "/tmp/docker.sock", + "mode": "ro" + }, + "host": { + "path": "/var/run/docker.sock" + } + } + ] + } + +At the top-level: + ++---------------+-------+-------------------------------------+ +| Property Name | Type | Description | ++===============+=======+=====================================+ +| volumes | array | Contains container and host objects | ++---------------+-------+-------------------------------------+ + +The ``container`` object contains: + + ++-----------------------+-----------------------+-------------------------------+ +| Property Name | Type | Description | ++=======================+=======================+===============================+ +| bind | string | path to the container | +| | | volume | ++-----------------------+-----------------------+-------------------------------+ +| mode | string | ro - indicates | +| | | read-only volume | ++-----------------------+-----------------------+-------------------------------+ +| | | w - indicates that | +| | | the contain can write | +| | | into the bind mount | ++-----------------------+-----------------------+-------------------------------+ + +The ``host`` object contains: + ++---------------+--------+-------------------------+ +| Property Name | Type | Description | ++===============+========+=========================+ +| path | string | path to the host volume | ++---------------+--------+-------------------------+ + +Here’s an example of the minimal JSON that must be provided as an input: + +.. code:: json + + "auxilary": { + "volumes": [ + { + "container": { + "bind": "/tmp/docker.sock" + }, + "host": { + "path": "/var/run/docker.sock" + } + } + ] + } + +In the example above, the container volume “/tmp/docker.sock” maps to +host volume “/var/run/docker.sock”. + + +Policy +~~~~~~~ + +Policy changes made in the Policy UI will be provided to the Docker +component by triggering a script that is defined here. + ++--------------------------------+---------+---------------------------+ +| Property Name | Type | Description | ++================================+=========+===========================+ +| reconfigure_type | string | *Required*. Current value | +| | | supported is ``policy`` | ++--------------------------------+---------+---------------------------+ +| script_path | string | *Required*. Current value | +| | | for ‘policy’ | +| | | reconfigure_type must be | +| | | “/opt/app/reconfigure.sh” | ++--------------------------------+---------+---------------------------+ + +Example: + +.. code:: json + + "auxilary": { + "policy": { + "reconfigure_type": "policy", + "script_path": "/opt/app/reconfigure.sh" + } + } + +The docker script interface is as follows: \`/opt/app/reconfigure.sh +$reconfigure_type {“updated policies”: , “application config”: } + ++---------------------+--------------+----------------------------------------+ +| Name | Type | Description | ++=====================+==============+========================================+ +| reconfigure_type | string | policy | ++---------------------+--------------+----------------------------------------+ +| updated_policies | json | TBD | ++---------------------+--------------+----------------------------------------+ +| updated_appl_config | json | complete generated app_config, not | +| | | fully-resolved, but ``policy-enabled`` | +| | | parameters have been updated. In order | +| | | to get the complete updated | +| | | app_config, the component would have | +| | | to call ``config-binding-service``. | ++---------------------+--------------+----------------------------------------+ + +Docker Component Spec - Complete Example +---------------------------------------- + +.. code:: json + + { + "self": { + "version": "1.0.0", + "name": "yourapp.component.kpi_anomaly", + "description": "Classifies VNF KPI data as anomalous", + "component_type": "docker" + }, + "streams": { + "subscribes": [{ + "format": "dcae.vnf.kpi", + "version": "1.0.0", + "route": "/data", + "type": "http" + }], + "publishes": [{ + "format": "yourapp.format.integerClassification", + "version": "1.0.0", + "config_key": "prediction", + "type": "http" + }] + }, + "services": { + "calls": [{ + "config_key": "vnf-db", + "request": { + "format": "dcae.vnf.meta", + "version": "1.0.0" + }, + "response": { + "format": "dcae.vnf.kpi", + "version": "1.0.0" + } + }], + "provides": [{ + "route": "/score-vnf", + "request": { + "format": "dcae.vnf.meta", + "version": "1.0.0" + }, + "response": { + "format": "yourapp.format.integerClassification", + "version": "1.0.0" + } + }] + }, + "parameters": [ + { + "name": "threshold", + "value": 0.75, + "description": "Probability threshold to exceed to be anomalous" + } + ], + "auxilary": { + "healthcheck": { + "type": "http", + "interval": "15s", + "timeout": "1s", + "endpoint": "/my-health" + } + }, + "artifacts": [{ + "uri": "fake.nexus.att.com/dcae/kpi_anomaly:1.0.0", + "type": "docker image" + }] + } diff --git a/docs/sections/design-components/component-specification/index-component-specification.rst b/docs/sections/design-components/component-specification/index-component-specification.rst new file mode 100644 index 00000000..76ef2129 --- /dev/null +++ b/docs/sections/design-components/component-specification/index-component-specification.rst @@ -0,0 +1,15 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Component Specification +======================= + +.. toctree:: + :maxdepth: 1 + + ./component-type-docker.rst + ./component-json-schema.rst + ./docker-specification.rst + ./dmaap-connection-objects.rst + ./streams-grid.rst + ./configuration-grid.rst \ No newline at end of file diff --git a/docs/sections/design-components/component-specification/streams-grid.rst b/docs/sections/design-components/component-specification/streams-grid.rst new file mode 100755 index 00000000..6105e9e1 --- /dev/null +++ b/docs/sections/design-components/component-specification/streams-grid.rst @@ -0,0 +1,149 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +.. _streams-grid: + +Streams Formatting Quick Reference +================================== + +Each of the following tables represents an example of a publisher and +its subscriber, which are of course, different components. This focuses +on the fields that are ‘different’ for each of these TYPEs, to +illustrate the relationship between ``config_key``, dmaap connection +object, and the generated configuration. Some notes on specific +properties: + +- ``config_key`` is an arbitrary string, chosen by the component + developer. It is returned in the generated configuration where it + contains specific values for the target connection +- ``format``, ``version``, and ``type`` properties in the subscriber + would match these properties in the publisher +- ``aaf_username`` and ``aaf_password`` may be different between the + publisher and the subscriber + +Using http +~~~~~~~~~~ + +*Publishing Component* +^^^^^^^^^^^^^^^^^^^^^^ + ++-----------------------------+----------------------------------------+ +| component \ | runtime platform generated config | +| spec | | ++=============================+========================================+ +| "streams":{   | "streams_publishes":{  | +| "publishes":[{ | "prediction":"10.100.1.100:32567/data" | +| "config_key":"prediction", |  | +| "format":"some-format", | | +| "type":"http", | | +| "version":"0.1.0"   } | | +| ]} | | ++-----------------------------+----------------------------------------+ + +*Subscribing Component* +^^^^^^^^^^^^^^^^^^^^^^^ + ++-----------------------------+----------------------------------------+ +| component | runtime platform generated config | +| spec | | ++=============================+========================================+ +| “streams”:{    | "N/A" | +| "subscribes":[{ | | +| "route":"/data", | | +| "format":"some-format", | | +| "type":"http" | | +| "version":"0.1.0"   } | | +| ]} | | ++-----------------------------+----------------------------------------+ + +Using Message Router +~~~~~~~~~~~~~~~~~~~~ + +.. publishing-component-1: + +*Publishing Component* +^^^^^^^^^^^^^^^^^^^^^^ + +Note: When deploying, this component should be deployed first so satisfy +downstream dependencies. Refer to the –force option in component ‘run’ +command for more information. + ++---------------+------------------------+-----------------------------------------------------------------------------+ +| component \ | Dmaap Connection \ | runtime platform generated \ | +| spec | Object | config | ++===============+========================+=============================================================================+ +| “streams”:{  | {     “dmaap_info”: | “streams_publishes”:{    | +|     “config_k\| {} \ *Note: For \ | “aaf_username”:“pub-user”,   | +| ey”:“mr_out\ | message router, this \ |   “type”:“message_router”,   | +| put”,     “t\ | object is identical \ |      “topic_url”:"https://we-are-message-router.us:3905/events/some-topic"\ | +| ype”:“messag\ | for the publisher and \| "streams_subscribes":{...} | +| e_router”,   | the subscriber* | | +|  }]} | | | ++---------------+------------------------+-----------------------------------------------------------------------------+ + +*Subscribing Component* +^^^^^^^^^^^^^^^^^^^^^^^ + ++---------------+------------------------+-----------------------------------------------------------------------------+ +| component \ | Dmaap Connection \ | runtime platform generated \ | +| spec | Object | config | ++===============+========================+=============================================================================+ +| “streams”:{  | {     “dmaap_info”: | “streams_publishes”:{…}, | +|     “config_k\| {} \ *Note: For \ | “streams_subscribes”:{    | +| ey”:“mr_inp\ | message router, this \ | “aaf_username”:“sub-user”,   | +| ut”,     “ty\ | object is identical \ |   “type”:“message_router”,   | +| pe”:“message\ | for the publisher and \|      “topic_url”:“https://we-are-message-router.us:3905/events/some-topic" | +| _router”,    | the subscriber* | | +| }]} | | | ++---------------+------------------------+-----------------------------------------------------------------------------+ + +Using Data Router +~~~~~~~~~~~~~~~~~ + +.. publishing-component-2: + +*Publishing Component* +^^^^^^^^^^^^^^^^^^^^^^ + ++---------------+-----------------------------------------------+-----------------------------------------------+ +| component spec| Dmaap Connection Object | runtime platform generated config | ++===============+===============================================+===============================================+ +| “streams”:{  | {    “dmaap_info”: { | streams_publishes“:{    ”typ\ | +| “config_key: |      “location”: | e“:”data_router“,       "location":"mtc00" | +| “dr_output" | “mtc00”,      | , | +| , "type": | “publish_url”: | "publish_url“: | +| “data_r\ | "https://we-are-data-router.us/feed/xyz"\ | "http://we-are-data-router.us/feed/xyz" | +| outer”,   }] | , | , | +| } | “log_url”:\ | "log_url“:\ | +| | \ | ”https://we-are-data-router.us/feed/xyz/logs" | +| | "https://we-are-data-router.us/feed/xyz/logs"\| , | +| | , | ”username“:”pub-user“, | +| | “username”: | ”publisher_id“:”123456\ | +| | “pub-user”,      | “}}, | +| | “password”: |  ”streams_subscribes“:{ | +| | “pub-password”,      | … } | +| | “publisher_id”: | | +| | “123456”}} | | ++---------------+-----------------------------------------------+-----------------------------------------------+ + +.. subscribing-component-1: + +*Subscribing Component* +^^^^^^^^^^^^^^^^^^^^^^^ + ++---------------+---------------------------------------------------+---------------------------------------------------------------------------+ +| component \ | Dmaap Connection \ | runtime platform generated \ | +| spec | Object | config | ++===============+===================================================+===========================================================================+ +| “streams”:{  | {      “dmaap_info”: | “streams_publishes”:{ … }, | +|     “config_k\| {      “location”: | “streams_subscribes”:{       | +| ey”:“dr_inp\ | “mtc00”,      | “type”:“data_router”,        | +| ut”,     “ty\ | “delivery_url”: |   “location”:“mtc00”, | +| pe”:“data_ro\ | "https://my-subscriber-app.dcae:8080/target-path"\|          “delivery_url”:"https://my-subscriber-app.dcae:8080/target-path"\| +| uter”, | \ | \ | +|     “route”: | , | , | +| “/target-pat\ |      “password”: | \        | +| h”} | “sub-password”,      | “username”:“sub-user”, | +| | “subscriber_id”: |          | +| | “789012”}} | “subscriber_id”:“789012”}} | ++---------------+---------------------------------------------------+---------------------------------------------------------------------------+ diff --git a/docs/sections/design-components/glossary.rst b/docs/sections/design-components/glossary.rst new file mode 100644 index 00000000..b1430e22 --- /dev/null +++ b/docs/sections/design-components/glossary.rst @@ -0,0 +1,169 @@ + +.. http://creativecommons.org/licenses/by/4.0 + +.. _glossary: + +Glossary +======== + +A&AI - Active and Available Inventory +------------------------------------- +Inventory DB for all network components + + +CLAMP +----- +Non DCAE Platform Component - Controls the input and processing for +Closed Loop services. + + +Closed Loop +----------- +Services designed to monitor and report back to a controlling function +that automatically deals with the event reported without human +interaction. + + + +Cloudify +-------- +Open Source application and network orchestration framework, based on +TOSCA used in DCAE to deploy platform and service components from +Cloudify Blueprints. + + +Cloudify Blueprints +------------------- +YAML formatted file used by Cloudify to deploy platform and service +components. Contains all the information needed for installation. + + +Consul +------ +Opensource Platform Component that supports Service Discovery, +Configuration, and Healthcheck. Refer to +`Architecture `__ for more information. + +Component +--------- +Refers to a DCAE service component which is a single micro-service that +is written to be run by the DCAE platform and to be composeable to form +a DCAE service. That composition occurs in the SDC. + + +Config Binding Service +---------------------- +DCAE Platform Component - Service Components use Config Binding Service +to access Consul and retrieve configuration variables. + + +Component Specification +----------------------- +JSON formatted file that fully describes a component and its interfaces + + +Data Format / Data Format Specification +--------------------------------------- +JSON formatted file that fully describes a components input or output + + +Deployment Handler +------------------ +DCAE Platform Component - Receives Input from DTI Handler, and talks to +Cloudify to deploy components. + + +Design-Time +----------- +Refers to when the System Designer uses a design tool to compose services +from components in the catalog. The Designer can provide input to +assign/override defaults for configuration for any parameter with the +property 'designer\_editable' set to 'true'. + + +Deploy-Time +----------- +Refers to when a service is being deployed. This can be done +automatically via the SDC Tool, or manually via the DCAE Dashboard or +CLAMP UI. When manually deployed, DevOps can provide input to +assign/override defaults for configuration for any parameter with the +property 'sourced\_at\_deployment' set to 'true'. + + +Docker +------ +Opensource Platform for development of containerized applications in the +cloud. Many DCAE service components and all DCAE collectors are written +utilizing Docker. + + +Dmaap +----- +AT&T data transportation service platform that supports message-based +topics and file-based feeds. Runs locally at the Edge and Centrally. + + +Inventory +--------- +DCAE Platform Component - Postgres DB containing Cloudify Blueprints for +platform and service components. + + +Policy +------ +Refers to the setting of configuration parameters for a component, by +Operations via the Policy UI. + + +Policy Handler +-------------- +DCAE Platform Component that received Policy updates from Policy UI + + +Policy UI +--------- +Non DCAE Component - Policy User Interace where Operations assigns +values to configuraton specified for this. + + +Run-Time +-------- +Refers to the when a service is running on the platform. Often used in +conjunction with DTI events which occur at Run-time. + + +SCH - Service Change Handler +---------------------------- +DCAE Platform Component - Receives updates from SDC and updates +Inventory + + +SDC - Service Design and Creation +--------------------------------- +ONAP design catalog for onboarding VNF/PNF packages + + +Self-Service +------------ +Refers to services that are supported by SDC, and that are automatically +installed as a result of a Service Designer's composition and submission +of a service. Only a handful of services are 'self-service' currently. +Most require manual effort to generate the Tosca Model files and +Cloudify Blueprints. + + +Service Component +----------------- +Microservice that provides network monitoring or analytic function on +the DCAE platform. + + +Service +------- +Generally composed of multiple service components, which is deployed to +the DCAE platform. + + +VNF - Virtualized Network Function +---------------------------------- +A network function that runs on one or more virtualized machines. diff --git a/docs/sections/design-components/images/1.png b/docs/sections/design-components/images/1.png new file mode 100644 index 00000000..e734a673 Binary files /dev/null and b/docs/sections/design-components/images/1.png differ diff --git a/docs/sections/design-components/images/10.png b/docs/sections/design-components/images/10.png new file mode 100644 index 00000000..9a7f75ad Binary files /dev/null and b/docs/sections/design-components/images/10.png differ diff --git a/docs/sections/design-components/images/11.png b/docs/sections/design-components/images/11.png new file mode 100644 index 00000000..fe5ed014 Binary files /dev/null and b/docs/sections/design-components/images/11.png differ diff --git a/docs/sections/design-components/images/12.png b/docs/sections/design-components/images/12.png new file mode 100644 index 00000000..f2d5a360 Binary files /dev/null and b/docs/sections/design-components/images/12.png differ diff --git a/docs/sections/design-components/images/13.png b/docs/sections/design-components/images/13.png new file mode 100644 index 00000000..c63d1361 Binary files /dev/null and b/docs/sections/design-components/images/13.png differ diff --git a/docs/sections/design-components/images/14.png b/docs/sections/design-components/images/14.png new file mode 100644 index 00000000..5b507b10 Binary files /dev/null and b/docs/sections/design-components/images/14.png differ diff --git a/docs/sections/design-components/images/15.png b/docs/sections/design-components/images/15.png new file mode 100644 index 00000000..1529ef04 Binary files /dev/null and b/docs/sections/design-components/images/15.png differ diff --git a/docs/sections/design-components/images/16.png b/docs/sections/design-components/images/16.png new file mode 100644 index 00000000..e3a7e36d Binary files /dev/null and b/docs/sections/design-components/images/16.png differ diff --git a/docs/sections/design-components/images/17.png b/docs/sections/design-components/images/17.png new file mode 100644 index 00000000..54347d18 Binary files /dev/null and b/docs/sections/design-components/images/17.png differ diff --git a/docs/sections/design-components/images/18.png b/docs/sections/design-components/images/18.png new file mode 100644 index 00000000..150e1c61 Binary files /dev/null and b/docs/sections/design-components/images/18.png differ diff --git a/docs/sections/design-components/images/19.png b/docs/sections/design-components/images/19.png new file mode 100644 index 00000000..b578b103 Binary files /dev/null and b/docs/sections/design-components/images/19.png differ diff --git a/docs/sections/design-components/images/2.png b/docs/sections/design-components/images/2.png new file mode 100644 index 00000000..20bd7a01 Binary files /dev/null and b/docs/sections/design-components/images/2.png differ diff --git a/docs/sections/design-components/images/20.png b/docs/sections/design-components/images/20.png new file mode 100644 index 00000000..80f56e92 Binary files /dev/null and b/docs/sections/design-components/images/20.png differ diff --git a/docs/sections/design-components/images/21.png b/docs/sections/design-components/images/21.png new file mode 100644 index 00000000..c92a2346 Binary files /dev/null and b/docs/sections/design-components/images/21.png differ diff --git a/docs/sections/design-components/images/22.png b/docs/sections/design-components/images/22.png new file mode 100644 index 00000000..bf4f1c02 Binary files /dev/null and b/docs/sections/design-components/images/22.png differ diff --git a/docs/sections/design-components/images/23.png b/docs/sections/design-components/images/23.png new file mode 100644 index 00000000..98fd7970 Binary files /dev/null and b/docs/sections/design-components/images/23.png differ diff --git a/docs/sections/design-components/images/24.png b/docs/sections/design-components/images/24.png new file mode 100644 index 00000000..2784fddf Binary files /dev/null and b/docs/sections/design-components/images/24.png differ diff --git a/docs/sections/design-components/images/25.png b/docs/sections/design-components/images/25.png new file mode 100644 index 00000000..55464c46 Binary files /dev/null and b/docs/sections/design-components/images/25.png differ diff --git a/docs/sections/design-components/images/26.png b/docs/sections/design-components/images/26.png new file mode 100644 index 00000000..9486bb80 Binary files /dev/null and b/docs/sections/design-components/images/26.png differ diff --git a/docs/sections/design-components/images/27.png b/docs/sections/design-components/images/27.png new file mode 100644 index 00000000..3c0bd2f3 Binary files /dev/null and b/docs/sections/design-components/images/27.png differ diff --git a/docs/sections/design-components/images/3.png b/docs/sections/design-components/images/3.png new file mode 100644 index 00000000..7d1d9df0 Binary files /dev/null and b/docs/sections/design-components/images/3.png differ diff --git a/docs/sections/design-components/images/4.png b/docs/sections/design-components/images/4.png new file mode 100644 index 00000000..69706f63 Binary files /dev/null and b/docs/sections/design-components/images/4.png differ diff --git a/docs/sections/design-components/images/5.png b/docs/sections/design-components/images/5.png new file mode 100644 index 00000000..ec158035 Binary files /dev/null and b/docs/sections/design-components/images/5.png differ diff --git a/docs/sections/design-components/images/6.png b/docs/sections/design-components/images/6.png new file mode 100644 index 00000000..b3f7e7bc Binary files /dev/null and b/docs/sections/design-components/images/6.png differ diff --git a/docs/sections/design-components/images/7.png b/docs/sections/design-components/images/7.png new file mode 100644 index 00000000..adb8a941 Binary files /dev/null and b/docs/sections/design-components/images/7.png differ diff --git a/docs/sections/design-components/images/8.png b/docs/sections/design-components/images/8.png new file mode 100644 index 00000000..e04c60ed Binary files /dev/null and b/docs/sections/design-components/images/8.png differ diff --git a/docs/sections/design-components/images/9.png b/docs/sections/design-components/images/9.png new file mode 100644 index 00000000..c18e9487 Binary files /dev/null and b/docs/sections/design-components/images/9.png differ diff --git a/docs/sections/design-components/images/DCAE-Mod-Architecture.png b/docs/sections/design-components/images/DCAE-Mod-Architecture.png new file mode 100644 index 00000000..ce5fca92 Binary files /dev/null and b/docs/sections/design-components/images/DCAE-Mod-Architecture.png differ diff --git a/docs/sections/design-components/images/Onboarding-with-DCAE-MOD.png b/docs/sections/design-components/images/Onboarding-with-DCAE-MOD.png new file mode 100644 index 00000000..4883b65e Binary files /dev/null and b/docs/sections/design-components/images/Onboarding-with-DCAE-MOD.png differ diff --git a/docs/sections/design-components/images/nifi-toolbar-components.png b/docs/sections/design-components/images/nifi-toolbar-components.png new file mode 100644 index 00000000..fa90afef Binary files /dev/null and b/docs/sections/design-components/images/nifi-toolbar-components.png differ diff --git a/docs/sections/design-components/index-onboarding.rst b/docs/sections/design-components/index-onboarding.rst new file mode 100644 index 00000000..1e859d1b --- /dev/null +++ b/docs/sections/design-components/index-onboarding.rst @@ -0,0 +1,18 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Design Platform +=============== + +.. toctree:: + :maxdepth: 1 + + ./overview.rst + ./requirements-guidelines.rst + ./blueprint_generator.rst + ./DCAE-MOD/DCAE-MOD-Architecture.rst + ./component-specification/index-component-specification.rst + ./component-specification/data-formats.rst + ./DCAE-MOD/DCAE-MOD-User-Guide.rst + ./glossary.rst + \ No newline at end of file diff --git a/docs/sections/design-components/overview.rst b/docs/sections/design-components/overview.rst new file mode 100755 index 00000000..654a0c11 --- /dev/null +++ b/docs/sections/design-components/overview.rst @@ -0,0 +1,107 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +.. _intro: + + +Overview +======== + +DCAE components are services that provide a specific functionality and +are generally written to be composable with other DCAE components, +although a component can run independently as well. The DCAE platform is +responsible for running and managing DCAE service components reliably. + +The DCAE Design platform aims to provide a common catalog of available DCAE +Service components, enabling designers to select required +components to construct and deploy composite flows into DCAE Runtime platform. + +Service component/MS to be onboarded and deployed into DCAE platform would +typically go through the following phases + + - Onboarding + - Design + - Runtime + +DCAE Design Platform supports onboarding and service design through MOD. + + +Onboarding is a process that ensures that the component is compliant +with the DCAE platform rules. The high level summary of the onboarding process +is: + +1. Defining the :doc:`data formats ` if they don’t already + exist. +2. Defining the :doc:`component specification <./component-specification/component-specification>` +3. Validate the component spec schema against + `Component Spec json schema `__ +4. Use :doc:`blueprint-generator tool <./blueprint_generator>` to generate Cloudify blueprint +5. Test the blueprint generated in DCAE Runtime Environment (using either Dashboard UI or Cloudify cli from bootstrap) +6. Using :doc:`DCAE-MOD <../DCAE-MOD/DCAE-MOD-User-Guide>` , publish the component and data formats into DCAE-MOD catalog. + (This step is required if Microservice needs to be deployed part of flow/usecase) + + +A Component requires one or more data formats. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A component is a software application that performs a function. It +doesn’t run independently; it depends upon other components. A +component’s function could require connecting to other components to +fulfill that function. A component could also be providing its function +as a service through an interface for other components to use. + +A component cannot connect to or be connected with any other component. +The upstream and downstream components must *speak* the same vocabulary +or *data format*. The output of an one component must match another +component’s input. This is necessary for components to function +correctly and without errors. + +The platform requires data formats to ensure that a component will be +run with other *compatible* components. + +Data formats can and should be shared by multiple components. + +Each Component requires a component specification. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The component specification is a JSON artifact that fully specifies the +component, it’s interfaces, and configuration. It’s standardized for +CDAP (deprecated) and Docker applications and is validated using a +:doc:`JSON schema <./component-specification/component-json-schema>`. + +The component specification fully specifies all the configuration +parameters of the component. This is used by the designer and by policy +(future) to configure the runtime behavior of the component. + +The component specification is used to *generate* application +configuration in a standardized JSON that the platform will make +available to the component. This application configuration JSON will +include: + +- Parameters that have been assigned values from the component + specification, policy, and/or the designer +- Connection details of downstream components + +The component specification is transformed by DCAE tooling (explained +later) into TOSCA models (one for the component, and in the future, one +for Policy). The TOSCA models then get transformed into Cloudify +blueprints. + +The component specification is used by: + + +- Blueprint Generator - Tool to generate standalone cloudify blueprint + using component spec. The blueprints can be uploaded into inventory + using Dashboard and triggered for deployment. +- MOD Platform - To onboard the microservice and maintain in catalog + enabling designer to compose new DCAE service flows and distribute + to DCAE Runtime platform. +- Policy (future) - TOSCA models are generated from the component + specification so that operations can create policy models used to + dynamically configure the component. +- Runtime platform - The component’s application configuration + (JSON) is generated from the component specification and will be + provided to the component at runtime (through ConfigBindingService + or Consul). + + \ No newline at end of file diff --git a/docs/sections/design-components/requirements-guidelines.rst b/docs/sections/design-components/requirements-guidelines.rst new file mode 100644 index 00000000..f633178c --- /dev/null +++ b/docs/sections/design-components/requirements-guidelines.rst @@ -0,0 +1,279 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Onboarding Pre-requisite +======================== + +Before a component is onboarded into DCAE, the component developer must ensure it +is compliant with ONAP & DCAE goals and requirement in order to correctly be deployed and be managed. This +page will discuss the changes which are grouped into the following +categories: + +- :any:`Configuration management via ConfigBindingService ` +- :any:`Docker images ` +- :any:`Policy Reconfiguration flow support ` +- :any:`Operational Requirement ` + + +.. _configuration_management: + +Configuration Management +------------------------ + +All configuration for a component is stored in CONSUL under the +components uniquely generated name which is provided by the environment +variable ``HOSTNAME`` as well as ``SERVICE_NAME``. It is then made +available to the component via a remote HTTP service call to CONFIG +BINDING SERVICE. + +The main entry in CONSUL for the component contains its +**generated application configuration**. This is based on the submitted +component specification, and consists of the *interfaces* (streams and +services/calls) and *parameters* sections. Other entries may exist as +well, under specific keys, such as :dmaap . Each key represents a +specific type of information and is also available to the component by +calling CONFIG BINDING SERVICE. More on this below. + +Components are required to pull their +**generated application configuration** at application startup using the environment +setting exposed during deployment. + + +Envs +~~~~ + +The platform provides a set of environment variables into each Docker +container: + ++----------------------------+--------------+----------------------------------------+ +| Name | Type | Description | ++============================+==============+========================================+ +| ``HOSTNAME`` | string | Unique name of the component instance | +| | | that is generated | ++----------------------------+--------------+----------------------------------------+ +| ``CONSUL_HOST`` | string | Hostname of the platform's Consul | +| | | instance | ++----------------------------+--------------+----------------------------------------+ +| ``CONFIG_BINDING_SERVICE`` | string | Hostname of the platform's config | +| | | binding service instance | +| | | | ++----------------------------+--------------+----------------------------------------+ +| ``DOCKER_HOST`` | string | Host of the target platform Docker | +| | | host to run the container on | ++----------------------------+--------------+----------------------------------------+ +| ``CBS_CONFIG_URL`` | string | Fully resolved URL to query config | +| | | from CONSUL via CBS | ++----------------------------+--------------+----------------------------------------+ + + +.. _config_binding_service: + +Config Binding Service +~~~~~~~~~~~~~~~~~~~~~~ + +The config binding service is a platform HTTP service that is +responsible for providing clients with its fully resolve configuration +JSON at startup, and also other configurations objects +when requested. + +At runtime, components should make an HTTP GET on: + +:: + + :/service_component/NAME + +For Docker components, NAME should be set to ``HOSTNAME``, which is +provided as an ENV variable to the container. + +The binding service integrates with the streams and services section of +the component specification. For example, if you specify that you call a +service: + +:: + + "services": { + "calls": [{ + "config_key": "vnf-db", + "request": { + "format": "dcae.vnf.meta", + "version": "1.0.0" + }, + "response": { + "format": "dcae.vnf.kpi", + "version": "1.0.0" + } + }], + ... + } + +Then the config binding service will find all available IP addresses of +services meeting the containers needs, and provide them to the container +under your ``config_key``: + +:: + + // your configuration + { + "vbf-db" : // see above + [IP:Port1, IP:Port2,…] // all of these meet your needs, choose one. + } + +Regarding ``:``, there is DNS +work going on to make this resolvable in a convenient way inside of your +container. + +For all Kubernetes deployments since El-Alto, an environment variable ``CBS_CONFIG_URL`` will be exposed +by platform (k8s plugins) providing the exact URL to be used for configuration retrieval. +Application can use this URL directly instead of constructing URL from HOSTNAME (which refers to ServiceComponentName) +and CONFIG_BINDING_SERVICE env's. By default, this URL will use HTTPS CBS interface + +If you are integrating with CBS SDK, then the DNS resolution and configuration fetch +are handled via library functions. + +Generated Application Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The DCAE platform uses the component specification to generate the +component’s application configuration provided at deployment time. The +component developer should expect to use this configuration JSON in the +component. + + +The following component spec snippet (from String Matching): + +:: + + "streams":{ + "subscribes": [{ + "format": "VES_specification", + "version": "4.27.2", + "type": "message_router", + "config_key" : "mr_input" + }], + "publishes": [{ + "format": "VES_specification", + "version": "4.27.2", + "config_key": "mr_output", + "type": "message_router" + }] + }, + "services":{ + "calls": [{ + "config_key" : "aai_broker_handle", + "verb": "GET", + "request": { + "format": "get_with_query_params", + "version": "1.0.0" + }, + "response": { + "format": "aai_broker_response", + "version": "3.0.0" + } + }], + "provides": [] + }, + +Will result in the following top level keys in the configuration + +:: + + "streams_publishes":{ + "mr_output":{ // notice the config key above + "aaf_password":"XXX", + "type":"message_router", + "dmaap_info":{ + "client_role": null, + "client_id": null, + "location": null, + "topic_url":"https://YOUR_HOST:3905/events/com.att.dcae.dmaap.FTL2.DCAE-CL-EVENT" // just an example + }, + "aaf_username":"XXX" + } + }, + "streams_subscribes":{ + "mr_input":{ // notice the config key above + "aaf_password":"XXX", + "type":"message_router", + "dmaap_info":{ + "client_role": null, + "client_id": null, + "location": null, + "topic_url":"https://YOUR_HOST:3905/events/com.att.dcae.dmaap.FTL2.TerrysStringMatchingTest" // just an example + }, + "aaf_username":"XXX" + } + }, + "services_calls":{ + "aai_broker_handle":[ // notice the config key above + "135.205.226.128:32768" // based on deployment time, just an example + ] + } + +These keys will always be populated whether they are empty or not. So +the minimum configuration you will get, (in the case of a component that +provides an HTTP service, doesn’t call any services, and has no streams, +is: + +:: + + "streams_publishes":{}, + "streams_subscribes":{}, + "services_calls":{} + +Thus your component should expect these well-known top level keys. + +DCAE SDK +~~~~~~~~ + +DCAE has SDK/libraries which can be used for service components for easy integration. + +- `Java Library `__ +- `Python Modules `__ + + + +.. _policy_reconfiguration: + +Policy Reconfiguration +---------------------- + +Components must provide a way to receive policy reconfiguration, that +is, configuration parameters that have been updated via the Policy UI. +The component developer must either periodically poll the ConfigBindingService API +to retrieve/refresh the new configuration or provides a script (defined in the :any:`Docker +auxiliary specification `) +that will be triggered when policy update is detected by the platform. + + +.. _docker_images: + +Docker Images +------------- + +Docker images must be pushed to the environment specific Nexus +repository. This requires tagging your build with the full name of you +image which includes the Nexus repository name. + +For ONAP microservices, the components images are expected to pushed into ONAP nexus +part of `ONAP CI jobs `__ + + +.. _operation_requirement: + +Operational Requirement +----------------------- + +Logging +~~~~~~~ + +All ONAP MS logging should follow logging specification defined by `logging project `__ + +The application log configuration must enable operation to choose if to be written into file or stdout or both during deployment. + + +S3P +~~~ +ONAP S3P (all scaling/resiliency/security/maintainability) goals should meet at the minimum level defined for DCAE project for the targeted release + +If the component is stateful, it should persist its state on external store (eg. pg, redis) to allow support for scaling and resiliency. This should be important design criteria for the component. If the components either publish/subscribe into DMAAP topic, then secure connection to DMAAP must be supported (platform will provide aaf_username/aaf_password for each topic as configuration). + diff --git a/docs/sections/images/1.png b/docs/sections/images/1.png deleted file mode 100644 index e734a673..00000000 Binary files a/docs/sections/images/1.png and /dev/null differ diff --git a/docs/sections/images/10.png b/docs/sections/images/10.png deleted file mode 100644 index 9a7f75ad..00000000 Binary files a/docs/sections/images/10.png and /dev/null differ diff --git a/docs/sections/images/11.png b/docs/sections/images/11.png deleted file mode 100644 index fe5ed014..00000000 Binary files a/docs/sections/images/11.png and /dev/null differ diff --git a/docs/sections/images/12.png b/docs/sections/images/12.png deleted file mode 100644 index f2d5a360..00000000 Binary files a/docs/sections/images/12.png and /dev/null differ diff --git a/docs/sections/images/13.png b/docs/sections/images/13.png deleted file mode 100644 index c63d1361..00000000 Binary files a/docs/sections/images/13.png and /dev/null differ diff --git a/docs/sections/images/14.png b/docs/sections/images/14.png deleted file mode 100644 index 5b507b10..00000000 Binary files a/docs/sections/images/14.png and /dev/null differ diff --git a/docs/sections/images/15.png b/docs/sections/images/15.png deleted file mode 100644 index 1529ef04..00000000 Binary files a/docs/sections/images/15.png and /dev/null differ diff --git a/docs/sections/images/16.png b/docs/sections/images/16.png deleted file mode 100644 index e3a7e36d..00000000 Binary files a/docs/sections/images/16.png and /dev/null differ diff --git a/docs/sections/images/17.png b/docs/sections/images/17.png deleted file mode 100644 index 54347d18..00000000 Binary files a/docs/sections/images/17.png and /dev/null differ diff --git a/docs/sections/images/18.png b/docs/sections/images/18.png deleted file mode 100644 index 150e1c61..00000000 Binary files a/docs/sections/images/18.png and /dev/null differ diff --git a/docs/sections/images/19.png b/docs/sections/images/19.png deleted file mode 100644 index b578b103..00000000 Binary files a/docs/sections/images/19.png and /dev/null differ diff --git a/docs/sections/images/2.png b/docs/sections/images/2.png deleted file mode 100644 index 20bd7a01..00000000 Binary files a/docs/sections/images/2.png and /dev/null differ diff --git a/docs/sections/images/20.png b/docs/sections/images/20.png deleted file mode 100644 index 80f56e92..00000000 Binary files a/docs/sections/images/20.png and /dev/null differ diff --git a/docs/sections/images/21.png b/docs/sections/images/21.png deleted file mode 100644 index c92a2346..00000000 Binary files a/docs/sections/images/21.png and /dev/null differ diff --git a/docs/sections/images/22.png b/docs/sections/images/22.png deleted file mode 100644 index bf4f1c02..00000000 Binary files a/docs/sections/images/22.png and /dev/null differ diff --git a/docs/sections/images/23.png b/docs/sections/images/23.png deleted file mode 100644 index 98fd7970..00000000 Binary files a/docs/sections/images/23.png and /dev/null differ diff --git a/docs/sections/images/24.png b/docs/sections/images/24.png deleted file mode 100644 index 2784fddf..00000000 Binary files a/docs/sections/images/24.png and /dev/null differ diff --git a/docs/sections/images/25.png b/docs/sections/images/25.png deleted file mode 100644 index 55464c46..00000000 Binary files a/docs/sections/images/25.png and /dev/null differ diff --git a/docs/sections/images/26.png b/docs/sections/images/26.png deleted file mode 100644 index 9486bb80..00000000 Binary files a/docs/sections/images/26.png and /dev/null differ diff --git a/docs/sections/images/27.png b/docs/sections/images/27.png deleted file mode 100644 index 3c0bd2f3..00000000 Binary files a/docs/sections/images/27.png and /dev/null differ diff --git a/docs/sections/images/3.png b/docs/sections/images/3.png deleted file mode 100644 index 7d1d9df0..00000000 Binary files a/docs/sections/images/3.png and /dev/null differ diff --git a/docs/sections/images/4.png b/docs/sections/images/4.png deleted file mode 100644 index 69706f63..00000000 Binary files a/docs/sections/images/4.png and /dev/null differ diff --git a/docs/sections/images/5.png b/docs/sections/images/5.png deleted file mode 100644 index ec158035..00000000 Binary files a/docs/sections/images/5.png and /dev/null differ diff --git a/docs/sections/images/6.png b/docs/sections/images/6.png deleted file mode 100644 index b3f7e7bc..00000000 Binary files a/docs/sections/images/6.png and /dev/null differ diff --git a/docs/sections/images/7.png b/docs/sections/images/7.png deleted file mode 100644 index adb8a941..00000000 Binary files a/docs/sections/images/7.png and /dev/null differ diff --git a/docs/sections/images/8.png b/docs/sections/images/8.png deleted file mode 100644 index e04c60ed..00000000 Binary files a/docs/sections/images/8.png and /dev/null differ diff --git a/docs/sections/images/9.png b/docs/sections/images/9.png deleted file mode 100644 index c18e9487..00000000 Binary files a/docs/sections/images/9.png and /dev/null differ diff --git a/docs/sections/images/DCAE-Mod-Architecture.png b/docs/sections/images/DCAE-Mod-Architecture.png deleted file mode 100644 index ce5fca92..00000000 Binary files a/docs/sections/images/DCAE-Mod-Architecture.png and /dev/null differ diff --git a/docs/sections/images/Onboarding-with-DCAE-MOD.png b/docs/sections/images/Onboarding-with-DCAE-MOD.png deleted file mode 100644 index 4883b65e..00000000 Binary files a/docs/sections/images/Onboarding-with-DCAE-MOD.png and /dev/null differ diff --git a/docs/sections/images/nifi-toolbar-components.png b/docs/sections/images/nifi-toolbar-components.png deleted file mode 100644 index fa90afef..00000000 Binary files a/docs/sections/images/nifi-toolbar-components.png and /dev/null differ diff --git a/docs/sections/release-notes.rst b/docs/sections/release-notes.rst index 849cd7b7..495f5918 100644 --- a/docs/sections/release-notes.rst +++ b/docs/sections/release-notes.rst @@ -3,11 +3,23 @@ .. Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. .. _release_notes: + + +================== DCAE Release Notes ================== +.. contents:: + :depth: 2 +.. + + +Version: 6.0.0 +============== + Abstract -------- + This document provides the release notes for the Frankfurt release (6.0.0) @@ -16,9 +28,10 @@ Summary **New Features** -DCAE improves upon previous release with the following new features: +DCAE improves upon previous release with the following new features - DCAE Platform Enhancement + - Introduction of Microservice and Onboarding Design (MOD) platform - Policy Notification support for DCAE components - Dynamic AAF certificate creation during component instantiation @@ -27,16 +40,22 @@ DCAE improves upon previous release with the following new features: - Blueprint generator tool to simplify deployment artifact creation -- Following new services are delivered with Frankfurt +- Following new services are delivered this release + - Event Processors + - PM Subscription Handler - DataLake Handlers - Analytics/RCA + - TCA-GEN2 + - Acumos Adapter (PoC) -Following is complete list of DCAE components available part of default ONAP/DCAE installation. +Below is complete list of DCAE components available part of default ONAP/DCAE installation. + - Platform components + - Cloudify Manager (helm chart) - Bootstrap container (helm chart) - Configuration Binding Service (helm chart) @@ -45,7 +64,9 @@ Following is complete list of DCAE components available part of default ONAP/DCA - Service Change Handler (helm chart) - Inventory API (helm chart) - Dashboard (helm chart) + - Service components + - VES Collector - Threshold Crossing Analytics (TCA/CDAP) - HV-VES Collector @@ -53,13 +74,16 @@ Following is complete list of DCAE components available part of default ONAP/DCA - Docker based Threshold Crossing Analytics (TCA-Gen2) - Holmes Rule Management * - Holmes Engine Management * + - Additional resources that DCAE utilizes deployed using ONAP common charts: + - Postgres Database - Mongo Database - Redis Cluster Database - Consul Cluster -Following are service components (mS) which can be deployed on-demand +Following service components (mS) can be deployed on-demand + - SNMPTrap Collector - RESTConf Collector - DataFile Collector @@ -71,6 +95,7 @@ Following are service components (mS) which can be deployed on-demand - PM-Subscription Handler Notes: + \* These components are delivered by the Holmes project. @@ -84,6 +109,7 @@ Under OOM (Kubernetes) deployment all DCAE component containers are deployed as **Bug Fixes** **Known Issues** + * Blueprint generator issue (DCAEGEN2-2140) * TCAgen2 Policy configuration support (DCAEGEN2-2198) @@ -152,8 +178,8 @@ Artifacts released: "dcaegen2/services/mapper", "", "onap/org.onap.dcaegen2.services.mapper.vesadapter.universalvesadaptor:1.0.1" "dcaegen2/services/pm-mapper", "", "onap/org.onap.dcaegen2.services.pm-mapper:1.3.1" "dcaegen2/services/prh", "", "onap/org.onap.dcaegen2.services.prh.prh-app-server:1.5.2" - "dcaegen2/services/son-handler", "", "onap/org.onap.dcaegen2.services.son-handler:2.0.0" - "dcaegen2/platform", "adapter/acumos", "onap/org.onap.dcaegen2.platform.adapter.acumos:1.0.1" + "dcaegen2/services/son-handler", "", "onap/org.onap.dcaegen2.services.son-handler:2.0.2" + "dcaegen2/platform", "adapter/acumos", "onap/org.onap.dcaegen2.platform.adapter.acumos:1.0.2" "dcaegen2/platform", "mod/bpgenerator", "Blueprint Generator 1.3.1 (jar)" "dcaegen2/services/sdk", "", "DCAE SDK 1.3.5 (jar)" "ccsdk/dashboard", "", "onap/org.onap.ccsdk.dashboard.ccsdk-app-os:1.3.2" @@ -165,11 +191,12 @@ Artifacts released: **References** + - `DCAE project page `_ - `Passing Badge information for DCAE `_ Version: 5.0.1 --------------- +============== The offical El-Alto release (rolls up all 5.0.0 early drop deliverables) focused on technical debts and SECCOM priority work-items. @@ -332,7 +359,7 @@ The following components are upgraded from Dublin/R4 and El-Alto EarlyDrop deliv - Description : Code optimization, bug fixes and SDK alignment Version: 5.0.0 --------------- +============== El-Alto Early-drop focused on technical debts and SECCOM priority work-items. @@ -425,7 +452,7 @@ The following components are upgraded from Dublin/R4. Version: 4.0.0 --------------- +============== :Release Date: 2019-06-06 @@ -627,7 +654,7 @@ The following components are upgraded from R3 Version: 3.0.1 --------------- +============== :Release Date: 2019-01-31 @@ -814,7 +841,7 @@ The following components are upgraded from R2: Version: 2.0.0 --------------- +============== :Release Date: 2018-06-07 @@ -930,7 +957,7 @@ The following components are upgraded from R1: Version: 1.0.0 --------------- +============== :Release Date: 2017-11-16 -- cgit 1.2.3-korg