diff options
Diffstat (limited to 'docs')
| -rw-r--r-- | docs/.gitignore | 3 | ||||
| -rw-r--r-- | docs/MUSIC_USAGE.jpg | bin | 0 -> 47005 bytes | |||
| -rw-r--r-- | docs/_static/css/ribbon.css | 63 | ||||
| -rwxr-xr-x | docs/_static/favicon.ico | bin | 0 -> 2102 bytes | |||
| -rw-r--r-- | docs/_static/logo_onap_2017.png | bin | 0 -> 12278 bytes | |||
| -rw-r--r-- | docs/architecture.rst | 51 | ||||
| -rw-r--r-- | docs/conf.py | 15 | ||||
| -rw-r--r-- | docs/conf.yaml | 7 | ||||
| -rw-r--r-- | docs/delivery.rst | 18 | ||||
| -rwxr-xr-x | docs/index.rst | 1 | ||||
| -rw-r--r-- | docs/multi.rst | 57 | ||||
| -rw-r--r-- | docs/offeredapis.rst | 16 | ||||
| -rw-r--r-- | docs/release-notes.rst | 50 | ||||
| -rw-r--r-- | docs/requirements-docs.txt | 15 | ||||
| -rw-r--r-- | docs/setup.rst | 11 | ||||
| -rw-r--r-- | docs/single.rst | 3 | ||||
| -rw-r--r-- | docs/swagger.json | 65 | ||||
| -rw-r--r-- | docs/tox.ini | 22 |
18 files changed, 259 insertions, 138 deletions
diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..43ca5b67 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +/.tox +/_build/* +/__pycache__/* diff --git a/docs/MUSIC_USAGE.jpg b/docs/MUSIC_USAGE.jpg Binary files differnew file mode 100644 index 00000000..66773c6c --- /dev/null +++ b/docs/MUSIC_USAGE.jpg diff --git a/docs/_static/css/ribbon.css b/docs/_static/css/ribbon.css new file mode 100644 index 00000000..6008cb1a --- /dev/null +++ b/docs/_static/css/ribbon.css @@ -0,0 +1,63 @@ +.ribbon { + z-index: 1000; + background-color: #a00; + overflow: hidden; + white-space: nowrap; + position: fixed; + top: 25px; + right: -50px; + -webkit-transform: rotate(45deg); + -moz-transform: rotate(45deg); + -ms-transform: rotate(45deg); + -o-transform: rotate(45deg); + transform: rotate(45deg); + -webkit-box-shadow: 0 0 10px #888; + -moz-box-shadow: 0 0 10px #888; + box-shadow: 0 0 10px #888; + +} + +.ribbon a { + border: 1px solid #faa; + color: #fff; + display: block; + font: bold 81.25% 'Helvetica Neue', Helvetica, Arial, sans-serif; + margin: 1px 0; + padding: 10px 50px; + text-align: center; + text-decoration: none; + text-shadow: 0 0 5px #444; + transition: 0.5s; +} + +.ribbon a:hover { + background: #c11; + color: #fff; +} + + +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td, .wy-table-responsive table th { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} + +@media screen and (max-width: 767px) { + .wy-table-responsive table td { + white-space: nowrap; + } +} + +/* fix width of the screen */ + +.wy-nav-content { + max-width: none; +} diff --git a/docs/_static/favicon.ico b/docs/_static/favicon.ico Binary files differnew file mode 100755 index 00000000..cb712ebd --- /dev/null +++ b/docs/_static/favicon.ico diff --git a/docs/_static/logo_onap_2017.png b/docs/_static/logo_onap_2017.png Binary files differnew file mode 100644 index 00000000..5d064f43 --- /dev/null +++ b/docs/_static/logo_onap_2017.png diff --git a/docs/architecture.rst b/docs/architecture.rst index ae29f9e0..73cfbc2d 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -1,19 +1,60 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 +.. _architecture: Architecture ============ -` See ONAP Wiki <https://wiki.onap.org/display/DW/MUSIC-Multi-site+State+Coordination+Service>`_ +Project Description +------------------- +To achieve five 9s of availability on three 9s or lower software and infrastructure in a cost-effective manner, ONAP components need to work in a reliable, active-active manner across multiple sites (platform-maturity resiliency level 3). A fundamental aspect of this is state management across geo-distributed sites in a reliable, scalable, highly available and efficient manner. This is an important and challenging problem because of three fundamental reasons: -Capabilities +- Current solutions for state-management of ONAP components like MariaDB clustering, that work very effectively within a site, may not scale across geo-distributed sites (e.g., Beijing, Amsterdam and Irvine) or allow partitioned operation (thereby compromising availability). This is mainly because WAN latencies are much higher across sites and frequent network partitions can occur. + +- ONAP components often have a diverse range of requirements in terms of state replication. While some components need to synchronously manage state across replicas, others may tolerate asynchronous replication. This diversity needs to be leveraged to provide better performance and higher availability across sites. + +- ONAP components often need to partition state across different replicas, perform consistent operations on them and ensure that on failover, the new owner has access to the latest state. The distributed protocols to achieve such consistent ownership is complex and replete with corners cases, especially in the face of network partitions. Currently, each component is building its own handcrafted solution which is wasteful and worse, can be erroneous. + +In this project, we identify common state management concerns across ONAP components and provide a multi-site state coordination/management service (MUSIC) with a rich suite of recipes that each ONAP component can simply configure and use for their state-management needs. + + +Functionality +------------- + +At its core, MUSIC provides a scalable sharded eventually-consistent data-store (Cassandra) wherein the access to the keys can be protected using a locking service (built on Zookeeper) that is tightly coupled with the data-store. ONAP components can use the MUSIC API directly to store and access their state across geo-distributed sites. This API enables ONAP components to achieve fine-grained flexible consistency on their state. + +MUSIC also provides a rich set of recipes (mdbc, prom, musicCAS, musicQ) that ONAP components can use to build multi-site active-active/active-passive/federated state-management solutions: + +- mdbc: The most crucial recipe is a multi-site database cache (mdbc) that enable ONAP components that maintain state in a SQL database to avail the benefits of MUSIC without compromising their need to use transactional SQL DBs. These ONAP components can rely on existing db clustering techniques like MariaDB for transactionality and complex querying within a site. mdbc will intercept each of these read/write calls to the db cluster and mirror this state to other geo-distributed sites through MUSIC either synchronously or asynchronously (configurable at a table-level). For example, components like the ONAP Service Orchestrator that use MariaDB to maintain state can directly use this recipe with no change to their SQL code. + +- prom: MUSIC provides a recipe for policy-driven ownership management (prom) of state for ONAP components to (1) partition state across replicas during both initial placement and during failures based on their individual policies (2) ensure correct transfer of state ownership across replicas during site failures and network partitions (3) ensure that the new owner has access to the most recent version of state (if needed). + +- musicCAS: The distributed compare and set is a powerful primitive that will allow ONAP components to update shared state in an atomic manner. This is currently being used by the ONAP HAS (homing service) that is structured a job scheduler with multiple workers trying to pick up client-submitted jobs, while ensuring that only one of them actually performs the job. + +- musicQ: Implementing a geo-distributed queue is a hard problem with many performance implications when data belonging to a queue is sharded across nodes. MUSIC provides a queue API that carefully structures the data to ensure good performance. ONAP HAS (mentioned above) uses this as its job queue. + + +Scope +----- +MUSIC is a shared service with recipes that individual ONAP components and micro-service can use for state replication, consistency management and state ownership across geo-distributed sites. MUSIC will make sure that the right service data is available at the right place, and at the right time to enable federated active-active operation of ONAP. For example, we envisage the use of MUSIC for multi-site state management in SO (to store Camunda state across sites), <SDN-C, AppC> (to store ODL related state across sites) , A&AI (to store its graph data) and most other ONAP components that need to manage state across sites. + + +Out of Scope ------------ +While MUSIC is an optional solution for state-management of ONAP components across sites, OOM will continue to manage component level and platform level deployment, scalability, redundancy, resiliency, self-healing and high availability on top of Kubernetes across sites for ONAP. -Usage Scenarios ---------------- +Usage +----- +MUSIC and its recipes export a REST API apart from mdbc which is implemented as a jdbc driver to enable seamless integration with SQL-based ONAP components. -Interactions + +Architecture ------------ +The figures below how MUSIC can be used in a general context and also provide a specific example of its potential usage in ONAP SO. + +.. image:: MUSIC_USAGE.jpg + + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..96a85251 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,15 @@ +from docs_conf.conf import * + +branch = 'latest' +master_doc = 'index' + +linkcheck_ignore = [ + 'http://localhost', +] + +intersphinx_mapping = {} + +html_last_updated_fmt = '%d-%b-%y %H:%M' + +def setup(app): + app.add_stylesheet("css/ribbon.css")
\ No newline at end of file diff --git a/docs/conf.yaml b/docs/conf.yaml new file mode 100644 index 00000000..ab592813 --- /dev/null +++ b/docs/conf.yaml @@ -0,0 +1,7 @@ +--- +project_cfg: onap +project: onap + +# Change this to ReleaseBranchName to modify the header +default-version: latest +# diff --git a/docs/delivery.rst b/docs/delivery.rst index f3f083a7..c90ff818 100644 --- a/docs/delivery.rst +++ b/docs/delivery.rst @@ -27,17 +27,17 @@ Example use of a block diagram. m -> y; m -> z; group l1 { - color = blue; - x; y; z; - } + color = blue; + x; y; z; + } group l2 { - color = yellow; - m; n; - } + color = yellow; + m; n; + } group l3 { - color = orange; - a; b; c; - } + color = orange; + a; b; c; + } } diff --git a/docs/index.rst b/docs/index.rst index 437a50d8..19822636 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,7 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 .. Copyright 2017 AT&T Intellectual Property. All rights reserved. +.. _master_index: Music Developer Documentation ============================= diff --git a/docs/multi.rst b/docs/multi.rst index bc26465f..380f4ea5 100644 --- a/docs/multi.rst +++ b/docs/multi.rst @@ -72,63 +72,6 @@ To access keyspace through cqlsh, login with credentials that are passed to MUSI -Zookeeper: ----------- -Once zookeeper has been installed on all the nodes, modify the **zk_install_location/conf/zoo.cfg** on all the nodes with the following lines: - -:: - - tickTime=2000 - dataDir=/opt/app/music/var/zookeeper - clientPort=2181 - initLimit=5 - syncLimit=2 - quorumListenOnAllIPs=true - server.1=public IP of node 1:2888:3888 - server.2=public IP of node 2:2888:3888 - server.3=public IP of node 3:2888:3888 - -Create the directory /var/zookeeper in all the machines and within that create a file called myid that contains the id of the machine. The machine running node.i will contain just the number i in the file myid. - -Start each of the nodes one by one from the zk_install_location/bin folder using the command: - - - - ./zkServer.sh start - -On each node check the file zookeeper.out in the zk_install_location/ bin to make sure all the machines are talking to each other and there are no errors. Note that while the machines are yet to come up there maybe error messages saying that connection has not yet been established. Clearly, this is ok. - - -If there are no errors, then from zk_install_location/bin simply run the following to get command line access to zookeeper. ./zkCli.sh - - -Run these commands on different machines to make sure the zk nodes are syncing. - -:: - - [zkshell] ls / - [zookeeper] - -Next, create a new znode by running - -:: - - create /zk_test my_data. - -This creates a new znode and associates the string "my_data" with the node. You should see: - -:: - - [zkshell] create /zk_test my_data - Created /zk_test - -Issue another ls / command to see what the directory looks like: - -:: - - [zkshell] ls / - [zookeeper, zk_test] - MUSIC Create a music.properties file and place it in /opt/app/music/etc at each node. Here is a sample of the file: cassandra.yaml:: diff --git a/docs/offeredapis.rst b/docs/offeredapis.rst index 3dcc7962..c75bd0c0 100644 --- a/docs/offeredapis.rst +++ b/docs/offeredapis.rst @@ -1,16 +1,16 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. This work is licensed under a + Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 +.. _offeredapis: Offered APIs ============ -.. note:: - * This section is used to describe the external interfaces offered by a software component - - * This section is typically: provided for a platform-component and sdk; and - referenced in developer guides and api reference manuals. - - * This note must be removed after content has been added. +This document describes the MUSIC (Multi-site State Coordination Service) API +To view API documentation in the interactive swagger UI download the following +and paste into the swagger tool here: https://editor.swagger.io + +:download:`swagger.json <swagger.json>` .. swaggerv2doc:: swagger.json diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 766fb123..5e8885da 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -1,60 +1,70 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. This work is licensed under a +.. Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 +.. _release_notes: Release Notes ============= -Initial Release for Beijing +Initial Release for Frankfurt -Version: 2.5.4 --------------- +Version: 3.2.40 +--------------- -:Release Date: 2018-06-07 +:Release Date: 2020-05-20 **New Features** -MUSIC provides a service with recipes that individual ONAP components and microservice can use for - - state replication across multiple geo-distributed sites - - flexible fine-grained consistency management of state - - policy driven state-ownership across replicas of the service +- MUSIC now runs on a springboot server, instead of a standalone tomcat server +- HTTPS support for clients through AAF certificates -**Bug Fixes** +- A background lock clean up daemon will periodically check the status of + current locks, cleaning up 'stale' references. + Clients should see a performance boost if they were previously dealing with + many stale locks. + +- Improved error messaging to the user, allowing clients to better debug their + applications + +- Continued adherence to ONAP S3P requirements - - `MUSIC-33 <https://jira.onap.org/projects/MUSIC/issues/MUSIC-33>`_ CriticalPut is causing HTTP Status 500 – Internal Server Error - - `MUSIC-34 <https://jira.onap.org/projects/MUSIC/issues/MUSIC-34>`_ CreateTable,InsertIntoTable,UpdateTable,DeleteFromTable,DropTable don't check resource existence +**Bug Fixes** + - `MUSIC-573 <https://jira.onap.org/projects/MUSIC/issues/MUSIC-573>`_ Pods still run as root + + - `MUSIC-557 <https://jira.onap.org/projects/MUSIC/issues/MUSIC-557>`_ Test coverage goals met, and migrated to new sonar location - - `MUSIC-35 <https://jira.onap.org/projects/MUSIC/issues/MUSIC-35>`_ Onboarding issues + - `MUSIC-530 <https://jira.onap.org/browse/MUSIC-530>`_ Security Vulnerability in pom.xml fix - - `MUSIC-36 <https://jira.onap.org/projects/MUSIC/issues/MUSIC-36>`_ Test Case failing - - `MUSIC-37 <https://jira.onap.org/projects/MUSIC/issues/MUSIC-37>`_ Acquire lock to return more information **Known Issues** N/A **Security Notes** -MUSIC code has been formally scanned during build time using NexusIQ and all Critical vulnerabilities have been addressed, items that remain open have been assessed for risk and determined to be false positive. The MUSIC open Critical security vulnerabilities and their risk assessment have been documented as part of the `project <https://wiki.onap.org/pages/viewpage.action?pageId=25439359>`_. +MUSIC code has been formally scanned during build time using NexusIQ and all Critical vulnerabilities have been addressed, items that remain open have been assessed for risk and determined to be false positive. The MUSIC open Critical security vulnerabilities and their risk assessment have been documented as part of the `project <https://wiki.onap.org/pages/viewpage.action?pageId=45285410>`_. Quick Links: - `MUSIC project page <https://wiki.onap.org/display/DW/MUSIC+Project>`_ +- `MUSIC Frankfurt Release <https://wiki.onap.org/display/DW/MUSIC+Frankfurt>`_ - `Passing Badge information for MUSIC <https://bestpractices.coreinfrastructure.org/en/projects/1722>`_ -- `Project Vulnerability Review Table for MUSIC <https://wiki.onap.org/pages/viewpage.action?pageId=25439359>`_ +- `MUSIC Architecture Page <TBD>`_ +- `Project Vulnerability Review Table for MUSIC <https://wiki.onap.org/pages/viewpage.action?pageId=64004601>`_ **Upgrade Notes** - N/A + N/A **Deprecation Notes** - N/A + N/A **Other** - N/A + N/A =========== diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 00000000..b3188ddd --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1,15 @@ +tox +Sphinx +doc8 +docutils +setuptools +six +sphinx_rtd_theme>=0.4.3 +sphinxcontrib-blockdiag +sphinxcontrib-needs>=0.2.3 +sphinxcontrib-nwdiag +sphinxcontrib-seqdiag +sphinxcontrib-swaggerdoc +sphinxcontrib-plantuml +sphinx_bootstrap_theme +lfdocs-conf diff --git a/docs/setup.rst b/docs/setup.rst index b12acd04..7fb5bf49 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -17,27 +17,20 @@ The main MUSIC dir should be:: # These also need to be set up /opt/app/music/etc /opt/app/music/logs - /opt/app/music/lib/zookeeper -When installing Tomcat, Cassandra and Zookeeper they should also be installed here.:: +When installing, Cassandra should also be installed here.:: /opt/app/music/apache-cassandra-n.n.n - /opt/app/music/zookeeper-n.n.n - /opt/app/music/apache-tomcat-n.n.n You could also create links from install dirs to a common name ie\::: ln -s /opt/app/music/apache-cassandra-n.n.n cassandra - ln -s /opt/app/music/zookeeper-n.n.n zookeeper - ln -s /opt/app/music/apache-tomcat-n.n.n tomcat -Cassandra and Zookeeper have data dirs.:: +Cassandra has data dirs.:: # For cassandra it should be (This is the default) /opt/app/music/cassandra/data - # For Zookeeper it should be - /opt/app/music/zookeeper/ Continue by selecting the link to the setup you are doing. diff --git a/docs/single.rst b/docs/single.rst index 060f02b4..632db5ac 100644 --- a/docs/single.rst +++ b/docs/single.rst @@ -15,8 +15,6 @@ Instructions - Ensure you have OpenJDK 8 on your machine. - Download Apache Cassandra 3.0, install into /opt/app/music and follow these instructions http://cassandra.apache.org/doc/latest/getting_started/installing.html till and including Step - By the end of this you should have Cassandra working. -- Download Apache Zookeeper 3.4.6, install into /opt/app/music and follow these instructions https://zookeeper.apache.org/doc/r3.4.6/zookeeperStarted.html pertaining to the standalone operation. By the end of this you should have Zookeeper working. -- Download the Version 8.5 Apache Tomcat and install it using these instructions https://tomcat.apache.org/download-80.cgi (this is for version 8.5). - Create a music.properties file and place it in /opt/app/music/etc/. Here is a sample of the file: music.properties:: @@ -30,7 +28,6 @@ music.properties:: # Optional current values are defaults ###################################### # If using docker this would point to the specific docker name. - #zookeeper.host=localhost #cassandra.host=localhost #music.ip=localhost diff --git a/docs/swagger.json b/docs/swagger.json index 27db769f..3bc67050 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -875,26 +875,43 @@ } } }, - "/v{version}/service/pingCassandra/{consistency}": { + "/v{version}/service/pingZookeeper": { "get": { "tags": [ "Healthcheck Api" ], "summary": "Get Health Status", "description": "", - "operationId": "cassandraStatus", + "operationId": "ZKStatus", "produces": [ "application/json" ], - "parameters": [ - { - "name": "consistency", - "in": "path", - "description": "Consistency level", - "required": true, - "type": "string" + "parameters": [], + "responses": { + "200": { + "description": "successful operation", + "schema": { + "type": "object", + "additionalProperties": { + "type": "object" + } + } } + } + } + }, + "/v{version}/service/musicHealthCheck": { + "get": { + "tags": [ + "Healthcheck Api" ], + "summary": "Get Health Status", + "description": "", + "operationId": "musicHealthCheck", + "produces": [ + "application/json" + ], + "parameters": [], "responses": { "200": { "description": "successful operation", @@ -908,18 +925,26 @@ } } }, - "/v{version}/service/pingZookeeper": { + "/v{version}/service/pingCassandra/{consistency}": { "get": { "tags": [ "Healthcheck Api" ], "summary": "Get Health Status", "description": "", - "operationId": "ZKStatus", + "operationId": "cassandraStatus", "produces": [ "application/json" ], - "parameters": [], + "parameters": [ + { + "name": "consistency", + "in": "path", + "description": "Consistency level", + "required": true, + "type": "string" + } + ], "responses": { "200": { "description": "successful operation", @@ -939,7 +964,7 @@ "Lock Api" ], "summary": "Create Lock", - "description": "Puts the requesting process in the q for this lock. The corresponding node will be created in zookeeper if it did not already exist. Lock Name is the \"key\" of the form keyspaceName.tableName.rowId", + "description": "Puts the requesting process in the q for this lock. The corresponding lock will be created if it did not already exist. Lock Name is the \"key\" of the form keyspaceName.tableName.rowId", "operationId": "createLockReference", "produces": [ "application/json" @@ -1269,20 +1294,6 @@ "description": "Application namespace", "required": true, "type": "string" - }, - { - "name": "userId", - "in": "header", - "description": "userId", - "required": true, - "type": "string" - }, - { - "name": "password", - "in": "header", - "description": "Password", - "required": true, - "type": "string" } ], "responses": { diff --git a/docs/tox.ini b/docs/tox.ini new file mode 100644 index 00000000..edac8c35 --- /dev/null +++ b/docs/tox.ini @@ -0,0 +1,22 @@ +[tox] +minversion = 1.6 +envlist = docs, +skipsdist = true + +[testenv:docs] +basepython = python3 +deps = -r{toxinidir}/requirements-docs.txt +commands = + sphinx-build -b html -n -d {envtmpdir}/doctrees ./ {toxinidir}/_build/html + echo "Generated docs available in {toxinidir}/_build/html" +whitelist_externals = + echo + git + sh + +[testenv:docs-linkcheck] +basepython = python3 +#deps = -r{toxinidir}/requirements-docs.txt +commands = echo "Link Checking not enforced" +#commands = sphinx-build -b linkcheck -d {envtmpdir}/doctrees ./ {toxinidir}/_build/linkcheck +whitelist_externals = echo |
