diff options
| -rw-r--r-- | docs/sections/apis/VesEvent.proto | 75 | ||||
| -rw-r--r-- | docs/sections/apis/ves-hv.rst | 38 | ||||
| -rw-r--r-- | docs/sections/installation_oom.rst | 37 | ||||
| -rw-r--r-- | docs/sections/services/ves-hv/configuration.rst | 31 | ||||
| -rw-r--r-- | docs/sections/services/ves-hv/delivery.rst | 9 | ||||
| -rw-r--r-- | docs/sections/services/ves-hv/design.rst | 51 | ||||
| -rw-r--r-- | docs/sections/services/ves-hv/index.rst | 24 | ||||
| -rw-r--r-- | docs/sections/services/ves-hv/installation.rst | 27 |
8 files changed, 274 insertions, 18 deletions
diff --git a/docs/sections/apis/VesEvent.proto b/docs/sections/apis/VesEvent.proto new file mode 100644 index 00000000..0f9e5e1f --- /dev/null +++ b/docs/sections/apis/VesEvent.proto @@ -0,0 +1,75 @@ +/* + * ============LICENSE_START======================================================= + * dcaegen2-collectors-veshv + * ================================================================================ + * Copyright (C) 2018 NOKIA + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +syntax = "proto3"; +package org.onap.ves; + +message VesEvent { + CommonEventHeader commonEventHeader=1; // required + + bytes eventFields=2; // required, payload + // this field contains a domain-specific GPB message + // the field being opaque (bytes), the decoding of the payload occurs in a separate step + // the name of the GPB message for domain XYZ is XYZFields + // e.g. for domain==HVMEAS, the GPB message is HVMEASFields +} + +// VES CommonEventHeader adapted to GPB (Google Protocol Buffers) +// Aligned with VES 7.0.1 schema, and extending to hvMeas Domain. + +message CommonEventHeader +{ + string version = 1; // required, "version of the gpb common event header" + string domain = 2; // required, "the eventing domain associated with the event", allowed values: + // FAULT, HEARTBEAT, MEASUREMENT, MOBILE_FLOW, OTHER, PNFREGISTRATION, SIP_SIGNALING, + // STATE_CHANGE, SYSLOG, THRESHOLD_CROSSING_ALERT, VOICE_QUALITY, HVMEAS + + uint32 sequence = 3; // required, "ordering of events communicated by an event source instance or 0 if not needed" + + enum Priority + { + PRIORITY_NOT_PROVIDED = 0; + HIGH = 1; + MEDIUM = 2; + NORMAL = 3; + LOW = 4; + } + Priority priority = 4; // required, "processing priority" + + string eventId = 5; // required, "event key that is unique to the event source" + string eventName = 6; // required, "unique event name" + string eventType = 7; // "for example - guest05, platform" + + uint64 lastEpochMicrosec = 8; // required, "the latest unix time aka epoch time associated with the event from any component--as microseconds elapsed since 1 Jan 1970 not including leap seconds" + uint64 startEpochMicrosec = 9; // required, "the earliest unix time aka epoch time associated with the event from any component--as microseconds elapsed since 1 Jan 1970 not including leap seconds" + + string nfNamingCode = 10; // "4 character network function type, aligned with vnf naming standards" + string nfcNamingCode = 11; // "3 character network function component type, aligned with vfc naming standards" + string nfVendorName = 12; // " Vendor Name providing the nf " + + bytes reportingEntityId = 13; // "UUID identifying the entity reporting the event, for example an OAM VM; must be populated by the ATT enrichment process" + string reportingEntityName = 14; // required, "name of the entity reporting the event, for example, an EMS name; may be the same as sourceName should match A&AI entry" + bytes sourceId = 15; // "UUID identifying the entity experiencing the event issue; must be populated by the ATT enrichment process" + string sourceName = 16; // required, "name of the entity experiencing the event issued use A&AI entry" + string timeZoneOffset = 17; // "Offset to GMT to indicate local time zone for the device" + string vesEventListenerVersion = 18; // required, "Version of the VesEvent Listener" + + reserved "InternalHeaderFields"; // "enrichment fields for internal VES Event Listener service use only, not supplied by event sources" + reserved 100; +} diff --git a/docs/sections/apis/ves-hv.rst b/docs/sections/apis/ves-hv.rst index d2071077..2045b6ba 100644 --- a/docs/sections/apis/ves-hv.rst +++ b/docs/sections/apis/ves-hv.rst @@ -14,5 +14,41 @@ HV-VES (High Volume VES) Overview ======== -TODO +Component description can be found under `HV-VES Collector`_. + +.. _HV-VES Collector: ../services/ves-hv/index.html + + +TCP Endpoint +============ + +HV-VES is exposed as NodePort service on Kubernetes cluster on port 30222/tcp. +It uses plain TCP connections tunneled in SSL/TLS or can be run in insecure manner without data encryption on the socket. +Without TLS client authentication/authorization is not possible. +Connections are stream-based (as opposed to request-based) and long running. + +Payload is binary-encoded, currently using Google Protocol Buffers representation of the VES Common Header. + +.. literalinclude:: VesEvent.proto + :language: protobuf + +The PROTO file, which contains the VES CommonHeader, comes with a binary-type Payload parameter, where domain-specific +data shall be placed. Domain-specific data are encoded as well with GPB, and they do require a domain-specific +PROTO file to decode the data. + +HV-VES makes routing decisions based mostly on the content of the **Domain** parameter in VES Common Header. + + +Healthcheck +=========== + +Inside HV-VES docker container runs small http service for healthcheck - exact port for this service can be configured +at deployment using `--health-check-api-port` command line option. + +This service exposes single endpoint **GET /health/ready** which returns **HTTP 200 OK** in case HV-VES is healthy +and ready for connections. Otherwise it returns **HTTP 503 Service Unavailable** with short reason of unhealthiness. + + + + diff --git a/docs/sections/installation_oom.rst b/docs/sections/installation_oom.rst index 1c4e449a..f5c58bed 100644 --- a/docs/sections/installation_oom.rst +++ b/docs/sections/installation_oom.rst @@ -120,10 +120,11 @@ DCAE Service Endpoints ---------------------- Below is a table of default hostnames and ports for DCAE component service endpoints in Kuubernetes deployment: - ================== ============================ ================================ + ================== ============================ ================================= Component Cluster Internal (host:port) Cluster external (svc_name:port) - ================== ============================ ================================ + ================== ============================ ================================= VES dcae-ves-collector:8080 xdcae-ves-collector.onap:30235 + HV-VES dcae-hv-ves-collector:6061 xdcae-hv-ves-collector.onap:30222 TCA dcae-tca-analytics:11011 xdcae-tca-analytics.onap:32010 Policy Handler policy-handler:25577 NA Deployment Handler deployment-handler:8443 NA @@ -131,19 +132,21 @@ Below is a table of default hostnames and ports for DCAE component service endpo Config binding config-binding-service:10000 NA DCAE Healthcheck dcae-healthcheck:80 NA Cloudify Manager dcae-cloudify-manager:80 NA - ================== ============================ ================================ - -In addition, a number of ONAP service endpoints that are used by DCAE components are listed as follows for reference by DCAE developers and testers: - ================== ============================ ================================ - Component Cluster Internal (host:port) Cluster external (svc_name:port) - ================== ============================ ================================ - Consul Server consul-server:8500 consul-server:30270 - Robot robot:88 robot:30209 TCP - Message router message-router:3904 message-router:30227 - Message router message-router:3905 message-router:30226 - MSB Discovery msb-discovery:10081 msb-discovery:30281 - Logging log-kibana:5601 log-kibana:30253 - AAI aai:8080 aai:30232 - AAI aai:8443 aai:30233 - ================== ============================ ================================ + ================== ============================ ================================= + +In addition, a number of ONAP service endpoints that are used by DCAE components are listed as follows +for reference by DCAE developers and testers: + ==================== ============================ ================================ + Component Cluster Internal (host:port) Cluster external (svc_name:port) + ==================== ============================ ================================ + Consul Server consul-server:8500 consul-server:30270 + Robot robot:88 robot:30209 TCP + Message router message-router:3904 message-router:30227 + Message router message-router:3905 message-router:30226 + Message router Kafka message-router-kafka:9092 NA + MSB Discovery msb-discovery:10081 msb-discovery:30281 + Logging log-kibana:5601 log-kibana:30253 + AAI aai:8080 aai:30232 + AAI aai:8443 aai:30233 + ==================== ============================ ================================ diff --git a/docs/sections/services/ves-hv/configuration.rst b/docs/sections/services/ves-hv/configuration.rst new file mode 100644 index 00000000..f2b726f4 --- /dev/null +++ b/docs/sections/services/ves-hv/configuration.rst @@ -0,0 +1,31 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Configuration +============= + +HV-VES expects to be able to fetch configuration directly from consul service in following JSON format: + +.. code-block:: json + + { + "dmaap.kafkaBootstrapServers": "kafka-host:9093", + "collector.routing": [ + { + "fromDomain": "PERF3GPP", + "toTopic": "topic-1" + }, + { + "fromDomain": "HEARTBEAT", + "toTopic": "topic-2" + }, + ... + ] + } + + +During ONAP OOM/Kubernetes deployment this configuration is created from HV-VES cloudify blueprint. + +Endpoint on which HV-VES seeks configuration can be configured during deployment as described in installation_. + +.. _installation: ./installation.html
\ No newline at end of file diff --git a/docs/sections/services/ves-hv/delivery.rst b/docs/sections/services/ves-hv/delivery.rst new file mode 100644 index 00000000..74db3c05 --- /dev/null +++ b/docs/sections/services/ves-hv/delivery.rst @@ -0,0 +1,9 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Delivery +======== + +**HV-VES** is delivered as a docker container and published in ONAP Nexus repository following image naming convention. + +Full image name is `onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-main`.
\ No newline at end of file diff --git a/docs/sections/services/ves-hv/design.rst b/docs/sections/services/ves-hv/design.rst new file mode 100644 index 00000000..8e7ce7ad --- /dev/null +++ b/docs/sections/services/ves-hv/design.rst @@ -0,0 +1,51 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Design +====== + + +Compatibility aspects (VES-JSON) +-------------------------------- + +HV-VES Collector has been designed as a high-volume variant of the existing VES(JSON) collector, and not a completely new collector. +HV-VES follows the VES-JSON schema - as much as possible + +- It uses a Google Protocol Buffers ( GPB/PROTO ) representation of the VES Common Header +- The PROTO files tend to use most encoding effective types defined by GPB to cover Common Header fields. +- It makes routing decisions based mostly on the content of the "Domain" parameter +- It allows to embed Payload of different types (by default PERF3GPP domain is included) + +Analytics applications impacts + +- An analytics application operating on high-volume data needs to be prepared to read directly from Kafka +- An analytics application need to operate on GPB encoded data in order to benefit from GPB encoding efficiencies +- It is assumed, that due to the nature of high volume data, there would have to be dedicated applications provided, +able to operate on such volumes of data. + +Extendability +------------- + +HV-VES was designed to allow for extendability - by adding new domain-specific PROTO files. + +The PROTO file, which contains the VES CommonHeader, comes with a binary-type Payload parameter, where domain-specific data shall be placed. +Domain-specific data are encoded as well with GPB, and they do require a domain-specific PROTO file to decode the data. +This domain-specific PROTO needs to be shared with analytics applications - HV-VES is not analyzing domain-specific data. + +In order to support the RT-PM use-case, HV-VES includes a "PERF3GPP" domain PROTO file, as within this domain, +the high volume data is expected to be reported to HV-VES collector. +Still, there are no limitations to define additional domains, based on existing VES domains (like Fault, Heartbeat) +or completely new domains. New domains can be added "when needed". + +GPB PROTO files are backwards compatible, and such a new domain could be added without affecting existing systems. + +Analytics applications will have to be as well equipped with this new domain-specific PROTO file. +Currently, these additional, domain specific proto files could be simply added to respective repos of HV-VES collector. + +Implementation details +---------------------- + +- Project Reactor is used as a backbone of the internal architecture. +- Netty is used by means of reactor-netty library. +- We are using Kotlin so we can write very concise code with great interoperability with existing Java libraries. +- Types defined in Λrrow library are also used when it improves readability or general cleanness of the code.
\ No newline at end of file diff --git a/docs/sections/services/ves-hv/index.rst b/docs/sections/services/ves-hv/index.rst index 0662660e..483ddbd7 100644 --- a/docs/sections/services/ves-hv/index.rst +++ b/docs/sections/services/ves-hv/index.rst @@ -7,5 +7,29 @@ High Volume VNF Event Streaming (HV-VES) Collector .. Add or remove sections below as appropriate for the platform component. + +**HV-VES collector** has been proposed, based on a need to process high-volumes of data generated frequently by a large +number of NFs. The driving use-case is described and published within presentation during Casablanca Release Developer Forum: +`Google Protocol Buffers versus JSON - 5G RAN use-case - comparison`_. + +.. _`Google Protocol Buffers versus JSON - 5G RAN use-case - comparison`: https://wiki.onap.org/download/attachments/25434845/Casablanca_Dev_Forum_GPB_comparison_20180621.pptx?version=1&modificationDate=1530275050000&api=v2 + +The goal of the collector is to support high volume data. It uses plain TCP connections. +Connections are stream-based (as opposed to request-based) and long running. +Payload is binary-encoded (currently using Google Protocol Buffers). +HV-VES uses direct connection to DMaaP's Kafka. +All these decisions were made in order to support high-volume data with minimal latency. + + High Volume VES Collector overview and functions +------------------------------------------------ + +.. toctree:: + :maxdepth: 1 + ./design.rst + ./configuration.rst + ./delivery.rst + ./installation.rst + `Offered APIs`_ +.. _`Offered APIs`: ../../apis/ves-hv.rst
\ No newline at end of file diff --git a/docs/sections/services/ves-hv/installation.rst b/docs/sections/services/ves-hv/installation.rst new file mode 100644 index 00000000..7dafeaed --- /dev/null +++ b/docs/sections/services/ves-hv/installation.rst @@ -0,0 +1,27 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Installation +============ + +To run HV-VES Collector container you need to specify required parameters: + +- listen-port - the port on which HV-VES will listen internally +- config-url - URL of HV-VES configuration on consul + +There are also optional configuration parameters: + +- health-check-api-port - Health check rest api listen port +- first-request-delay - Delay of first request to consul in seconds +- request-interval - Interval of consul configuration requests in seconds +- ssl-disable - Disable SSL encryption +- key-store - Key store in PKCS12 format path +- key-store-password - Key store password +- trust-store - File with trusted certificate bundle in PKCS12 format path +- trust-store-password - Trust store password +- idle-timeout-sec - Idle timeout for remote hosts. After given time without any data exchange the connection might be closed + + +These parameters can be configured either by passing command line option during `docker run` call or +by specifying environment variables named after command line option name +rewritten using `UPPER_SNAKE_CASE` and prepended with `VESHV_` prefix e.g. `VESHV_LISTEN_PORT`.
\ No newline at end of file |