Publish HV-VES documentation including perf3gpp

Change-Id: I8d3e926ce6bc99f981d52eb72fcbe92e8ce67cde
Signed-off-by: kjaniak <kornel.janiak@nokia.com>
Issue-ID: DCAEGEN2-859

3 files changed, 185 insertions, 14 deletions

diff --git a/docs/sections/apis/ves-hv/MeasDataCollection.proto b/docs/sections/apis/ves-hv/MeasDataCollection.proto
new file mode 100644
index 00000000..978cb28a
--- /dev/null
+++ b/docs/sections/apis/ves-hv/MeasDataCollection.proto
@@ -0,0 +1,78 @@
+/*
+ * ============LICENSE_START=======================================================
+ * dcaegen2-collectors-veshv
+ * ================================================================================
+ * Copyright (C) 2018 NOKIA
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+syntax = "proto3";
+package org.onap.ves;
+
+// Definition for RTPM, structure aligned with 3GPP PM format optimized for RTPM delivery pre-standard TS 28.550 V2.0.0 (2018-09).
+// Some field details are taken from 3GPP TS 32.436 V15.0.0 (2018-06) ASN.1 file.
+// Note (2018-09): work is in progress for 3GPP TS 28.550. Changes will be made, if needed, to align with final version.
+// Differences/additions to 3GPP TS 28.550 are marked with "%%".
+
+message MeasDataCollection                  // top-level message 
+{
+    // %% Combined messageFileHeader, measData (single instance), messageFileFooter (not needed: timestamp = collectionBeginTime + granularityPeriod).
+    string formatVersion = 1;               // required, current value "28.550 2.0"
+    uint32 granularityPeriod = 2;           // required, duration in seconds, %% moved from MeasInfo (single reporting period per event)
+    string measuredEntityUserName = 3;      // network function user definable name ("userLabel") defined for the measured entity in 3GPP TS 28.622
+    string measuredEntityDn = 4;            // DN as per 3GPP TS 32.300
+    string measuredEntitySoftwareVersion = 5;
+    repeated string measObjInstIdList = 6; // %%: optional, monitored object LDNs as per 3GPP TS 32.300 and 3GPP TS 32.432
+    repeated MeasInfo measInfo = 7; 
+}
+
+message MeasInfo
+{
+    oneof MeasInfoId {                      // measurement group identifier
+        uint32 iMeasInfoId = 1;             // identifier as integer (%%: more compact)
+        string measInfoId = 2;              // identifier as string (more generic)
+    }
+
+    oneof MeasTypes {                       // measurement identifiers associated with the measurement results
+        IMeasTypes iMeasTypes = 3;          // identifiers as integers (%%: more compact)
+        SMeasTypes measTypes = 4;           // identifiers as strings (more generic)
+    }
+    // Needed only because GPB does not support repeated fields directly inside 'oneof'
+    message IMeasTypes { repeated uint32 iMeasType = 1; }
+    message SMeasTypes { repeated string measType = 1; }
+
+    string jobId = 5;
+    repeated MeasValue measValues = 6;      // performance measurements grouped by measurement object
+}
+
+message MeasValue
+{
+    oneof MeasObjInstId {                   // monitored object LDN as per 3GPP TS 32.300 and 3GPP TS 32.432
+        string measObjInstId = 1;           // LDN itself
+        uint32 measObjInstIdListIdx = 2;    // %%: index into measObjInstIdList (zero-based)
+    }
+    repeated MeasResult measResults = 3;
+    bool suspectFlag = 4;
+    map<string, string> measObjAddlFlds = 5; // %%: optional per-object data (name/value HashMap)
+}
+
+message MeasResult
+{
+    uint32 p = 1;                           // Index in the MeasTypes array (zero-based), needed only if measResults has fewer elements than MeasTypes
+    oneof xValue {
+        sint64 iValue = 2;
+        double rValue = 3;
+        bool isNull = 4;
+    }
+}
diff --git a/docs/sections/apis/ves-hv/Perf3gppFields.proto b/docs/sections/apis/ves-hv/Perf3gppFields.proto
new file mode 100644
index 00000000..453d1062
--- /dev/null
+++ b/docs/sections/apis/ves-hv/Perf3gppFields.proto
@@ -0,0 +1,37 @@
+/*
+ * ============LICENSE_START=======================================================
+ * dcaegen2-collectors-veshv
+ * ================================================================================
+ * Copyright (C) 2018 NOKIA
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+syntax = "proto3";
+package org.onap.ves;
+import "MeasDataCollection.proto";              // for 3GPP PM format
+
+message Perf3gppFields
+{
+    string perf3gppFieldsVersion = 1;           // required, current value "1.0"
+    MeasDataCollection measDataCollection = 2;  // required
+    // Based on 3GPP TS 28.550
+    // Logical mapping from 3GPP to ONAP header fields:
+    // 3GPP MeasFileHeader     ONAP/VES CommonEventHeader
+    // senderName              sourceName
+    // senderType              nfNamingCode + nfcNamingCode
+    // vendorName              nfVendorName
+    // collectionBeginTime     startEpochMicrosec
+    // timestamp               lastEpochMicrosec
+    map<string, string> eventAddlFlds = 3;      // optional per-event data (name/value HashMap)
+}
diff --git a/docs/sections/apis/ves-hv/index.rst b/docs/sections/apis/ves-hv/index.rst
index 799f92d4..b707d9fd 100644
--- a/docs/sections/apis/ves-hv/index.rst
+++ b/docs/sections/apis/ves-hv/index.rst
@@ -8,7 +8,7 @@ HV-VES (High Volume VES)
 :Date: 2018-10-05
 
 .. contents::
-    :depth: 3
+    :depth: 4
 ..
 
 Overview
@@ -18,42 +18,98 @@ Component description can be found under `HV-VES Collector`_.
 
 .. _HV-VES Collector: ../../services/ves-hv/index.html
 
+.. _tcp_endpoint:
 
 TCP Endpoint
 ============
 
 HV-VES is exposed as NodePort service on Kubernetes cluster on port 30222/tcp.
-It uses plain TCP connections tunneled in SSL/TLS or can be run in insecure manner without data encryption on the socket.
+It uses plain, insecure TCP connection without socket data encryption. In Casablanca release, there is an experimental option to enable SSL/TLS (see :ref:`authorization`).
 Without TLS client authentication/authorization is not possible.
-Connections are stream-based (as opposed to request-based) and long running.
+Connections are stream-based (as opposed to request-based) and long-running.
 
 Communication is wrapped with thin Wire Transfer Protocol, which mainly provides delimitation.
 
 .. literalinclude:: WTP.asn
     :language: asn
 
-Payload is binary-encoded, currently using Google Protocol Buffers representation of the VES Common Header.
+Payload is binary-encoded, using Google Protocol Buffers (GPB) representation of the VES Event.
 
 .. literalinclude:: VesEvent.proto
     :language: protobuf
 
-The PROTO file, which contains the VES CommonHeader, comes with a binary-type Payload parameter, where domain-specific
-data shall be placed. Domain-specific data are encoded as well with GPB, and they do require a domain-specific
-PROTO file to decode the data.
+HV-VES makes routing decisions based mostly on the content of the **Domain** parameter in the VES Common Event Header.
 
-HV-VES makes routing decisions based mostly on the content of the **Domain** parameter in VES Common Header.
+The PROTO file, which contains the VES CommonEventHeader, comes with a binary-type Payload (eventFields) parameter, where domain-specific
+data should be placed. Domain-specific data are encoded as well with GPB. A domain-specific PROTO file is required to decode the data.
 
+Domain **perf3gpp**
+===================
 
-Healthcheck
-===========
+The purpose of the **perf3gpp** domain is to deliver performance measurements from a network function (NF) to ONAP in 3GPP format.
+The first application of this domain is frequent periodic delivery of structured RAN PM data commonly referred to as Real Time PM (RTPM).
+The equipment sends an event right after collecting the PM data for a granularity period.
 
-Inside HV-VES docker container runs small http service for healthcheck - exact port for this service can be configured
-at deployment using `--health-check-api-port` command line option.
+The characteristics of each event in the **perf3gpp** domain:
 
-This service exposes single endpoint **GET /health/ready** which returns **HTTP 200 OK** in case HV-VES is healthy
-and ready for connections. Otherwise it returns **HTTP 503 Service Unavailable** with short reason of unhealthiness.
+- Single measured entity, for example, BTS
+- Single granularity period (collection *begin time* and *duration*)
+- Optional top-level grouping in one or more PM groups
+- Grouping in one or more measured objects, for example, cells
+- One or more reported PM values for each measured object
 
+Due to the single granularity period per event, single equipment supporting multiple concurrent granularity periods might send more than one event at a given reporting time.
 
+The **perf3gpp** domain is based on 3GPP specifications:
 
+- `3GPP TS 28.550 <http://www.3gpp.org/ftp//Specs/archive/28_series/28.550/>`_
+- `3GPP TS 32.431 <http://www.3gpp.org/ftp//Specs/archive/32_series/32.431/>`_
+- `3GPP TS 32.436 <http://www.3gpp.org/ftp//Specs/archive/32_series/32.436/>`_
 
+The event structure is changed in comparison to the one presented in 3GPP technical specifications. The 3GPP structure is enhanced to provide support for efficient transport.
 
+Definitions for the **perf3gpp** domain are stored in Perf3gppFields.proto and MeasDataCollection.proto, listed below:
+
+.. literalinclude:: Perf3gppFields.proto
+    :language: protobuf
+
+.. literalinclude:: MeasDataCollection.proto
+    :language: protobuf
+
+
+API towards DMaaP
+=================
+
+HV-VES Collector forwards incoming messages to a particular DMaaP Kafka topic based on the domain and configuration. Every Kafka record is comprised of a key and a value. In case of HV-VES:
+
+- **Kafka record key** is a GPB-encoded `CommonEventHeader`.
+- **Kafka record value** is a GPB-encoded `VesEvent` (`CommonEventHeader` and domain-specific `eventFields`).
+
+In both cases raw bytes might be extracted using ``org.apache.kafka.common.serialization.ByteArrayDeserializer``. The resulting bytes might be further passed to ``parseFrom`` methods included in classes generated from GPB definitions. WTP is not used here - it is only used in communication between PNF/VNF and the collector.
+
+
+.. _hv_ves_behaviors:
+
+HV-VES behaviors
+================
+
+Connections with HV-VES are stream-based (as opposed to request-based) and long-running. In case of interrupted or closed connection, the collector logs such event but does not try to reconnect to client.
+Communication is wrapped with thin Wire Transfer Protocol, which mainly provides delimitation. Wire Transfer Protocol Frame:
+
+- is dropped after decoding and validating and only GPB is used in further processing.
+- has to start with **MARKER_BYTE**, as defined in protocol specification (see :ref:`tcp_endpoint`). If **MARKER_BYTE** is invalid, HV-VES disconnects from client.
+
+HV-VES decodes only CommonEventHeader from GPB message received. Collector does not decode or validate the rest of the GPB message and publishes it to Kafka topic intact.
+Kafka topic for publishing events with specific domain can be configured through Consul service as described in :ref:`run_time_configuration`.
+In case of Kafka service unavailability, the collector drops currently handled messages and disconnects the client.
+
+Messages handling:
+
+- HV-VES Collector skips messages with unknown/invalid GPB CommonEventHeader format.
+- HV-VES Collector skips messages with unsupported domain. Domain is unsupported if there is no route for it in configuration (see :ref:`run_time_configuration`).
+- HV-VES Collector skips messages with invalid Wire Frame format, unsupported WTP version or inconsistencies of data in the frame (other than invalid **MARKER_BYTE**).
+- HV-VES Collector interrupts connection when it encounters a message with too big GPB payload. Default maximum size and ways to change it are described in :ref:`deployment`.
+
+.. note:: xNF (VNF/PNF) can split  messages bigger than 1 MiB and set `sequence` field in CommonEventHeader accordingly. It is advised to use smaller than 1 MiB messages for GPBs encoding/decoding efficiency.
+
+- Skipped messages (for any of the above reasons) might not leave any trace in HV-VES logs.