VES Collector - Event Ordering

https://jira.onap.org/browse/DCAEGEN2-1483

Change-Id: I28b0e871ce570a3cf4c0d2e08d040b66eb6db3aa
Issue-ID: DCAEGEN2-1483
Signed-off-by: Zlatko Murgoski <zlatko.murgoski@nokia.com>

6 files changed, 233 insertions, 177 deletions

diff --git a/dpo/blueprint/blueprint_ves.yaml b/dpo/blueprint/blueprint_ves.yaml
index 2dbc5a66..43158f1c 100644
--- a/dpo/blueprint/blueprint_ves.yaml
+++ b/dpo/blueprint/blueprint_ves.yaml
@@ -15,169 +15,240 @@
 # ============LICENSE_END=========================================================
 #
 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
-tosca_definitions_version: cloudify_dsl_1_3
-
-description: >
-  This handcrafted blueprint will install the ves collector and provision the needed message router topics. This blueprint can be used to verify that a platform installation is operational and working correctly.
 
+tosca_definitions_version: cloudify_dsl_1_3
 imports:
-- http://www.getcloudify.org/spec/cloudify/3.4/types.yaml
-- https://NEXUS_REPO_HOST:8443/repository/NEXUS_RAW/type_files/docker/2.2.0/node-type.yaml
-- https://NEXUS_REPO_HOST:8443/repository/NEXUS_RAW/type_files/relationship/1.0.0/node-type.yaml
-- http://NEXUS_REPO_HOST:8081/repository/NEXUS_RAW/type_files/dmaap/dmaap_mr.yaml
-
+  - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml
+  - https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.plugins/R4/k8splugin/1.4.5/k8splugin_types.yaml
+  - https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.plugins/R4/dcaepolicyplugin/2.3.0/dcaepolicyplugin_types.yaml
 inputs:
-
-  service_id:
-    description: Unique id used for an instance of this DCAE service. Use deployment id
-    default: 'foobar'
-  location_id:
-    default: 'solutioning-central'
-  docker_host_override:
-    default: 'component_dockerhost'
-
-  topic00_aaf_username:
-  topic00_aaf_password:
-  topic00_location:
-    default: mtc5
-  topic00_client_role:
-    default: com.att.dcae.member
-
-  topic01_aaf_username:
-  topic01_aaf_password:
-  topic01_location:
-    default: mtc5
-  topic01_client_role:
-    default: com.att.dcae.member
-
-  topic02_aaf_username:
-  topic02_aaf_password:
-  topic02_location:
-    default: mtc5
-  topic02_client_role:
-    default: com.att.dcae.member
-
-  topic03_aaf_username:
-  topic03_aaf_password:
-  topic03_location:
-    default: mtc5
-  topic03_client_role:
-    default: com.att.dcae.member
-
+  collector.dmaap.streamid:
+    type: string
+    default: "fault=ves-fault,ves-fault-secondary|syslog=ves-syslog,ves-syslog-secondary|heartbeat=ves-heartbeat,ves-heartbeat-secondary|measurementsForVfScaling=ves-measurement,ves-measurement-secondary|mobileFlow=ves-mobileflow,ves-mobileflow-secondary|other=ves-other,ves-other-secondary|stateChange=ves-statechange,ves-statechange-secondary|thresholdCrossingAlert=ves-thresholdCrossingAlert,ves-thresholdCrossingAlert-secondary|voiceQuality=ves-voicequality,ves-voicequality-secondary|sipSignaling=ves-sipsignaling,ves-sipsignaling-secondary|notification=ves-notification,ves-notification-secondary|pnfRegistration=ves-pnfRegistration,ves-pnfRegistration-secondary"
+  external_port:
+    type: string
+    description: Kubernetes node port on which collector is exposed
+    default: "30235"
+  header.authlist:
+    type: string
+    default: "sample1,$2a$10$pgjaxDzSuc6XVFEeqvxQ5u90DKJnM/u7TJTcinAlFJVaavXMWf/Zi|userid1,$2a$10$61gNubgJJl9lh3nvQvY9X.x4e5ETWJJ7ao7ZhJEvmfJigov26Z6uq|userid2,$2a$10$G52y/3uhuhWAMy.bx9Se8uzWinmbJa.dlm1LW6bYPdPkkywLDPLiy"
+  log_directory:
+    type: string
+    default: "/opt/app/VESCollector/logs"
+  replicas:
+    type: integer
+    description: number of instances
+    default: 1
+  tag_version:
+    type: string
+    default: "nexus.onap.org:10001/onap/org.onap.dcaegen2.collectors.ves.vescollector:1.3"
+  ves_fault_publish_url:
+    type: string
+  ves_fault_secondary_publish_url:
+    type: string
+  ves_heartbeat_publish_url:
+    type: string
+  ves_heartbeat_secondary_publish_url:
+    type: string
+  ves_measurement_publish_url:
+    type: string
+  ves_measurement_secondary_publish_url:
+    type: string
+  ves_mobileflow_publish_url:
+    type: string
+  ves_mobileflow_secondary_publish_url:
+    type: string
+  ves_notification_publish_url:
+    type: string
+  ves_notification_secondary_publish_url:
+    type: string
+  ves_other_publish_url:
+    type: string
+  ves_other_secondary_publish_url:
+    type: string
+  ves_pnfRegistration_publish_url:
+    type: string
+  ves_pnfRegistration_secondary_publish_url:
+    type: string
+  ves_sipsignaling_publish_url:
+    type: string
+  ves_sipsignaling_secondary_publish_url:
+    type: string
+  ves_statechange_publish_url:
+    type: string
+  ves_statechange_secondary_publish_url:
+    type: string
+  ves_syslog_publish_url:
+    type: string
+  ves_syslog_secondary_publish_url:
+    type: string
+  ves_thresholdCrossingAlert_publish_url:
+    type: string
+  ves_thresholdCrossingAlert_secondary_publish_url:
+    type: string
+  ves_voicequality_publish_url:
+    type: string
+  ves_voicequality_secondary_publish_url:
+    type: string
 node_templates:
-
-  topic00:
-    type: dcae.nodes.Topic
-    properties:
-      topic_name: sec-fault-unsecure
-
-  topic01:
-    type: dcae.nodes.Topic
-    properties:
-      topic_name: sec-measurement
-
-  topic02:
-    type: dcae.nodes.Topic
-    properties:
-      topic_name: sec-measurement-unsecure
-
-  topic03:
-    type: dcae.nodes.Topic
-    properties:
-      topic_name: sec-fault
-
-  component00:
-    type: dcae.nodes.DockerContainerForComponentsUsingDmaap
-    properties:
-      service_component_type:
-        'dcae-controller-ves-collector'
-      service_id:
-        { get_input: service_id }
-      location_id:
-        { get_input: location_id }
-      application_config:
-        collector.keystore.passwordfile: "/opt/app/dcae-certificate/.password"
-        collector.service.secure.port: -1
-        tomcat.maxthreads: '200'
-        collector.keystore.file.location: "/opt/app/dcae-certificate/keystore.jks"
-        auth.method: "noAuth"
-        collector.service.port: 8080
-        streams_publishes:
-          sec_fault_unsecure:
-            aaf_password: { get_input: topic00_aaf_password }
-            dmaap_info: "<<topic00>>"
-            type: message_router
-            aaf_username: { get_input: topic00_aaf_username }
-          sec_measurement:
-            aaf_password: { get_input: topic01_aaf_password }
-            aaf_username: { get_input: topic01_aaf_username }
-            type: message_router
-            dmaap_info: "<<topic01>>"
-          sec_measurement_unsecure:
-            aaf_password: { get_input: topic02_aaf_password }
-            aaf_username: { get_input: topic02_aaf_username }
-            dmaap_info: "<<topic02>>"
-            type: message_router
-          sec_fault:
-            aaf_password: { get_input: topic03_aaf_password }
-            aaf_username: { get_input: topic03_aaf_username }
-            dmaap_info: "<<topic03>>"
-            type: message_router
-        services_calls: {}
-        collector.schema.checkflag: 1
-        collector.dmaap.streamid: fault=sec_fault,roadm-sec-to-hp|syslog=sec_syslog|heartbeat=sec_heartbeat|measurementsForVfScaling=sec_measurement|mobileFlow=sec_mobileflow|other=sec_other|stateChange=sec_statechange|thresholdCrossingAlert=sec_thresholdCrossingAlert
-        header.authlist: userid1,base64encodepwd1|userid2,base64encodepwd2
-        streams_subscribes: {}
-        collector.inputQueue.maxPending: 8096
-        collector.schema.file: "./etc/CommonEventFormat_27.2.json"
-      image:
-        NEXUS_REPO_HOST:18443/dcae-dev-raw/dcae-controller-ves-collector:1.1.3
-      docker_config:
-        healthcheck:
-          type: "http"
-          interval: "15s"
-          timeout: "1s"
-          endpoint: "/"
-      streams_publishes:
-      - name: topic00
-        location: { get_input: topic00_location }
-        client_role: { get_input: topic00_client_role }
-        type: message_router
-      - name: topic01
-        location: { get_input: topic01_location }
-        client_role: { get_input: topic01_client_role }
-        type: message_router
-      - name: topic02
-        location: { get_input: topic02_location }
-        client_role: { get_input: topic02_client_role }
-        type: message_router
-      - name: topic03
-        location: { get_input: topic03_location }
-        client_role: { get_input: topic03_client_role }
-        type: message_router
-      streams_subscribes: []
-    relationships:
-    - type: dcae.relationships.component_contained_in
-      target: docker_host
-    - type: dcae.relationships.publish_events
-      target: topic00
-    - type: dcae.relationships.publish_events
-      target: topic01
-    - type: dcae.relationships.publish_events
-      target: topic02
-    - type: dcae.relationships.publish_events
-      target: topic03
+  dcae-ves-collector:
+    type: dcae.nodes.ContainerizedPlatformComponent
     interfaces:
       cloudify.interfaces.lifecycle:
-        stop:
+        start:
           inputs:
-            cleanup_image:
-              True
-
-  docker_host:
-    type: dcae.nodes.SelectedDockerHost
+            ports:
+              - concat: ["8443:", {get_input: external_port }]
     properties:
-      location_id:
-        { get_input: location_id }
-      docker_host_override:
-        { get_input: docker_host_override }
+      application_config:
+        service_calls: []
+        stream_publishes:
+          ves-fault:
+            dmaap_info:
+              topic_url:
+                get_input: ves_fault_publish_url
+            type: message router
+          ves-fault-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_fault_secondary_publish_url
+            type: message router
+          ves-heartbeat:
+            dmaap_info:
+              topic_url:
+                get_input: ves_heartbeat_publish_url
+            type: message router
+          ves-heartbeat-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_heartbeat_secondary_publish_url
+            type: message router
+          ves-measurement:
+            dmaap_info:
+              topic_url:
+                get_input: ves_measurement_publish_url
+            type: message router
+          ves-measurement-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_measurement_secondary_publish_url
+            type: message router
+          ves-mobileflow:
+            dmaap_info:
+              topic_url:
+                get_input: ves_mobileflow_publish_url
+            type: message router
+          ves-mobileflow-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_mobileflow_secondary_publish_url
+            type: message router
+          ves-notification:
+            dmaap_info:
+              topic_url:
+                get_input: ves_notification_publish_url
+            type: message router
+          ves-notification-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_notification_secondary_publish_url
+            type: message router
+          ves-other:
+            dmaap_info:
+              topic_url:
+                get_input: ves_other_publish_url
+            type: message router
+          ves-other-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_other_secondary_publish_url
+            type: message router
+          ves-pnfRegistration:
+            dmaap_info:
+              topic_url:
+                get_input: ves_pnfRegistration_publish_url
+            type: message router
+          ves-pnfRegistration-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_pnfRegistration_secondary_publish_url
+            type: message router
+          ves-sipsignaling:
+            dmaap_info:
+              topic_url:
+                get_input: ves_sipsignaling_publish_url
+            type: message router
+          ves-sipsignaling-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_sipsignaling_secondary_publish_url
+            type: message router
+          ves-statechange:
+            dmaap_info:
+              topic_url:
+                get_input: ves_statechange_publish_url
+            type: message router
+          ves-statechange-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_statechange_secondary_publish_url
+            type: message router
+          ves-syslog:
+            dmaap_info:
+              topic_url:
+                get_input: ves_syslog_publish_url
+            type: message router
+          ves-syslog-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_syslog_secondary_publish_url
+            type: message router
+          ves-thresholdCrossingAlert:
+            dmaap_info:
+              topic_url:
+                get_input: ves_thresholdCrossingAlert_publish_url
+            type: message router
+          ves-thresholdCrossingAlert-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_thresholdCrossingAlert_secondary_publish_url
+            type: message router
+          ves-voicequality:
+            dmaap_info:
+              topic_url:
+                get_input: ves_voicequality_publish_url
+            type: message router
+          ves-voicequality-secondary:
+            dmaap_info:
+              topic_url:
+                get_input: ves_voicequality_secondary_publish_url
+            type: message router
+        stream_subcribes: {}
+        auth.method: noAuth
+        collector.dmaap.streamid:
+          get_input: collector.dmaap.streamid
+        collector.keystore.file.location: /opt/app/dcae-certificate/keystore.jks
+        collector.keystore.passwordfile: /opt/app/dcae-certificate/.password
+        collector.schema.checkflag: 1
+        collector.schema.file: {"v1":"./etc/CommonEventFormat_27.2.json","v2":"./etc/CommonEventFormat_27.2.json","v3":"./etc/CommonEventFormat_27.2.json","v4":"./etc/CommonEventFormat_27.2.json","v5":"./etc/CommonEventFormat_28.4.1.json","v7":"./etc/CommonEventFormat_30.json"}
+        collector.service.port: 8080
+        collector.service.secure.port: 8443
+        collector.truststore.file.location: /opt/app/dcae-certificate/truststore.jks
+        collector.truststore.passwordfile: /opt/app/dcae-certificate/.trustpassword
+        event.transform.flag: 1
+        header.authlist:
+          get_input: header.authlist
+        tomcat.maxthreads: 200
+      docker_config:
+        interval: 15s
+        timeout: 1s
+        type: https
+        endpoint: /healthcheck
+      image:
+        get_input: tag_version
+      log_info:
+        get_input: log_directory
+      dns_name: dcae-ves-collector
+      replicas:
+        get_input: replicas
+      name: dcae-ves-collector
diff --git a/dpo/data-formats/ConsulConfig.json b/dpo/data-formats/ConsulConfig.json
index ea65522b..89348bfa 100644
--- a/dpo/data-formats/ConsulConfig.json
+++ b/dpo/data-formats/ConsulConfig.json
@@ -6,7 +6,6 @@
   "collector.service.port": "8080",
   "collector.schema.file": "{\"v1\":\"./etc/CommonEventFormat_27.2.json\",\"v2\":\"./etc/CommonEventFormat_27.2.json\",\"v3\":\"./etc/CommonEventFormat_27.2.json\",\"v4\":\"./etc/CommonEventFormat_27.2.json\",\"v5\":\"./etc/CommonEventFormat_28.4.1.json\",\"v7\":\"./etc/CommonEventFormat_30.0.1.json\"}",
   "collector.keystore.passwordfile": "/opt/app/VESCollector/etc/passwordfile",
-  "collector.inputQueue.maxPending": "8096",
   "streams_publishes": {
     "ves-measurement": {
       "type": "message_router",
diff --git a/dpo/spec/vescollector-componentspec.json b/dpo/spec/vescollector-componentspec.json
index 4e2eb970..1b47268c 100644
--- a/dpo/spec/vescollector-componentspec.json
+++ b/dpo/spec/vescollector-componentspec.json
@@ -1,6 +1,6 @@
 {
   "self": {
-    "version": "1.3.0",
+    "version": "1.5.0",
     "name": "dcae-ves-collector",
     "description": "Collector for receiving VES events through restful interface",
     "component_type": "docker"
@@ -281,14 +281,6 @@
       "designer_editable": false
     },
     {
-      "name": "collector.inputQueue.maxPending",
-      "value": 8096,
-      "description": "Maximum queue limit across domains collector will queue before event is published",
-      "sourced_at_deployment": false,
-      "policy_editable": false,
-      "designer_editable": false
-    },
-    {
       "name": "collector.dmaap.streamid",
       "value": "fault=ves-fault,ves-fault-secondary|syslog=ves-syslog,ves-syslog-secondary|heartbeat=ves-heartbeat,ves-heartbeat-secondary|measurementsForVfScaling=ves-measurement,ves-measurement-secondary|mobileFlow=ves-mobileflow,ves-mobileflow-secondary|other=ves-other,ves-other-secondary|stateChange=ves-statechange,ves-statechange-secondary|thresholdCrossingAlert=ves-thresholdCrossingAlert,ves-thresholdCrossingAlert-secondary|voiceQuality=ves-voicequality,ves-voicequality-secondary|sipSignaling=ves-sipsignaling,ves-sipsignaling-secondary|notification=ves-notification,ves-notification-secondary|pnfRegistration=ves-pnfRegistration,ves-pnfRegistration-secondary",
       "description": "domain-to-streamid mapping used by VESCollector to distributes events based on domain. Both primary and secondary config_key are included for resilency (multiple streamid can be included commma separated). The streamids MUST match to topic config_keys. For single site without resiliency deployment - configkeys with -secondary suffix can be removed",
@@ -299,7 +291,7 @@
     {
       "name": "auth.method",
       "value": "noAuth",
-      "description": "Basic Authentication flag; when enabled only secure port will be supported.",
+      "description": "Property to manage application mode, possible configurations: noAuth - default option - no security (http) , certOnly - auth by certificate (https), basicAuth - auth by basic auth username and password (https),certBasicAuth - auth by certificate and basic auth username / password (https),",
       "sourced_at_deployment": false,
       "policy_editable": false,
       "designer_editable": false
@@ -379,13 +371,14 @@
       }
     ],
     "ports": [
+      "8080:8080",
       "8443:8443"
     ]
   },
   "artifacts": [
     {
       "type": "docker image",
-      "uri": "nexus.onap.org:10001/onap/org.onap.dcaegen2.collectors.ves.vescollector:1.3"
+      "uri": "nexus.onap.org:10001/onap/org.onap.dcaegen2.collectors.ves.vescollector:latest"
     }
   ]
 }
diff --git a/dpo/tosca_model/schema.yaml b/dpo/tosca_model/schema.yaml
index 6c1b2757..c44a4f7e 100644
--- a/dpo/tosca_model/schema.yaml
+++ b/dpo/tosca_model/schema.yaml
@@ -195,8 +195,6 @@ node_types:
     properties:
       docker_collector.dmaap.streamid:
         type: string
-      docker_collector.inputQueue.maxPending:
-        type: string
       docker_collector.keystore.file.location:
         type: string
       docker_collector.keystore.passwordfile:
diff --git a/dpo/tosca_model/template.yaml b/dpo/tosca_model/template.yaml
index 73b4ad38..2f132e12 100644
--- a/dpo/tosca_model/template.yaml
+++ b/dpo/tosca_model/template.yaml
@@ -26,7 +26,6 @@ topology_template:
       type: dcae.nodes.dockerApp.ves
       properties:
         docker_collector.dmaap.streamid: fault=sec_fault,roadm-sec-to-hp|syslog=sec_syslog|heartbeat=sec_heartbeat|measurementsForVfScaling=sec_measurement|mobileFlow=sec_mobileflow|other=sec_other|stateChange=sec_statechange|thresholdCrossingAlert=sec_thresholdCrossingAlert
-        docker_collector.inputQueue.maxPending: '8096'
         docker_collector.keystore.file.location: /opt/app/dcae-certificate/keystore.jks
         docker_collector.keystore.passwordfile: /opt/app/dcae-certificate/.password
         docker_collector.schema.checkflag: '1'
diff --git a/dpo/tosca_model/translate.yaml b/dpo/tosca_model/translate.yaml
index 284f34bf..f6b7a23e 100644
--- a/dpo/tosca_model/translate.yaml
+++ b/dpo/tosca_model/translate.yaml
@@ -24,8 +24,6 @@ topology_template:
   inputs:
     docker_collector.dmaap.streamid:
       type: string
-    docker_collector.inputQueue.maxPending:
-      type: string
     docker_collector.keystore.file.location:
       type: string
     docker_collector.keystore.passwordfile:
@@ -105,8 +103,6 @@ topology_template:
         application_config:
           collector.dmaap.streamid:
             get_input: docker_collector.dmaap.streamid
-          collector.inputQueue.maxPending:
-            get_input: docker_collector.inputQueue.maxPending
           collector.keystore.file.location:
             get_input: docker_collector.keystore.file.location
           collector.keystore.passwordfile: