Extend pm-mapper dev tools of DFC

Issue-ID: DCAEGEN2-3182
Signed-off-by: Tomasz Wrobel <tomasz.wrobel@nokia.com>
Change-Id: I9e228624df44951d2b5f25bad63db5905b9d611a

18 files changed, 199 insertions, 21 deletions

diff --git a/tools/development/Makefile b/tools/development/Makefile
index fc5cf39..90a162e 100644
--- a/tools/development/Makefile
+++ b/tools/development/Makefile
@@ -1,5 +1,5 @@
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,6 +25,12 @@ setup-local:
 run-pm-mapper:
 	@./run-pm-mapper.sh
 
+run-data-file:
+	@./run-datafile.sh
+
+remove-datafile:
+	@docker rm -f datafile-dev
+
 config-dmaap:
 	@./config-dmaap.sh
 
diff --git a/tools/development/README.md b/tools/development/README.md
index 9dcf35b..9dcd2ba 100644
--- a/tools/development/README.md
+++ b/tools/development/README.md
@@ -75,6 +75,34 @@ docker logs -f pmmapper
 docker logs -f mr-simulator
 ```
 
+### DataFileCollector
+```
+docker logs -f datafile-dev
+```
+
+## Development and running  DataFile Collector
+DFC is fetching events from mr-simulator, based on event properties file is downloading and pushing to DataRouter
+
+1. Downloaded file properties: location, fileType, fileName could be set in 'tools/development/message-router/mrserver.js', see function handleVesNotificationRequest(res)
+2. Publish file properties and DFC configuration could be set in 'tools/development/resources/datafile/mount_config.yaml'
+3. Logging level and other DFC/Spring properties could be set in 'tools/development/resources/datafile/spring_application.yaml'
+
+### Running DFC
+To run DFC execute following scripts
+```
+make setup-all
+make run-data-file
+```
+
+Bulk-pm flow:
+DFC fetch message from MR-Simulator, download file from sftp-server, push file to DataRouter.
+DataRouter push file to pm-mapper. 
+Pm-mapper send message to MR-Simulator.
+
+To stop DFC execute: 
+``` 
+make remove-datafile
+```
 
 #####Info:
 Certificate validity - 2023 August
diff --git a/tools/development/clean-environment.sh b/tools/development/clean-environment.sh
index 22fe370..dcb288f 100755
--- a/tools/development/clean-environment.sh
+++ b/tools/development/clean-environment.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,4 +18,6 @@ source ./env/containers_ip
 
 docker stop pmmapper
 docker rm pmmapper
+docker stop datafile-dev
+docker rm datafile-dev
 docker-compose down
diff --git a/tools/development/config-dmaap.sh b/tools/development/config-dmaap.sh
index 7b3a807..07a7e54 100755
--- a/tools/development/config-dmaap.sh
+++ b/tools/development/config-dmaap.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/tools/development/docker-compose.yml b/tools/development/docker-compose.yml
index 3ac9b0a..c8e6014 100644
--- a/tools/development/docker-compose.yml
+++ b/tools/development/docker-compose.yml
@@ -54,6 +54,18 @@ services:
       pmmapper-network:
         ipv4_address: $NODE_IP
 
+  sftp:
+    container_name: sftp-server
+    image: atmoz/sftp
+    ports:
+      - "2222:22"
+    command: admin:admin:1001
+    volumes:
+      - ./files-publisher/files/:/home/admin/upload
+    networks:
+      pmmapper-network:
+        ipv4_address: $SFTP_SERVER_IP
+
   mariadb:
     image: nexus3.onap.org:10001/mariadb:10.2.14
     container_name: mariadb
diff --git a/tools/development/dr-mount/node.properties b/tools/development/dr-mount/node.properties
index dd796e7..1ff80db 100644
--- a/tools/development/dr-mount/node.properties
+++ b/tools/development/dr-mount/node.properties
@@ -3,7 +3,7 @@
 # * org.onap.dmaap
 # * ===========================================================================
 # * Copyright � 2017 AT&T Intellectual Property. All rights reserved.
-# * Copyright (C) 2022 Nokia Intellectual Property
+# * Copyright (C) 2022 Nokia. All rights reserved.
 # * ===========================================================================
 # * Licensed under the Apache License, Version 2.0 (the "License");
 # * you may not use this file except in compliance with the License.
diff --git a/tools/development/dr-mount/provserver.properties b/tools/development/dr-mount/provserver.properties
index 8caa52a..21c47eb 100644
--- a/tools/development/dr-mount/provserver.properties
+++ b/tools/development/dr-mount/provserver.properties
@@ -3,7 +3,7 @@
 # * org.onap.dmaap
 # * ===========================================================================
 # * Copyright � 2017 AT&T Intellectual Property. All rights reserved.
-# * Copyright (C) 2022 Nokia Intellectual Property
+# * Copyright (C) 2022. All rights reserved.
 # * ===========================================================================
 # * Licensed under the Apache License, Version 2.0 (the "License");
 # * you may not use this file except in compliance with the License.
diff --git a/tools/development/env/containers_ip b/tools/development/env/containers_ip
index daaf3f5..fa6e69b 100755
--- a/tools/development/env/containers_ip
+++ b/tools/development/env/containers_ip
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA. All rights reserved.
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,3 +22,5 @@ export MARIADB_IP=172.18.0.6
 export NODE_IP=172.18.0.7
 export PMMAPPER_IP=172.18.0.8
 export FILES_PUBLISHER_IP=172.18.0.9
+export DFC_IP=172.18.0.10
+export SFTP_SERVER_IP=172.18.0.10
diff --git a/tools/development/message-router/mrserver.js b/tools/development/message-router/mrserver.js
index ace7f9b..c5414de 100644
--- a/tools/development/message-router/mrserver.js
+++ b/tools/development/message-router/mrserver.js
@@ -1,6 +1,6 @@
 /*
  * ============LICENSE_START=======================================================
- *  Copyright (C) 2021 Nokia.
+ *  Copyright (C) 2021-2022 Nokia. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,39 @@
  * SPDX-License-Identifier: Apache-2.0
  * ============LICENSE_END=========================================================
  */
+
+function handleVesNotificationRequest(res) {
+  let MEAS_COLLEC = "org.3GPP.32.435#measCollec"
+  let MEAS_DATA = "org.3GPP.28.532#measData"
+  let filetType = MEAS_COLLEC
+  let fileUrl = "sftp://admin:admin@sftp-server:22/upload/A20181002.0000-1000-0015-1000_5G.xml.gz"
+  let filePublishName = "A_28532_measData_test_onap.xml"
+  res.writeHead(200, {'Content-Type': 'application/json'});
+  let result = "[\"{\\\"event\\\":{\\\"commonEventHeader\\\":{\\\"startEpochMicrosec\\\":8745745764578,\\\"eventId\\\":\\\"FileReady_1797490e-10ae-4d48-9ea7-3d7d790b25e1\\\",\\\"timeZoneOffset\\\":\\\"UTC+05.30\\\",\\\"internalHeaderFields\\\":{\\\"collectorTimeStamp\\\":\\\"Thu, 05 12 2022 06:20:00 UTC\\\"},\\\"priority\\\":\\\"Normal\\\",\\\"version\\\":\\\"4.0.1\\\",\\\"reportingEntityName\\\":\\\"NOK6061ZW3\\\",\\\"sequence\\\":0,\\\"domain\\\":\\\"notification\\\",\\\"lastEpochMicrosec\\\":8745745764578,\\\"eventName\\\":\\\"Notification_gnb-Nokia_FileReady\\\",\\\"vesEventListenerVersion\\\":\\\"7.0.1\\\",\\\"sourceName\\\":\\\"NOK6061ZW3\\\"},\\\"notificationFields\\\":{\\\"notificationFieldsVersion\\\":\\\"2.0\\\",\\\"changeType\\\":\\\"FileReady\\\",\\\"changeIdentifier\\\":\\\"PM_MEAS_FILES\\\",\\\"arrayOfNamedHashMap\\\":[{\\\"name\\\":\\\""
+      + filePublishName
+      + "\\\",\\\"hashMap\\\":{\\\"location\\\":\\\""
+      + fileUrl
+      + "\\\",\\\"fileFormatType\\\":\\\""
+      + filetType
+      + "\\\",\\\"fileFormatVersion\\\":\\\"V1\\\",\\\"compression\\\":\\\"gzip\\\"}}]}}}\"]";
+
+  res.end(result);
+}
+
+function handleAllTypeRequest(res, req) {
+  res.writeHead(200, {'Content-Type': 'text/plain'});
+  console.log('Received message');
+  req.on('data', chunk => {
+    console.log(`-----MESSAGE_CONTENT_BEGIN-----\n ${chunk}`);
+    console.log('-----MESSAGE_CONTENT_END-----');
+  });
+}
+
+function isVesNotificationRequest(req) {
+  var vesNotificationTopic = "unauthenticated.VES_NOTIFICATION_OUTPUT"
+  return req.url.includes(vesNotificationTopic);
+}
+
 var httpServer = function () {
   var http = require('http'),
 
@@ -30,12 +63,14 @@ var httpServer = function () {
       },
 
       processHttpRequest = function (req, res) {
-        res.writeHead(200, {'Content-Type': 'text/plain'});
-        console.log('Received message');
-        req.on('data', chunk => {
-          console.log(`-----MESSAGE_CONTENT_BEGIN-----\n ${chunk}`);
-          console.log('-----MESSAGE_CONTENT_END-----');
-        });
+
+        if (isVesNotificationRequest(req)) {
+          console.log("Received VES_NOTIFICATION request ")
+          handleVesNotificationRequest(res);
+          return;
+        }
+
+        handleAllTypeRequest(res, req);
         setTimeout(() => {
           res.end('Published' + ' Successfully.\n');
         }, 100)
diff --git a/tools/development/resources/datafile/mount_config.yaml b/tools/development/resources/datafile/mount_config.yaml
new file mode 100644
index 0000000..284c598
--- /dev/null
+++ b/tools/development/resources/datafile/mount_config.yaml
@@ -0,0 +1,29 @@
+dmaap.certificateConfig.keyCert: /opt/app/datafile/etc/cert/cert.p12
+dmaap.certificateConfig.keyPasswordPath: /opt/app/datafile/etc/cert/p12.pass
+dmaap.certificateConfig.trustedCa: /opt/app/datafile/etc/cert/trust.jks
+dmaap.certificateConfig.trustedCaPasswordPath: /opt/app/datafile/etc/cert/trust.pass
+dmaap.dmaapConsumerConfiguration.consumerGroup: OpenDcae-c12
+dmaap.dmaapConsumerConfiguration.consumerId: C12
+dmaap.dmaapConsumerConfiguration.timeoutMs: -1
+dmaap.security.enableDmaapCertAuth: false
+dmaap.security.keyStorePasswordPath: /opt/app/datafile/etc/cert/jks.pass
+dmaap.security.keyStorePath: /opt/app/datafile/etc/cert/cert.jks
+dmaap.security.trustStorePasswordPath: /opt/app/datafile/etc/cert/trust.pass
+dmaap.security.trustStorePath: /opt/app/datafile/etc/cert/trust.jks
+service_calls: []
+sftp.security.strictHostKeyChecking: true
+streams_publishes:
+  PM_MEAS_FILES:
+    dmaap_info:
+      location: loc00
+      log_url: https://dmaap-dr-prov/feedlog/1
+      password: pmmapper
+      publish_url: http://dmaap-dr-prov:8080/publish/1
+      publisher_id: 1.85gkg
+      username: pmmapper
+    type: data_router
+streams_subscribes:
+  dmaap_subscriber:
+    dmaap_info:
+      topic_url: http://mr-simulator:3904/events/unauthenticated.VES_NOTIFICATION_OUTPUT
+    type: message_router
diff --git a/tools/development/resources/datafile/spring_application.yaml b/tools/development/resources/datafile/spring_application.yaml
new file mode 100644
index 0000000..a3c3036
--- /dev/null
+++ b/tools/development/resources/datafile/spring_application.yaml
@@ -0,0 +1,31 @@
+spring:
+  profiles:
+    active: prod
+management:
+  endpoints:
+    web:
+      exposure:
+        include: "loggers,logfile,health,info,metrics"
+server:
+  port: 8433
+  ssl:
+    key-store-type: PKCS12
+    key-store-password: ericssondfc
+    key-store: classpath:keystore.jks
+    key-password: ericssondfc
+    keyAlias: tomcat-localhost
+logging:
+  level:
+    ROOT: INFO
+    org.springframework: INFO
+    org.springframework.data: INFO
+    org.springframework.web.reactive.function.client.ExchangeFunctions: INFO
+    org.onap.dcaegen2.collectors.datafile: TRACE
+    org.onap.dcaegen2: TRACE
+  file: /var/log/ONAP/application.log
+app:
+  filepath: config/datafile_endpoints_test.json
+
+springdoc:
+  show-actuator: true
+  swagger-ui.disable-swagger-default-url: true
diff --git a/tools/development/restart-pm-mapper.sh b/tools/development/restart-pm-mapper.sh
index 3574dd5..86f8e83 100755
--- a/tools/development/restart-pm-mapper.sh
+++ b/tools/development/restart-pm-mapper.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021 NOKIA
+# Copyright (C) 2021 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/tools/development/run-datafile.sh b/tools/development/run-datafile.sh
new file mode 100755
index 0000000..2b1f070
--- /dev/null
+++ b/tools/development/run-datafile.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+# Copyright (C) 2022 Nokia. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+source ./env/containers_ip
+IMAGE=nexus3.onap.org:10003/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server:latest
+TARGET_CONFIG_PATH=/app-config/application_config.yaml
+TARGET_SPRING_CONFIG=/opt/app/datafile/config/application.yaml
+
+docker run -d -p 8100:8100 -p 8000:8000 \
+  --mount type=bind,source="$PWD/resources/datafile/mount_config.yaml",target="$TARGET_CONFIG_PATH" \
+  --mount type=bind,source="$PWD/resources/datafile/spring_application.yaml",target="$TARGET_SPRING_CONFIG" \
+  -e "JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n" \
+  -e "CONFIG_BINDING_SERVICE=0.0.0.0" \
+  -e "CONFIG_BINDING_SERVICE_SERVICE_PORT=10000" \
+  -e "CBS_CLIENT_CONFIG_PATH=$TARGET_CONFIG_PATH" \
+  --add-host "dmaap-dr-node:$DR_NODE_IP" \
+  --add-host "dmaap-dr-prov:$DR_PROV_IP" \
+  --network=development_pmmapper-network \
+  --name=datafile-dev $IMAGE
diff --git a/tools/development/run-pm-mapper.sh b/tools/development/run-pm-mapper.sh
index 91c7766..15f1e0f 100755
--- a/tools/development/run-pm-mapper.sh
+++ b/tools/development/run-pm-mapper.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,9 +18,10 @@ source ./env/containers_ip
 IMAGE=onap/org.onap.dcaegen2.services.pm-mapper:latest
 
 TARGET_CONFIG_PATH=/app-config/application_config.yaml
-docker run -d -p 8081:8081 \
+docker run -d -p 8081:8081 -p 5005:5005 \
   --mount type=bind,source="$PWD/certs",target="/opt/app/pm-mapper/etc/certs/" \
   --mount type=bind,source="$PWD/resources/mount_config.yaml",target="$TARGET_CONFIG_PATH" \
+  -e "JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,address=*:5005,server=y,suspend=n" \
   -e "CONFIG_BINDING_SERVICE=0.0.0.0" \
   -e "CONFIG_BINDING_SERVICE_SERVICE_PORT=10000" \
   -e "CBS_CLIENT_CONFIG_PATH=$TARGET_CONFIG_PATH" \
diff --git a/tools/development/send-meas-collec.sh b/tools/development/send-meas-collec.sh
index 6f70bd7..6d36332 100755
--- a/tools/development/send-meas-collec.sh
+++ b/tools/development/send-meas-collec.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/tools/development/send-meas-data.sh b/tools/development/send-meas-data.sh
index 7c1a77b..729469a 100755
--- a/tools/development/send-meas-data.sh
+++ b/tools/development/send-meas-data.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/tools/development/setup-local.sh b/tools/development/setup-local.sh
index 94a115d..aa4874f 100755
--- a/tools/development/setup-local.sh
+++ b/tools/development/setup-local.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ source ./env/containers_ip
 
 sed -i 's/datarouter-mariadb/'$MARIADB_IP'/g' $SCRIPT_DIR/dr-mount/provserver.properties
 
-docker-compose -f $SCRIPT_DIR/docker-compose.yml up -d mariadb node files-publisher
+docker-compose -f $SCRIPT_DIR/docker-compose.yml up -d mariadb node files-publisher sftp
 
 echo "Waiting for MariaDB to come up healthy..."
 for i in {1..30}; do
diff --git a/tools/development/subscribe-pm-mapper.sh b/tools/development/subscribe-pm-mapper.sh
index 1b8a532..790bfe6 100755
--- a/tools/development/subscribe-pm-mapper.sh
+++ b/tools/development/subscribe-pm-mapper.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.