summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTomasz Wrobel <tomasz.wrobel@nokia.com>2022-05-12 13:28:07 +0200
committerTomasz Wrobel <tomasz.wrobel@nokia.com>2022-05-31 12:15:26 +0200
commitef8a1b1f910492af8d46dd0e97a044a7e7389c8c (patch)
tree4801854783db78ff86f974c393ed8506a4c89898
parent965cf0ea3cd08fdad7ec28a5078032dada455124 (diff)
Extend pm-mapper dev tools of DFC
Issue-ID: DCAEGEN2-3182 Signed-off-by: Tomasz Wrobel <tomasz.wrobel@nokia.com> Change-Id: I9e228624df44951d2b5f25bad63db5905b9d611a
-rw-r--r--Changelog.md1
-rw-r--r--tools/development/Makefile8
-rw-r--r--tools/development/README.md28
-rwxr-xr-xtools/development/clean-environment.sh4
-rwxr-xr-xtools/development/config-dmaap.sh2
-rw-r--r--tools/development/docker-compose.yml12
-rw-r--r--tools/development/dr-mount/node.properties2
-rw-r--r--tools/development/dr-mount/provserver.properties2
-rwxr-xr-xtools/development/env/containers_ip4
-rw-r--r--tools/development/message-router/mrserver.js49
-rw-r--r--tools/development/resources/datafile/mount_config.yaml29
-rw-r--r--tools/development/resources/datafile/spring_application.yaml31
-rwxr-xr-xtools/development/restart-pm-mapper.sh2
-rwxr-xr-xtools/development/run-datafile.sh32
-rwxr-xr-xtools/development/run-pm-mapper.sh5
-rwxr-xr-xtools/development/send-meas-collec.sh2
-rwxr-xr-xtools/development/send-meas-data.sh2
-rwxr-xr-xtools/development/setup-local.sh4
-rwxr-xr-xtools/development/subscribe-pm-mapper.sh2
19 files changed, 200 insertions, 21 deletions
diff --git a/Changelog.md b/Changelog.md
index abefe57..c38af1f 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
## [1.9.0] - 2022/05/13
### Changed
- [DCAEGEN2-3037] Disable TLS in DataRouter (CSIT)
+- [DCAEGEN2-3182] Extend development tools
## [1.8.0] - 2022/01/24
diff --git a/tools/development/Makefile b/tools/development/Makefile
index fc5cf39..90a162e 100644
--- a/tools/development/Makefile
+++ b/tools/development/Makefile
@@ -1,5 +1,5 @@
# ============LICENSE_START=======================================================
-# Copyright (C) 2021 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -25,6 +25,12 @@ setup-local:
run-pm-mapper:
@./run-pm-mapper.sh
+run-data-file:
+ @./run-datafile.sh
+
+remove-datafile:
+ @docker rm -f datafile-dev
+
config-dmaap:
@./config-dmaap.sh
diff --git a/tools/development/README.md b/tools/development/README.md
index 9dcf35b..9dcd2ba 100644
--- a/tools/development/README.md
+++ b/tools/development/README.md
@@ -75,6 +75,34 @@ docker logs -f pmmapper
docker logs -f mr-simulator
```
+### DataFileCollector
+```
+docker logs -f datafile-dev
+```
+
+## Development and running DataFile Collector
+DFC is fetching events from mr-simulator, based on event properties file is downloading and pushing to DataRouter
+
+1. Downloaded file properties: location, fileType, fileName could be set in 'tools/development/message-router/mrserver.js', see function handleVesNotificationRequest(res)
+2. Publish file properties and DFC configuration could be set in 'tools/development/resources/datafile/mount_config.yaml'
+3. Logging level and other DFC/Spring properties could be set in 'tools/development/resources/datafile/spring_application.yaml'
+
+### Running DFC
+To run DFC execute following scripts
+```
+make setup-all
+make run-data-file
+```
+
+Bulk-pm flow:
+DFC fetch message from MR-Simulator, download file from sftp-server, push file to DataRouter.
+DataRouter push file to pm-mapper.
+Pm-mapper send message to MR-Simulator.
+
+To stop DFC execute:
+```
+make remove-datafile
+```
#####Info:
Certificate validity - 2023 August
diff --git a/tools/development/clean-environment.sh b/tools/development/clean-environment.sh
index 22fe370..dcb288f 100755
--- a/tools/development/clean-environment.sh
+++ b/tools/development/clean-environment.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,4 +18,6 @@ source ./env/containers_ip
docker stop pmmapper
docker rm pmmapper
+docker stop datafile-dev
+docker rm datafile-dev
docker-compose down
diff --git a/tools/development/config-dmaap.sh b/tools/development/config-dmaap.sh
index 7b3a807..07a7e54 100755
--- a/tools/development/config-dmaap.sh
+++ b/tools/development/config-dmaap.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/tools/development/docker-compose.yml b/tools/development/docker-compose.yml
index 3ac9b0a..c8e6014 100644
--- a/tools/development/docker-compose.yml
+++ b/tools/development/docker-compose.yml
@@ -54,6 +54,18 @@ services:
pmmapper-network:
ipv4_address: $NODE_IP
+ sftp:
+ container_name: sftp-server
+ image: atmoz/sftp
+ ports:
+ - "2222:22"
+ command: admin:admin:1001
+ volumes:
+ - ./files-publisher/files/:/home/admin/upload
+ networks:
+ pmmapper-network:
+ ipv4_address: $SFTP_SERVER_IP
+
mariadb:
image: nexus3.onap.org:10001/mariadb:10.2.14
container_name: mariadb
diff --git a/tools/development/dr-mount/node.properties b/tools/development/dr-mount/node.properties
index dd796e7..1ff80db 100644
--- a/tools/development/dr-mount/node.properties
+++ b/tools/development/dr-mount/node.properties
@@ -3,7 +3,7 @@
# * org.onap.dmaap
# * ===========================================================================
# * Copyright � 2017 AT&T Intellectual Property. All rights reserved.
-# * Copyright (C) 2022 Nokia Intellectual Property
+# * Copyright (C) 2022 Nokia. All rights reserved.
# * ===========================================================================
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
diff --git a/tools/development/dr-mount/provserver.properties b/tools/development/dr-mount/provserver.properties
index 8caa52a..21c47eb 100644
--- a/tools/development/dr-mount/provserver.properties
+++ b/tools/development/dr-mount/provserver.properties
@@ -3,7 +3,7 @@
# * org.onap.dmaap
# * ===========================================================================
# * Copyright � 2017 AT&T Intellectual Property. All rights reserved.
-# * Copyright (C) 2022 Nokia Intellectual Property
+# * Copyright (C) 2022. All rights reserved.
# * ===========================================================================
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
diff --git a/tools/development/env/containers_ip b/tools/development/env/containers_ip
index daaf3f5..fa6e69b 100755
--- a/tools/development/env/containers_ip
+++ b/tools/development/env/containers_ip
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA. All rights reserved.
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -22,3 +22,5 @@ export MARIADB_IP=172.18.0.6
export NODE_IP=172.18.0.7
export PMMAPPER_IP=172.18.0.8
export FILES_PUBLISHER_IP=172.18.0.9
+export DFC_IP=172.18.0.10
+export SFTP_SERVER_IP=172.18.0.10
diff --git a/tools/development/message-router/mrserver.js b/tools/development/message-router/mrserver.js
index ace7f9b..c5414de 100644
--- a/tools/development/message-router/mrserver.js
+++ b/tools/development/message-router/mrserver.js
@@ -1,6 +1,6 @@
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2021 Nokia.
+ * Copyright (C) 2021-2022 Nokia. All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,6 +17,39 @@
* SPDX-License-Identifier: Apache-2.0
* ============LICENSE_END=========================================================
*/
+
+function handleVesNotificationRequest(res) {
+ let MEAS_COLLEC = "org.3GPP.32.435#measCollec"
+ let MEAS_DATA = "org.3GPP.28.532#measData"
+ let filetType = MEAS_COLLEC
+ let fileUrl = "sftp://admin:admin@sftp-server:22/upload/A20181002.0000-1000-0015-1000_5G.xml.gz"
+ let filePublishName = "A_28532_measData_test_onap.xml"
+ res.writeHead(200, {'Content-Type': 'application/json'});
+ let result = "[\"{\\\"event\\\":{\\\"commonEventHeader\\\":{\\\"startEpochMicrosec\\\":8745745764578,\\\"eventId\\\":\\\"FileReady_1797490e-10ae-4d48-9ea7-3d7d790b25e1\\\",\\\"timeZoneOffset\\\":\\\"UTC+05.30\\\",\\\"internalHeaderFields\\\":{\\\"collectorTimeStamp\\\":\\\"Thu, 05 12 2022 06:20:00 UTC\\\"},\\\"priority\\\":\\\"Normal\\\",\\\"version\\\":\\\"4.0.1\\\",\\\"reportingEntityName\\\":\\\"NOK6061ZW3\\\",\\\"sequence\\\":0,\\\"domain\\\":\\\"notification\\\",\\\"lastEpochMicrosec\\\":8745745764578,\\\"eventName\\\":\\\"Notification_gnb-Nokia_FileReady\\\",\\\"vesEventListenerVersion\\\":\\\"7.0.1\\\",\\\"sourceName\\\":\\\"NOK6061ZW3\\\"},\\\"notificationFields\\\":{\\\"notificationFieldsVersion\\\":\\\"2.0\\\",\\\"changeType\\\":\\\"FileReady\\\",\\\"changeIdentifier\\\":\\\"PM_MEAS_FILES\\\",\\\"arrayOfNamedHashMap\\\":[{\\\"name\\\":\\\""
+ + filePublishName
+ + "\\\",\\\"hashMap\\\":{\\\"location\\\":\\\""
+ + fileUrl
+ + "\\\",\\\"fileFormatType\\\":\\\""
+ + filetType
+ + "\\\",\\\"fileFormatVersion\\\":\\\"V1\\\",\\\"compression\\\":\\\"gzip\\\"}}]}}}\"]";
+
+ res.end(result);
+}
+
+function handleAllTypeRequest(res, req) {
+ res.writeHead(200, {'Content-Type': 'text/plain'});
+ console.log('Received message');
+ req.on('data', chunk => {
+ console.log(`-----MESSAGE_CONTENT_BEGIN-----\n ${chunk}`);
+ console.log('-----MESSAGE_CONTENT_END-----');
+ });
+}
+
+function isVesNotificationRequest(req) {
+ var vesNotificationTopic = "unauthenticated.VES_NOTIFICATION_OUTPUT"
+ return req.url.includes(vesNotificationTopic);
+}
+
var httpServer = function () {
var http = require('http'),
@@ -30,12 +63,14 @@ var httpServer = function () {
},
processHttpRequest = function (req, res) {
- res.writeHead(200, {'Content-Type': 'text/plain'});
- console.log('Received message');
- req.on('data', chunk => {
- console.log(`-----MESSAGE_CONTENT_BEGIN-----\n ${chunk}`);
- console.log('-----MESSAGE_CONTENT_END-----');
- });
+
+ if (isVesNotificationRequest(req)) {
+ console.log("Received VES_NOTIFICATION request ")
+ handleVesNotificationRequest(res);
+ return;
+ }
+
+ handleAllTypeRequest(res, req);
setTimeout(() => {
res.end('Published' + ' Successfully.\n');
}, 100)
diff --git a/tools/development/resources/datafile/mount_config.yaml b/tools/development/resources/datafile/mount_config.yaml
new file mode 100644
index 0000000..284c598
--- /dev/null
+++ b/tools/development/resources/datafile/mount_config.yaml
@@ -0,0 +1,29 @@
+dmaap.certificateConfig.keyCert: /opt/app/datafile/etc/cert/cert.p12
+dmaap.certificateConfig.keyPasswordPath: /opt/app/datafile/etc/cert/p12.pass
+dmaap.certificateConfig.trustedCa: /opt/app/datafile/etc/cert/trust.jks
+dmaap.certificateConfig.trustedCaPasswordPath: /opt/app/datafile/etc/cert/trust.pass
+dmaap.dmaapConsumerConfiguration.consumerGroup: OpenDcae-c12
+dmaap.dmaapConsumerConfiguration.consumerId: C12
+dmaap.dmaapConsumerConfiguration.timeoutMs: -1
+dmaap.security.enableDmaapCertAuth: false
+dmaap.security.keyStorePasswordPath: /opt/app/datafile/etc/cert/jks.pass
+dmaap.security.keyStorePath: /opt/app/datafile/etc/cert/cert.jks
+dmaap.security.trustStorePasswordPath: /opt/app/datafile/etc/cert/trust.pass
+dmaap.security.trustStorePath: /opt/app/datafile/etc/cert/trust.jks
+service_calls: []
+sftp.security.strictHostKeyChecking: true
+streams_publishes:
+ PM_MEAS_FILES:
+ dmaap_info:
+ location: loc00
+ log_url: https://dmaap-dr-prov/feedlog/1
+ password: pmmapper
+ publish_url: http://dmaap-dr-prov:8080/publish/1
+ publisher_id: 1.85gkg
+ username: pmmapper
+ type: data_router
+streams_subscribes:
+ dmaap_subscriber:
+ dmaap_info:
+ topic_url: http://mr-simulator:3904/events/unauthenticated.VES_NOTIFICATION_OUTPUT
+ type: message_router
diff --git a/tools/development/resources/datafile/spring_application.yaml b/tools/development/resources/datafile/spring_application.yaml
new file mode 100644
index 0000000..a3c3036
--- /dev/null
+++ b/tools/development/resources/datafile/spring_application.yaml
@@ -0,0 +1,31 @@
+spring:
+ profiles:
+ active: prod
+management:
+ endpoints:
+ web:
+ exposure:
+ include: "loggers,logfile,health,info,metrics"
+server:
+ port: 8433
+ ssl:
+ key-store-type: PKCS12
+ key-store-password: ericssondfc
+ key-store: classpath:keystore.jks
+ key-password: ericssondfc
+ keyAlias: tomcat-localhost
+logging:
+ level:
+ ROOT: INFO
+ org.springframework: INFO
+ org.springframework.data: INFO
+ org.springframework.web.reactive.function.client.ExchangeFunctions: INFO
+ org.onap.dcaegen2.collectors.datafile: TRACE
+ org.onap.dcaegen2: TRACE
+ file: /var/log/ONAP/application.log
+app:
+ filepath: config/datafile_endpoints_test.json
+
+springdoc:
+ show-actuator: true
+ swagger-ui.disable-swagger-default-url: true
diff --git a/tools/development/restart-pm-mapper.sh b/tools/development/restart-pm-mapper.sh
index 3574dd5..86f8e83 100755
--- a/tools/development/restart-pm-mapper.sh
+++ b/tools/development/restart-pm-mapper.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021 NOKIA
+# Copyright (C) 2021 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/tools/development/run-datafile.sh b/tools/development/run-datafile.sh
new file mode 100755
index 0000000..2b1f070
--- /dev/null
+++ b/tools/development/run-datafile.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+# Copyright (C) 2022 Nokia. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+source ./env/containers_ip
+IMAGE=nexus3.onap.org:10003/onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server:latest
+TARGET_CONFIG_PATH=/app-config/application_config.yaml
+TARGET_SPRING_CONFIG=/opt/app/datafile/config/application.yaml
+
+docker run -d -p 8100:8100 -p 8000:8000 \
+ --mount type=bind,source="$PWD/resources/datafile/mount_config.yaml",target="$TARGET_CONFIG_PATH" \
+ --mount type=bind,source="$PWD/resources/datafile/spring_application.yaml",target="$TARGET_SPRING_CONFIG" \
+ -e "JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n" \
+ -e "CONFIG_BINDING_SERVICE=0.0.0.0" \
+ -e "CONFIG_BINDING_SERVICE_SERVICE_PORT=10000" \
+ -e "CBS_CLIENT_CONFIG_PATH=$TARGET_CONFIG_PATH" \
+ --add-host "dmaap-dr-node:$DR_NODE_IP" \
+ --add-host "dmaap-dr-prov:$DR_PROV_IP" \
+ --network=development_pmmapper-network \
+ --name=datafile-dev $IMAGE
diff --git a/tools/development/run-pm-mapper.sh b/tools/development/run-pm-mapper.sh
index 91c7766..15f1e0f 100755
--- a/tools/development/run-pm-mapper.sh
+++ b/tools/development/run-pm-mapper.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,9 +18,10 @@ source ./env/containers_ip
IMAGE=onap/org.onap.dcaegen2.services.pm-mapper:latest
TARGET_CONFIG_PATH=/app-config/application_config.yaml
-docker run -d -p 8081:8081 \
+docker run -d -p 8081:8081 -p 5005:5005 \
--mount type=bind,source="$PWD/certs",target="/opt/app/pm-mapper/etc/certs/" \
--mount type=bind,source="$PWD/resources/mount_config.yaml",target="$TARGET_CONFIG_PATH" \
+ -e "JAVA_TOOL_OPTIONS=-agentlib:jdwp=transport=dt_socket,address=*:5005,server=y,suspend=n" \
-e "CONFIG_BINDING_SERVICE=0.0.0.0" \
-e "CONFIG_BINDING_SERVICE_SERVICE_PORT=10000" \
-e "CBS_CLIENT_CONFIG_PATH=$TARGET_CONFIG_PATH" \
diff --git a/tools/development/send-meas-collec.sh b/tools/development/send-meas-collec.sh
index 6f70bd7..6d36332 100755
--- a/tools/development/send-meas-collec.sh
+++ b/tools/development/send-meas-collec.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/tools/development/send-meas-data.sh b/tools/development/send-meas-data.sh
index 7c1a77b..729469a 100755
--- a/tools/development/send-meas-data.sh
+++ b/tools/development/send-meas-data.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/tools/development/setup-local.sh b/tools/development/setup-local.sh
index 94a115d..aa4874f 100755
--- a/tools/development/setup-local.sh
+++ b/tools/development/setup-local.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ source ./env/containers_ip
sed -i 's/datarouter-mariadb/'$MARIADB_IP'/g' $SCRIPT_DIR/dr-mount/provserver.properties
-docker-compose -f $SCRIPT_DIR/docker-compose.yml up -d mariadb node files-publisher
+docker-compose -f $SCRIPT_DIR/docker-compose.yml up -d mariadb node files-publisher sftp
echo "Waiting for MariaDB to come up healthy..."
for i in {1..30}; do
diff --git a/tools/development/subscribe-pm-mapper.sh b/tools/development/subscribe-pm-mapper.sh
index 1b8a532..790bfe6 100755
--- a/tools/development/subscribe-pm-mapper.sh
+++ b/tools/development/subscribe-pm-mapper.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# ============LICENSE_START=======================================================
-# Copyright (C) 2021-2022 NOKIA
+# Copyright (C) 2021-2022 Nokia. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.