netconf-pnp-simulator: convenient TLS and SSH configuration

- Simple SSH and TLS configuration. Instead of specific Netopeer2
  XML configuration files, the user only needs to provide:
  For SSH: id_XXX.pub
  For TLS: server_key.pem, server_cert.pem, and ca.pem

- SSH and TLS can be reconfigured at runtime by running
  /opt/bin/reconfigure-ssh.sh and /opt/bin/reconfigure-tls.sh respectively

- Improved log readability by using zlog (on C applications) and loguru
  for Python

See the updated documentation under ../docs for more information.

Issue-ID: INT-1516
Change-Id: I21052d2524f0610c6197875a544113cce1a02787
Signed-off-by: ebo <eliezio.oliveira@est.tech>

3 files changed, 303 insertions, 0 deletions

diff --git a/test/mocks/netconf-pnp-simulator/engine/patches/Netopeer2/02-zlog.patch b/test/mocks/netconf-pnp-simulator/engine/patches/Netopeer2/02-zlog.patch
new file mode 100644
index 000000000..804b6525c
--- /dev/null
+++ b/test/mocks/netconf-pnp-simulator/engine/patches/Netopeer2/02-zlog.patch
@@ -0,0 +1,105 @@
+diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt
+index f0c82c1..99c6a3d 100755
+--- a/server/CMakeLists.txt
++++ b/server/CMakeLists.txt
+@@ -130,6 +130,13 @@ add_library(serverobj OBJECT ${srcs})
+ # netopeer2-server target
+ add_executable(netopeer2-server $<TARGET_OBJECTS:serverobj> main.c)
+
++# dependencies - zlog
++find_library(ZLOG zlog)
++if(NOT ZLOG)
++    message(FATAL_ERROR "Unable to find zlog library.")
++endif()
++target_link_libraries(netopeer2-server ${ZLOG})
++
+ # dependencies - pthread
+ set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+ find_package(Threads REQUIRED)
+diff --git a/server/log.c b/server/log.c
+index e660635..6b8117b 100644
+--- a/server/log.c
++++ b/server/log.c
+@@ -27,6 +27,8 @@
+ #include <nc_server.h>
+ #include <sysrepo.h>
+
++#include <zlog.h>
++
+ volatile uint8_t np2_verbose_level;
+ uint8_t np2_libssh_verbose_level;
+ uint8_t np2_sr_verbose_level;
+@@ -102,44 +104,24 @@ np2_err_location(void)
+ static void
+ np2log(int priority, const char *fmt, ...)
+ {
+-    char *format;
+     va_list ap;
+
+     va_start(ap, fmt);
+-    vsyslog(priority, fmt, ap);
+-    va_end(ap);
+-
+-    if (np2_stderr_log) {
+-        format = malloc(11 + strlen(fmt) + 2);
+-        if (!format) {
+-            fprintf(stderr, "ERROR: Memory allocation failed (%s:%d)", __FILE__, __LINE__);
+-            return;
+-        }
+-
+-        switch (priority) {
+-        case LOG_ERR:
+-            sprintf(format, "[ERR]: %s\n", fmt);
++    switch (priority) {
++        case LOG_INFO:
++            vdzlog_info(fmt, ap);
+             break;
+         case LOG_WARNING:
+-            sprintf(format, "[WRN]: %s\n", fmt);
+-            break;
+-        case LOG_INFO:
+-            sprintf(format, "[INF]: %s\n", fmt);
++            vdzlog_warn(fmt, ap);
+             break;
+         case LOG_DEBUG:
+-            sprintf(format, "[DBG]: %s\n", fmt);
++            vdzlog_debug(fmt, ap);
+             break;
+         default:
+-            sprintf(format, "[UNKNOWN]: %s\n", fmt);
++            vdzlog_error(fmt, ap);
+             break;
+-        }
+-
+-        va_start(ap, fmt);
+-        vfprintf(stderr, format, ap);
+-        va_end(ap);
+-
+-        free(format);
+     }
++    va_end(ap);
+ }
+
+ /**
+diff --git a/server/main.c b/server/main.c
+index 601e8a8..9d28931 100644
+--- a/server/main.c
++++ b/server/main.c
+@@ -39,6 +39,8 @@
+ #include <nc_server.h>
+ #include <sysrepo.h>
+
++#include <zlog.h>
++
+ #include "common.h"
+ #include "operations.h"
+ #include "netconf_monitoring.h"
+@@ -1545,6 +1547,8 @@ main(int argc, char *argv[])
+     openlog("netopeer2-server", LOG_PID, LOG_DAEMON);
+     np2_stderr_log = 1;
+
++    dzlog_init("/opt/etc/zlog.conf", "netopeer2-server");
++
+     /* process command line options */
+     while ((c = getopt(argc, argv, OPTSTRING)) != -1) {
+         switch (c) {
diff --git a/test/mocks/netconf-pnp-simulator/engine/patches/supervisor/01-std-log-format.patch b/test/mocks/netconf-pnp-simulator/engine/patches/supervisor/01-std-log-format.patch
new file mode 100644
index 000000000..528a37415
--- /dev/null
+++ b/test/mocks/netconf-pnp-simulator/engine/patches/supervisor/01-std-log-format.patch
@@ -0,0 +1,26 @@
+diff --git a/supervisor/loggers.py b/supervisor/loggers.py
+index 84d47ae..d23db3c 100644
+--- a/supervisor/loggers.py
++++ b/supervisor/loggers.py
+@@ -287,7 +287,7 @@ class LogRecord:
+             now = time.time()
+             msecs = (now - long(now)) * 1000
+             part1 = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now))
+-            asctime = '%s,%03d' % (part1, msecs)
++            asctime = '%s.%03d' % (part1, msecs)
+             levelname = LOG_LEVELS_BY_NUM[self.level]
+             msg = as_string(self.msg)
+             if self.kw:
+diff --git a/supervisor/options.py b/supervisor/options.py
+index 4e98340..fc19300 100644
+--- a/supervisor/options.py
++++ b/supervisor/options.py
+@@ -1463,7 +1463,7 @@ class ServerOptions(Options):
+
+     def make_logger(self):
+         # must be called after realize() and after supervisor does setuid()
+-        format = '%(asctime)s %(levelname)s %(message)s\n'
++        format = '%(asctime)s %(levelname)-5s [supervisor] %(message)s\n'
+         self.logger = loggers.getLogger(self.loglevel)
+         if self.nodaemon:
+             loggers.handle_stdout(self.logger, format)
diff --git a/test/mocks/netconf-pnp-simulator/engine/patches/sysrepo/02-zlog.patch b/test/mocks/netconf-pnp-simulator/engine/patches/sysrepo/02-zlog.patch
new file mode 100644
index 000000000..0223563c3
--- /dev/null
+++ b/test/mocks/netconf-pnp-simulator/engine/patches/sysrepo/02-zlog.patch
@@ -0,0 +1,172 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 14c8467..5af087e 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -105,6 +105,11 @@ configure_file(${PROJECT_SOURCE_DIR}/inc/sysrepo/values.h.in ${PROJECT_BINARY_DI
+ configure_file(${PROJECT_SOURCE_DIR}/inc/sysrepo/xpath.h ${PROJECT_BINARY_DIR}/inc/sysrepo/xpath.h COPYONLY)
+
+ # find required libraries
++find_library(ZLOG zlog)
++if(NOT ZLOG)
++    message(FATAL_ERROR "zlog must be installed.")
++endif()
++
+ find_package(EV REQUIRED)
+ include_directories(${EV_INCLUDE_DIR})
+
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 342ad9d..d026a81 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -85,9 +85,9 @@ add_dependencies(SR_SRC COMMON)
+ add_dependencies(SR_ENGINE COMMON)
+
+ if(USE_AVL_LIB)
+-    set(LINK_LIBRARIES pthread ${AVL_LIBRARIES} ${EV_LIBRARIES} ${PROTOBUF-C_LIBRARIES} ${YANG_LIBRARIES})
++    set(LINK_LIBRARIES pthread ${AVL_LIBRARIES} ${EV_LIBRARIES} ${PROTOBUF-C_LIBRARIES} ${YANG_LIBRARIES} ${ZLOG})
+ else(USE_AVL_LIB)
+-    set(LINK_LIBRARIES pthread ${REDBLACK_LIBRARIES} ${EV_LIBRARIES} ${PROTOBUF-C_LIBRARIES} ${YANG_LIBRARIES})
++    set(LINK_LIBRARIES pthread ${REDBLACK_LIBRARIES} ${EV_LIBRARIES} ${PROTOBUF-C_LIBRARIES} ${YANG_LIBRARIES} ${ZLOG})
+ endif(USE_AVL_LIB)
+
+ #handle rt library that doesn't exist on OS X
+diff --git a/src/common/sr_logger.c b/src/common/sr_logger.c
+index 8dd6f31..ea94044 100644
+--- a/src/common/sr_logger.c
++++ b/src/common/sr_logger.c
+@@ -29,6 +29,8 @@
+ #include <stdarg.h>
+ #include <pthread.h>
+
++#include <zlog.h>
++
+ #include "sr_common.h"
+ #include "sr_logger.h"
+
+@@ -76,6 +78,7 @@ void
+ sr_logger_init(const char *app_name)
+ {
+ #if SR_LOGGING_ENABLED
++    dzlog_init("/opt/etc/zlog.conf", app_name);
+     if (NULL != sr_syslog_identifier) {
+         /* if some syslog identifier was already set, release it as we are going to set new one */
+         free((char*)sr_syslog_identifier);
+diff --git a/src/common/sr_logger.h b/src/common/sr_logger.h
+index 37c3487..c95a68d 100644
+--- a/src/common/sr_logger.h
++++ b/src/common/sr_logger.h
+@@ -31,6 +31,8 @@
+ #include <syslog.h>
+ #include <pthread.h>
+
++#include <zlog.h>
++
+ #include "sr_constants.h"
+
+ /**
+@@ -156,37 +158,31 @@ extern __thread char strerror_buf [SR_MAX_STRERROR_LEN]; /**< thread local buffe
+ /**
+  * Internal output macro
+  */
+-#define SR_LOG__INTERNAL(LL, MSG, ...) \
+-    do { \
+-        if (sr_ll_stderr >= LL) \
+-            SR_LOG__STDERR(LL, MSG, __VA_ARGS__) \
+-        if (sr_ll_syslog >= LL) \
+-            SR_LOG__SYSLOG(LL, MSG, __VA_ARGS__) \
+-        if (NULL != sr_log_callback) \
+-            SR_LOG__CALLBACK(LL, MSG, __VA_ARGS__) \
+-    } while(0)
+-
+ #if SR_LOGGING_ENABLED
+
+ /** Prints an error message (with format specifiers). */
+-#define SR_LOG_ERR(MSG, ...) SR_LOG__INTERNAL(SR_LL_ERR, MSG, __VA_ARGS__)
++#define SR_LOG_ERR(MSG, ...) dzlog(__FILE__, sizeof(__FILE__)-1, __func__, sizeof(__func__)-1, \
++        __LINE__, ZLOG_LEVEL_ERROR, MSG, __VA_ARGS__)
+ /** Prints an error message. */
+-#define SR_LOG_ERR_MSG(MSG) SR_LOG__INTERNAL(SR_LL_ERR, MSG "%s", "")
++#define SR_LOG_ERR_MSG(MSG) SR_LOG_ERR(MSG "%s", "")
+
+ /** Prints a warning message (with format specifiers). */
+-#define SR_LOG_WRN(MSG, ...) SR_LOG__INTERNAL(SR_LL_WRN, MSG, __VA_ARGS__)
++#define SR_LOG_WRN(MSG, ...) dzlog(__FILE__, sizeof(__FILE__)-1, __func__, sizeof(__func__)-1, \
++        __LINE__, ZLOG_LEVEL_WARN, MSG, __VA_ARGS__)
+ /** Prints a warning message. */
+-#define SR_LOG_WRN_MSG(MSG) SR_LOG__INTERNAL(SR_LL_WRN, MSG "%s", "")
++#define SR_LOG_WRN_MSG(MSG) SR_LOG_WRN(MSG "%s", "")
+
+ /** Prints an informational message (with format specifiers). */
+-#define SR_LOG_INF(MSG, ...) SR_LOG__INTERNAL(SR_LL_INF, MSG, __VA_ARGS__)
++#define SR_LOG_INF(MSG, ...) dzlog(__FILE__, sizeof(__FILE__)-1, __func__, sizeof(__func__)-1, \
++        __LINE__, ZLOG_LEVEL_INFO, MSG, __VA_ARGS__)
+ /** Prints an informational message. */
+-#define SR_LOG_INF_MSG(MSG) SR_LOG__INTERNAL(SR_LL_INF, MSG "%s", "")
++#define SR_LOG_INF_MSG(MSG) SR_LOG_INF(MSG "%s", "")
+
+ /** Prints a development debug message (with format specifiers). */
+-#define SR_LOG_DBG(MSG, ...) SR_LOG__INTERNAL(SR_LL_DBG, MSG, __VA_ARGS__)
++#define SR_LOG_DBG(MSG, ...) dzlog(__FILE__, sizeof(__FILE__)-1, __func__, sizeof(__func__)-1, \
++        __LINE__, ZLOG_LEVEL_DEBUG, MSG, __VA_ARGS__)
+ /** Prints a development debug message. */
+-#define SR_LOG_DBG_MSG(MSG) SR_LOG__INTERNAL(SR_LL_DBG, MSG "%s", "")
++#define SR_LOG_DBG_MSG(MSG) SR_LOG_DBG(MSG "%s", "")
+
+ #else
+ #define SR_LOG_ERR(...)
+diff --git a/src/executables/sysrepocfg.c b/src/executables/sysrepocfg.c
+index 0000951..f48ed5e 100644
+--- a/src/executables/sysrepocfg.c
++++ b/src/executables/sysrepocfg.c
+@@ -2000,6 +2000,9 @@ main(int argc, char* argv[])
+         }
+     }
+
++    /* init logger */
++    sr_logger_init("sysrepocfg");
++
+     /* set log levels */
+     sr_log_stderr(SR_LL_ERR);
+     sr_log_syslog(SR_LL_NONE);
+diff --git a/src/executables/sysrepoctl.c b/src/executables/sysrepoctl.c
+index 3b02e7d..60ffd7e 100644
+--- a/src/executables/sysrepoctl.c
++++ b/src/executables/sysrepoctl.c
+@@ -1311,6 +1311,9 @@ main(int argc, char* argv[])
+         search_dir_count = 1;
+     }
+
++    /* init logger */
++    sr_logger_init("sysrepoctl");
++
+     /* set log levels */
+     sr_log_stderr(SR_LL_ERR);
+     sr_log_syslog(SR_LL_NONE);
+diff --git a/src/clientlib/client_library.c b/src/clientlib/client_library.c
+index c3da2e5..b3beab7 100644
+--- a/src/clientlib/client_library.c
++++ b/src/clientlib/client_library.c
+@@ -377,6 +377,11 @@ sr_connect(const char *app_name, const sr_conn_options_t opts, sr_conn_ctx_t **c
+
+     CHECK_NULL_ARG2(app_name, conn_ctx_p);
+
++    if (0 == connections_cnt) {
++        /* this is the first connection - initialize logging */
++        sr_logger_init(app_name);
++    }
++
+     SR_LOG_DBG_MSG("Connecting to Sysrepo Engine.");
+
+     /* create the connection */
+@@ -385,11 +390,6 @@ sr_connect(const char *app_name, const sr_conn_options_t opts, sr_conn_ctx_t **c
+
+     pthread_mutex_lock(&global_lock);
+
+-    if (0 == connections_cnt) {
+-        /* this is the first connection - initialize logging */
+-        sr_logger_init(app_name);
+-    }
+-
+     /* attempt to connect to sysrepo daemon socket */
+     rc = cl_socket_connect(connection, SR_DAEMON_SOCKET);
+     if (SR_ERR_OK != rc) {