adding snmpV3 support

Change-Id: I6250e30fa1aa2516a16c4906628be8cc904fbc71
Issue-ID: DCAEGEN2-630
Signed-off-by: Ladue, David (dl3158) <dl3158@att.com>

8 files changed, 609 insertions, 101 deletions

diff --git a/snmptrap/healthcheck.sh b/snmptrap/healthcheck.sh
index d23c859..371c18a 100755
--- a/snmptrap/healthcheck.sh
+++ b/snmptrap/healthcheck.sh
@@ -21,5 +21,7 @@
 # health. A better possible health check is if the distribution client provided a
 # health check call.
 
-exit 0
-
+# run standard status command, exit with results
+/opt/app/snmptrap/bin/snmptrapd.sh status > /dev/null 2>&1
+ret=$?
+exit ${ret}
diff --git a/snmptrap/mod/trapd_get_cbs_config.py b/snmptrap/mod/trapd_get_cbs_config.py
index 524f1c2..1506dca 100644
--- a/snmptrap/mod/trapd_get_cbs_config.py
+++ b/snmptrap/mod/trapd_get_cbs_config.py
@@ -86,6 +86,8 @@ def get_cbs_config():
             stdout_logger(msg)
             try:
                 tds.c_config = json.load(open(_cbs_sim_json_file))
+                msg = ("%s loaded and parsed successfully" % _cbs_sim_json_file)
+                stdout_logger(msg)
             except Exception as e:
                 msg = "Unable to load CBS_SIM_JSON " + _cbs_sim_json_file + \
                     " (invalid json?) - FATAL ERROR, exiting"
@@ -94,25 +96,25 @@ def get_cbs_config():
 
     # recalc timeout, set default if not present
     try:
-        tds.timeout_seconds = tds.c_config['publisher.http_timeout_milliseconds'] / 1000.0
+        tds.timeout_seconds = tds.c_config['publisher']['http_timeout_milliseconds'] * 1000.0
     except Exception as e:
         tds.timeout_seconds = 1.5
 
     # recalc seconds_between_retries, set default if not present
     try:
-        tds.seconds_between_retries = tds.c_config['publisher.http_milliseconds_between_retries'] / 1000.0
+        tds.seconds_between_retries = tds.c_config['publisher']['http_milliseconds_between_retries'] * 1000.0
     except Exception as e:
         tds.seconds_between_retries = .750
 
     # recalc min_severity_to_log, set default if not present
     try:
-        tds.minimum_severity_to_log = tds.c_config['files.minimum_severity_to_log']
+        tds.minimum_severity_to_log = tds.c_config['files']['minimum_severity_to_log']
     except Exception as e:
         tds.minimum_severity_to_log = 3
 
     try:
-        tds.publisher_retries = tds.c_config['publisher.http_retries']
+        tds.publisher_retries = tds.c_config['publisher']['http_retries']
     except Exception as e:
-        tds.publisher_retries = 3
+        tds.publisher_retries = 2
 
     return True
diff --git a/snmptrap/mod/trapd_http_session.py b/snmptrap/mod/trapd_http_session.py
index b34c19d..3efca21 100644
--- a/snmptrap/mod/trapd_http_session.py
+++ b/snmptrap/mod/trapd_http_session.py
@@ -53,6 +53,68 @@ def init_session_obj():
     try:
         _loc_session = requests.Session()
     except Exception as e:
-        return None
+        msg = "Unable to create new http session - FATAL ERROR, exiting"
+        ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_FATAL, tds.CODE_GENERAL, msg)
+        stdout_logger(msg)
+        cleanup_and_exit(1, tds.pid_file_name)
 
     return _loc_session
+
+
+# # # # # # # # # # # # #
+# fx: close_session_obj
+# # # # # # # # # # # # #
+def close_session_obj(_loc_http_requ_session):
+    """
+    Closes existing http request session object
+    :Parameters:
+      _loc_http_requ_session
+    :Exceptions:
+      session object creation
+        this function will throw an exception if unable to create
+        a new session object
+    :Keywords:
+      http request session
+    :Variables:
+      none
+    """
+
+
+    # Close existing session if present.
+    if _loc_http_requ_session is not None:
+        try:
+            _loc_http_requ_session.close()
+            return True
+        except Exception as e:
+            msg = "Unable to close current http session - FATAL ERROR, exiting"
+            ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_FATAL, tds.CODE_GENERAL, msg)
+            stdout_logger(msg)
+            cleanup_and_exit(1, tds.pid_file_name)
+
+
+# # # # # # # # # # # # #
+# fx: reset_session_obj
+# # # # # # # # # # # # #
+def reset_session_obj(_loc_http_requ_session):
+    """
+    Closes existing http request session object
+    and re-opens with current config vals
+    :Parameters:
+      _loc_http_requ_session
+    :Exceptions:
+      session object creation
+        this function will throw an exception if unable to create
+        a new session object
+    :Keywords:
+      http request session
+    :Variables:
+      none
+    """
+
+
+    # close existing http_requ_session if present
+    ret = close_session_obj(_loc_http_requ_session)
+
+    # open new http_requ_session
+    _loc_http_requ_session = init_session_obj()
+    return _loc_http_requ_session
diff --git a/snmptrap/mod/trapd_io.py b/snmptrap/mod/trapd_io.py
index ef67c69..d079cbe 100644
--- a/snmptrap/mod/trapd_io.py
+++ b/snmptrap/mod/trapd_io.py
@@ -117,7 +117,7 @@ def open_eelf_logs():
         # open various ecomp logs - if any fails, exit
 
         tds.eelf_error_file_name = (
-            tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_error'])
+            tds.c_config['files']['eelf_base_dir'] + "/" + tds.c_config['files']['eelf_error'])
         tds.eelf_error_fd = open_file(tds.eelf_error_file_name)
 
     except Exception as e:
@@ -127,7 +127,7 @@ def open_eelf_logs():
 
     try:
         tds.eelf_debug_file_name = (
-            tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_debug'])
+            tds.c_config['files']['eelf_base_dir'] + "/" + tds.c_config['files']['eelf_debug'])
         tds.eelf_debug_fd = open_file(tds.eelf_debug_file_name)
 
     except Exception as e:
@@ -137,7 +137,7 @@ def open_eelf_logs():
 
     try:
         tds.eelf_audit_file_name = (
-            tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_audit'])
+            tds.c_config['files']['eelf_base_dir'] + "/" + tds.c_config['files']['eelf_audit'])
         tds.eelf_audit_fd = open_file(tds.eelf_audit_file_name)
     except Exception as e:
         msg = "Error opening eelf audit log : " + str(e)
@@ -146,7 +146,7 @@ def open_eelf_logs():
 
     try:
         tds.eelf_metrics_file_name = (
-            tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_metrics'])
+            tds.c_config['files']['eelf_base_dir'] + "/" + tds.c_config['files']['eelf_metrics'])
         tds.eelf_metrics_fd = open_file(tds.eelf_metrics_file_name)
     except Exception as e:
         msg = "Error opening eelf metric log : " + str(e)
diff --git a/snmptrap/mod/trapd_settings.py b/snmptrap/mod/trapd_settings.py
index be87e26..308a2f2 100644
--- a/snmptrap/mod/trapd_settings.py
+++ b/snmptrap/mod/trapd_settings.py
@@ -115,7 +115,7 @@ def init():
 
     # <json log of traps published>
     global json_traps_filename
-    json_log_filename = ""
+    json_traps_filename = ""
     global json_traps_fd
     json_fd = None
     # </json log of traps published>
@@ -131,6 +131,17 @@ def init():
     global pid_file_name
     pid_file_name = ""
 
+    # <stormwatch>
+    global sw_participant_oid_dict
+    sw_participant_oid_dict = {}
+
+    global sw_count_dict
+    sw_count_dict = {}
+
+    global sw_interval_in_seconds
+    sw_interval_in_seconds = 60
+    # </stormwatch >
+
     # <logging types and severities>
     global LOG_TYPES
     global LOG_TYPE_NONE
diff --git a/snmptrap/mod/trapd_snmpv3.py b/snmptrap/mod/trapd_snmpv3.py
new file mode 100644
index 0000000..97c048c
--- /dev/null
+++ b/snmptrap/mod/trapd_snmpv3.py
@@ -0,0 +1,195 @@
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+"""
+module for snmpv3 support
+
+- loads various USM values for engineID/users
+
+"""
+
+__docformat__ = 'restructuredtext'
+
+import json
+import os
+import sys
+import string
+import time
+import traceback
+import collections
+import pprint
+
+from pysnmp.entity import engine, config
+from pysnmp.carrier.asyncore.dgram import udp
+from pysnmp.entity.rfc3413 import ntfrcv
+from pysnmp.proto.api import v2c
+
+import trapd_settings as tds
+from trapd_exit import cleanup_and_exit
+from trapd_io import stdout_logger, ecomp_logger
+
+prog_name = os.path.basename(__file__)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# module: init
+#         FMDL: should this re-establish listener, with 
+#               blank V3 config attribs?
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+def init():
+
+    global v3_config_dict
+    v3_config_dict = {}
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# module: load_snmpv3_credentials
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+def load_snmpv3_credentials (_py_config, _snmp_engine, _cbs_config):
+    """
+    Add V3 credentials from CBS config to receiver config 
+    so traps will be recieved from specified engines/users
+    :Parameters:
+      _config: snmp entity config
+    :Exceptions:
+    """
+
+    # add V3 credentials from CBS json structure to running config
+    try:
+       v3_users=_cbs_config["snmpv3_config"]["usm_users"]
+    except Exception as e:
+        msg = ("No V3 users defined")
+        ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg)
+        return _py_config, _snmp_engine
+
+    for v3_user in v3_users:
+
+        # engineId
+        try:
+            ctx_engine_id=v3_user['engineId']
+        except Exception as e:
+            ctx_engine_id=None
+
+        # user
+        try:
+            userName=v3_user['user']
+        except Exception as e:
+            userName=None
+
+        # authorization
+        #     find options at -> site-packages/pysnmp/entity/config.py
+        # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
+
+        # print("Checking auth for %s" % (userName))
+
+        # usmHMACMD5Auth
+        try:
+            authKey=v3_user['usmHMACMD5Auth']
+            authProtocol=config.usmHMACMD5AuthProtocol 
+        except Exception as e:
+            # usmHMACSHAAuth
+            try:
+                authKey=v3_user['usmHMACSHAAuth']
+                authProtocol=config.usmHMAC192SHA256AuthProtocol
+            except Exception as e:
+                # usmNoAuth
+                try:
+                    authKey=v3_user['usmNoAuth']
+                    authProtocol=config.usmNoAuthProtocol
+                except Exception as e:
+                    # FMDL:  default to NoAuth, or error/skip entry?
+                    msg = ("No auth specified for user %s ?" % (userName))
+                    authKey=None
+                    authProtocol=config.usmNoAuthProtocol
+                    ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg)
+                    # break
+
+        # privacy
+        #     find options at -> site-packages/pysnmp/entity/config.py
+        # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
+
+        # print("Checking priv for %s" % (userName))
+
+        # usm3DESEDEPriv
+        try:
+            privKey=v3_user['usm3DESEDEPriv']
+            privProtocol=config.usm3DESEDEPrivProtocol
+        except Exception as e:
+            # usmAesCfb128
+            try:
+                privKey=v3_user['usmAesCfb128']
+                privProtocol=config.usmAesCfb128Protocol
+            except Exception as e:
+                # usmAesCfb192
+                try:
+                    privKey=v3_user['usmAesCfb192']
+                    privProtocol=config.usmAesCfb192Protocol
+                except Exception as e:
+                    # usmAesCfb256
+                    try:
+                        privKey=v3_user['usmAesCfb256']
+                        privProtocol=config.usmAesCfb256Protocol
+                    except Exception as e:
+                        # usmDESPriv
+                        try:
+                            privKey=v3_user['usmDESPriv']
+                            privProtocol=config.usmDESPrivProtocol
+                        except Exception as e:
+                            # usmNoPriv
+                            try:
+                                privKey=v3_user['usmNoPriv']
+                                privProtocol=config.usmNoPrivProtocol
+                            except Exception as e:
+                                # FMDL:  default to NoPriv, or error/skip entry?
+                                msg = ("No priv specified for user %s" % (userName))
+                                ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg)
+                                privKey=None
+                                privProtocol=config.usmNoPrivProtocol
+                                # break
+
+        # msg = ("userName: %s authKey: %s authProtocol: %s privKey: %s privProtocol: %s engineId: %s % (userName, authKey, authProtocol, privKey, privProtocol, ctx_engine_id))
+        msg = ("userName: %s authKey: **** authProtocol: %s privKey: **** privProtocol: %s engineId: ****" % (userName, authProtocol, privProtocol))
+        ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg)
+
+        # user: usr-md5-des, auth: MD5, priv DES, contextEngineId: 8000000001020304
+        # this USM entry is used for TRAP receiving purposes
+
+        # help(addV3User) returns -> 
+        # addV3User(snmpEngine, userName, authProtocol=(1, 3, 6, 1, 6, 3, 10, 1, 1, 1), authKey=None, privProtocol=(1, 3, 6, 1, 6, 3, 10, 1, 2, 1), priv    Key=None, securityEngineId=None, securityName=None, contextEngineId=None)
+
+        if ctx_engine_id is not None: 
+            config.addV3User(
+                _snmp_engine, userName,
+                authProtocol, authKey,
+                privProtocol, privKey,
+                contextEngineId=v2c.OctetString(hexValue=ctx_engine_id)
+            )
+        else:
+            config.addV3User(
+                _snmp_engine, userName,
+                authProtocol, authKey,
+                privProtocol, privKey
+            )
+
+    return _py_config, _snmp_engine
diff --git a/snmptrap/snmptrapd.py b/snmptrap/snmptrapd.py
index f435c30..cbf004b 100644
--- a/snmptrap/snmptrapd.py
+++ b/snmptrap/snmptrapd.py
@@ -28,7 +28,7 @@ As traps arrive they are decomposed and transformed into a JSON message which
 is published to a dmaap instance that has been defined by controller.
 
 :Parameters:
-    usage:  snmptrapd.py [-v]
+    usage:  snmptrapd.py
 :Keywords:
     onap dcae snmp trap publish dmaap
 """
@@ -65,14 +65,15 @@ from pysnmp.carrier.asyncio.dgram import udp, udp6
 # from pysnmp.carrier.asyncore.dgram import udp
 from pysnmp.entity.rfc3413 import ntfrcv
 from pysnmp.proto.api import v2c
+from pysnmp import debug
 
 # dcae_snmptrap
 import trapd_settings as tds
 from trapd_runtime_pid import save_pid, rm_pid
 from trapd_get_cbs_config import get_cbs_config
 from trapd_exit import cleanup_and_exit
-from trapd_http_session import init_session_obj
-
+from trapd_http_session import init_session_obj, close_session_obj, reset_session_obj
+from trapd_snmpv3 import init, load_snmpv3_credentials
 from trapd_io import roll_all_logs, open_eelf_logs, roll_file, open_file, close_file, ecomp_logger, stdout_logger
 
 prog_name = os.path.basename(__file__)
@@ -95,7 +96,7 @@ def usage_err():
     """
 
     print('Incorrect usage invoked.  Correct usage:')
-    print('  %s [-v]' % prog_name)
+    print('  %s' % prog_name)
     cleanup_and_exit(1, "undefined")
 
 
@@ -127,36 +128,34 @@ def load_all_configs(_signum, _frame):
         ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_DETAILED,
                      tds.CODE_GENERAL, msg)
 
-    # Initialize dmaap requests session object. Close existing session
-    # if applicable.
-    if tds.http_requ_session is not None:
-        tds.http_requ_session.close()
-
-    tds.http_requ_session = init_session_obj()
-    if tds.http_requ_session is None:
-        msg = "Unable to create new http session - FATAL ERROR, exiting"
-        ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_FATAL, tds.CODE_GENERAL, msg)
-        stdout_logger(msg)
-        cleanup_and_exit(1, tds.pid_file_name)
-
-    # re-request config from config binding service
-    # (either broker, or json file override)
+    # re-request config (from broker, or local json file 
+    # if broker not present)
     if not get_cbs_config():
-        msg = "error (re)loading CBS config - FATAL ERROR, exiting"
+        msg = "Error (re)loading CBS config - FATAL ERROR, exiting"
         stdout_logger(msg)
         cleanup_and_exit(1, tds.pid_file_name)
     else:
-        current_runtime_config_file_name = tds.c_config['files.runtime_base_dir'] + \
+        current_runtime_config_file_name = tds.c_config['files']['runtime_base_dir'] + \
             "/tmp/current_config.json"
-
-        msg = "current config logged to : %s" % current_runtime_config_file_name
+        if int(_signum) != 0:
+            msg = "updated config logged to : %s" % current_runtime_config_file_name
+        else:
+            msg = "current config logged to : %s" % current_runtime_config_file_name
         ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg)
 
         with open(current_runtime_config_file_name, 'w') as outfile:
             json.dump(tds.c_config, outfile)
 
-        # if here, config re-read successfully
-        return True
+    # reset http session based on latest config
+    tds.http_requ_session = reset_session_obj(tds.http_requ_session)
+
+    # FMDL: add with stormWatch
+    # reload sw participating entries, reset counter dictionary
+    # sw.interval_in_seconds, sw.participant_oid_dict = load_sw_participant_dict(tds.c_config['trap_config'])
+    # sw.counter_dict = init_counter_dict()
+
+    # if here, config re-read successfully
+    return True
 
 # # # # # # # # # # # # #
 # fx: log_all_arriving_traps
@@ -166,12 +165,12 @@ def load_all_configs(_signum, _frame):
 def log_all_arriving_traps():
 
     # roll logs as needed/defined in files.roll_frequency
-    if tds.c_config['files.roll_frequency'] == "minute":
+    if tds.c_config['files']['roll_frequency'] == "minute":
         curr_minute = datetime.datetime.now().minute
         if curr_minute != tds.last_minute:
             roll_all_logs()
             tds.last_minute = curr_minute
-    elif tds.c_config['files.roll_frequency'] == "hour":
+    elif tds.c_config['files']['roll_frequency'] == "hour":
         curr_hour = datetime.datetime.now().hour
         if curr_hour != tds.last_hour:
             roll_all_logs()
@@ -183,7 +182,7 @@ def log_all_arriving_traps():
             roll_all_logs()
             tds.last_day = curr_day
 
-    # now log latest arriving trap
+    # always log arriving trap
     try:
         # going for:
         #    1520971776 Tue Mar 13 16:09:36 2018; 1520971776 2018-03-13 16:09:36 DCAE-COLLECTOR-UCSNMP 15209717760049 .1.3.6.1.4.1.2636.4.1.6 gfpmt5pcs10.oss.att.com 135.91.10.139 12.123.1.240 12.123.1.240 2 varbinds: [0] .1.3.6.1.2.1.1.3.0 {10} 1212058366 140 days, 6:49:43.66 [1] .1.3.6.1.6.3.1.1.4.1.0 {6} .1.3.6.1.4.1.2636.4.1.6 [2] .1.3.6.1.4.1.2636.3.1.15.1.1.2.4.0.0 {2} 2 [3] .1.3.6.1.4.1.2636.3.1.15.1.2.2.4.0.0 {2} 4 [4] .1.3.6.1.4.1.2636.3.1.15.1.3.2.4.0.0 {2} 0 [5] .1.3.6.1.4.1.2636.3.1.15.1.4.2.4.0.0 {2} 0 [6] .1.3.6.1.4.1.2636.3.1.15.1.5.2.4.0.0 {4} PEM 3 [7] .1.3.6.1.4.1.2636.3.1.15.1.6.2.4.0.0 {2} 7 [8] .1.3.6.1.4.1.2636.3.1.15.1.7.2.4.0.0 {2} 4 [9] .1.3.6.1.6.3.18.1.3.0 {7} 12.123.1.240
@@ -273,7 +272,7 @@ def post_dmaap():
     k = 0
     dmaap_pub_success = False
 
-    while not dmaap_pub_success and k < (int(tds.c_config['publisher.http_retries'])):
+    while not dmaap_pub_success and k < (int(tds.c_config['publisher']['http_retries'])):
         try:
             if tds.c_config['streams_publishes']['sec_fault_unsecure']['aaf_username'] == "" or tds.c_config['streams_publishes']['sec_fault_unsecure']['aaf_username'] is None:
                 msg = "%d trap(s) : %s - attempt %d (unsecure)" % (
@@ -323,7 +322,7 @@ def post_dmaap():
 
         k += 1
 
-        if k < tds.c_config['publisher.http_retries']:
+        if k < tds.c_config['publisher']['http_retries']:
             msg = "sleeping %.4f seconds and retrying" % (
                 tds.seconds_between_retries)
             ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_DETAILED,
@@ -380,9 +379,40 @@ def snmp_engine_observer_cb(snmp_engine, execpoint, variables, cbCtx):
     :Variables:
     """
 
+    # All sorts of goodies available:
+    # print('Execution point: %s' % execpoint)
+    # print('* transportDomain: %s' % '.'.join([str(x) for x in variables['transportDomain']]))
+    # print('* transportAddress: %s' % '@'.join([str(x) for x in variables['transportAddress']]))
+    # print('* securityModel: %s' % variables['securityModel'])
+    # print('* securityName: %s' % variables['securityName'])
+    # print('* securityLevel: %s' % variables['securityLevel'])
+    # print('* contextEngineId: %s' % variables['contextEngineId'].prettyPrint())
+    # print('* contextName: %s' % variables['contextName'].prettyPrint())
+    # print('* PDU: %s' % variables['pdu'].prettyPrint())
+    # V1 only:
+    # print('* enterprise: %s' % variables['pdu']['enterprise'].prettyPrint())
+    # V1 name (e.g. coldstart, warmstart):
+    # print('* generic: %s' % variables['pdu']['generic-trap'].prettyPrint())
+    # print('* generic: %d' % variables['pdu']['generic-trap'])
+    # print('* specific: %s' % variables['pdu']['specific-trap'].prettyPrint())
+    # print('* specific: %d' % variables['pdu']['specific-trap'])
+
     # init dictionary on new trap
     tds.trap_dict = {}
 
+    # FMDL.CHECK_WITH_DOWNSTREAM_CONSUMERS: get rid of round for millisecond val
+    # epoch_second = int(round(time.time()))
+    epoch_msecond = time.time()
+    epoch_second = int(round(epoch_msecond))
+    if epoch_second == tds.last_epoch_second:
+        tds.traps_in_epoch += 1
+    else:
+        tds.traps_in_epoch = 0
+    tds.last_epoch_second = epoch_second
+    traps_in_epoch_04d = format(tds.traps_in_epoch, '04d')
+    tds.trap_dict['epoch_serno'] = int(
+        (str(epoch_second) + str(traps_in_epoch_04d)))
+
     # assign uuid to trap
     tds.trap_dict["uuid"] = str(uuid_mod.uuid1())
 
@@ -413,7 +443,7 @@ def snmp_engine_observer_cb(snmp_engine, execpoint, variables, cbCtx):
 
         tds.dns_cache_ip_to_name[ip_addr_str] = agent_fqdn
         tds.dns_cache_ip_expires[ip_addr_str] = (
-            time.time() + tds.c_config['cache.dns_cache_ttl_seconds'])
+            time.time() + tds.c_config['cache']['dns_cache_ttl_seconds'])
         msg = "cache for %s (%s) updated - set to expire at %d" % \
             (agent_fqdn, ip_addr_str, tds.dns_cache_ip_expires[ip_addr_str])
         ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_DETAILED,
@@ -424,40 +454,35 @@ def snmp_engine_observer_cb(snmp_engine, execpoint, variables, cbCtx):
     tds.trap_dict["community"] = ""
     tds.trap_dict["community len"] = 0
 
-    # FMDL.CHECK_WITH_DOWNSTREAM_CONSUMERS: get rid of round for millisecond val
-    # epoch_second = int(round(time.time()))
-    epoch_msecond = time.time()
-    epoch_second = int(round(epoch_msecond))
-    if epoch_second == tds.last_epoch_second:
-        tds.traps_in_epoch += 1
-    else:
-        tds.traps_in_epoch = 0
-    tds.last_epoch_second = epoch_second
-    traps_in_epoch_04d = format(tds.traps_in_epoch, '04d')
-    tds.trap_dict['epoch_serno'] = int(
-        (str(epoch_second) + str(traps_in_epoch_04d)))
-
     snmp_version = variables['securityModel']
     if snmp_version == 1:
         tds.trap_dict["protocol version"] = "v1"
+        enterprise = variables['pdu']['enterprise'].prettyPrint()
+        generic_trap = variables['pdu']['generic-trap']
+        specific_trap = variables['pdu']['specific-trap']
+        if generic_trap < 6:
+            tds.trap_dict["notify OID"] = str(enterprise) + "." + str(specific_trap)
+        else:
+            tds.trap_dict["notify OID"] = str(enterprise) + ".0." + str(specific_trap)
+        tds.trap_dict["notify OID len"] = (tds.trap_dict["notify OID"].count('.') + 1)
+        tds.trap_dict["sysUptime"] = variables['pdu']['time-stamp'].prettyPrint()
     else:
         if snmp_version == 2:
             tds.trap_dict["protocol version"] = "v2c"
         else:
             if snmp_version == 3:
                 tds.trap_dict["protocol version"] = "v3"
+                tds.trap_dict["security level"] = str(variables['securityLevel'])
+                tds.trap_dict["context name"] = str(
+                    variables['contextName'].prettyPrint())
+                tds.trap_dict["security name"] = str(variables['securityName'])
+                tds.trap_dict["security engine"] = str(
+                    variables['contextEngineId'].prettyPrint())
             else:
                 tds.trap_dict["protocol version"] = "unknown"
 
-    if snmp_version == 3:
-        tds.trap_dict["protocol version"] = "v3"
-        tds.trap_dict["security level"] = str(variables['securityLevel'])
-        tds.trap_dict["context name"] = str(
-            variables['contextName'].prettyPrint())
-        tds.trap_dict["security name"] = str(variables['securityName'])
-        tds.trap_dict["security engine"] = str(
-            variables['contextEngineId'].prettyPrint())
-    tds.trap_dict['time received'] = epoch_msecond
+    # tds.trap_dict['time received'] = epoch_msecond
+    tds.trap_dict['time received'] = epoch_second
     tds.trap_dict['trap category'] = (
         tds.c_config['streams_publishes']['sec_fault_unsecure']['dmaap_info']['topic_url']).split('/')[-1]
 
@@ -486,7 +511,7 @@ def add_varbind_to_json(vb_idx, vb_oid, vb_type, vb_val):
 
     _individual_vb_dict = {}
 
-    if tds.trap_dict["protocol version"] == "v2c":
+    if tds.trap_dict["protocol version"] == "v2c" or tds.trap_dict["protocol version"] == "v3":
         # if v2c and first 2 varbinds, special handling required - e.g. put
         # in trap_dict, not vb_json_str
         if vb_idx == 0:
@@ -537,10 +562,18 @@ def notif_receiver_cb(snmp_engine, stateReference, contextEngineId, contextName,
       callback trap arrival
     :Variables:
     """
-
     msg = "processing varbinds for %s" % (tds.trap_dict["uuid"])
     ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_DETAILED, tds.CODE_GENERAL, msg)
 
+    # help(snmp_engine)
+    # print(snmp_engine)
+    # help(varBinds)
+    # print(varBinds)
+    # help(cbCtx)
+    # print(cbCtx)
+    # for key, val in cbCtx:
+    #     print(key, val)
+
     # FMDL update reset location when batching publishes
     vb_idx = 0
 
@@ -573,6 +606,9 @@ def notif_receiver_cb(snmp_engine, stateReference, contextEngineId, contextName,
     msg = "trap %s : %s" % (tds.trap_dict["uuid"], curr_trap_json_str)
     ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_DETAILED, tds.CODE_GENERAL, msg)
 
+    # always log arriving traps
+    log_all_arriving_traps()
+
     # now have a complete json message for this trap in "curr_trap_json_str"
     tds.traps_since_last_publish += 1
     milliseconds_since_last_publish = (time.time() - tds.last_pub_time) * 1000
@@ -588,17 +624,14 @@ def notif_receiver_cb(snmp_engine, stateReference, contextEngineId, contextName,
             ', ' + tds.trap_dict["uuid"]
         tds.all_traps_str = tds.all_traps_str + ', ' + curr_trap_json_str
 
-    # always log arriving traps
-    log_all_arriving_traps()
-
     # publish to dmaap after last varbind is processed
-    if tds.traps_since_last_publish >= tds.c_config['publisher.max_traps_between_publishes']:
+    if tds.traps_since_last_publish >= tds.c_config['publisher']['max_traps_between_publishes']:
         msg = "num traps since last publish (%d) exceeds threshold (%d) - publish traps" % (
-            tds.traps_since_last_publish, tds.c_config['publisher.max_traps_between_publishes'])
+            tds.traps_since_last_publish, tds.c_config['publisher']['max_traps_between_publishes'])
         ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_DETAILED,
                      tds.CODE_GENERAL, msg)
         post_dmaap()
-    elif milliseconds_since_last_publish >= tds.c_config['publisher.max_milliseconds_between_publishes']:
+    elif milliseconds_since_last_publish >= tds.c_config['publisher']['max_milliseconds_between_publishes']:
         msg = "num milliseconds since last publish (%.0f) exceeds threshold - publish traps" % milliseconds_since_last_publish
         ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_DETAILED,
                      tds.CODE_GENERAL, msg)
@@ -633,13 +666,17 @@ if __name__ == "__main__":
     # init vars
     tds.init()
     
+    # FMDL: add with stormWatch
+    # init sw vars
+    # sw.init()
+    
     # Set initial startup hour for rolling logfile
     tds.last_hour = datetime.datetime.now().hour
     
     # get config binding service (CBS) values (either broker, or json file override)
     load_all_configs(0, 0)
     msg = "%s : %s version %s starting" % (
-        prog_name, tds.c_config['snmptrap.title'], tds.c_config['snmptrap.version'])
+        prog_name, tds.c_config['snmptrapd']['title'], tds.c_config['snmptrapd']['version'])
     stdout_logger(msg)
     
     # open various ecomp logs
@@ -650,18 +687,21 @@ if __name__ == "__main__":
         msg = "WARNING:  '-v' argument present.  All diagnostic messages will be logged.  This can slow things down, use only when needed."
         tds.minimum_severity_to_log = 0
         stdout_logger(msg)
+        # use specific flags or 'all' for full debugging
+        help(debug.setLogger)
+        debug.setLogger(debug.Debug('dsp', 'msgproc'))
     
     # name and open arriving trap log
-    tds.arriving_traps_filename = tds.c_config['files.runtime_base_dir'] + "/" + \
-        tds.c_config['files.log_dir'] + "/" + \
-        (tds.c_config['files.arriving_traps_log'])
+    tds.arriving_traps_filename = tds.c_config['files']['runtime_base_dir'] + "/" + \
+        tds.c_config['files']['log_dir'] + "/" + \
+        (tds.c_config['files']['arriving_traps_log'])
     tds.arriving_traps_fd = open_file(tds.arriving_traps_filename)
     msg = ("arriving traps logged to: %s" % tds.arriving_traps_filename)
     stdout_logger(msg)
     ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg)
     
     # name and open json trap log
-    tds.json_traps_filename = tds.c_config['files.runtime_base_dir'] + "/" + tds.c_config['files.log_dir'] + "/" + "DMAAP_" + (
+    tds.json_traps_filename = tds.c_config['files']['runtime_base_dir'] + "/" + tds.c_config['files']['log_dir'] + "/" + "DMAAP_" + (
         tds.c_config['streams_publishes']['sec_fault_unsecure']['dmaap_info']['topic_url'].split('/')[-1]) + ".json"
     tds.json_traps_fd = open_file(tds.json_traps_filename)
     msg = ("published traps logged to: %s" % tds.json_traps_filename)
@@ -672,8 +712,8 @@ if __name__ == "__main__":
     signal.signal(signal.SIGUSR1, load_all_configs)
     
     # save current PID for future/external reference
-    tds.pid_file_name = tds.c_config['files.runtime_base_dir'] + \
-        '/' + tds.c_config['files.pid_dir'] + '/' + prog_name + ".pid"
+    tds.pid_file_name = tds.c_config['files']['runtime_base_dir'] + \
+        '/' + tds.c_config['files']['pid_dir'] + '/' + prog_name + ".pid"
     msg = "Runtime PID file: %s" % tds.pid_file_name
     ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg)
     rc = save_pid(tds.pid_file_name)
@@ -681,7 +721,7 @@ if __name__ == "__main__":
     # Get the event loop for this thread
     loop = asyncio.get_event_loop()
     
-    # Create SNMP engine with autogenernated engineID pre-bound
+    # Create SNMP engine with autogenerated engineID pre-bound
     # to socket transport dispatcher
     snmp_engine = engine.SnmpEngine()
     
@@ -690,12 +730,17 @@ if __name__ == "__main__":
     # # # # # # # # # # # #
     
     # UDP over IPv4
-    # FMDL:  add check for presense of ipv4_interface prior to attempting add OR just put entire thing in try/except clause
     try:
-        ipv4_interface = tds.c_config['protocols.ipv4_interface']
-        ipv4_port = tds.c_config['protocols.ipv4_port']
+        ipv4_interface = tds.c_config['protocols']['ipv4_interface']
+        ipv4_port = tds.c_config['protocols']['ipv4_port']
     
         try:
+            # FIXME:  this doesn't appear to throw an exception even if
+            #         the userID is unable (perms) to bind to port
+            #
+            #         We may need to open raw port using other 
+            #         means to confirm proper privileges (then
+            #         close it and reopen w/ pysnmp api)
             config.addTransport(
                 snmp_engine,
                 udp.domainName + (1,),
@@ -716,10 +761,9 @@ if __name__ == "__main__":
     
     
     # UDP over IPv6
-    # FMDL:  add check for presense of ipv6_interface prior to attempting add OR just put entire thing in try/except clause
     try:
-        ipv6_interface = tds.c_config['protocols.ipv6_interface']
-        ipv6_port = tds.c_config['protocols.ipv6_port']
+        ipv6_interface = tds.c_config['protocols']['ipv6_interface']
+        ipv6_port = tds.c_config['protocols']['ipv6_port']
     
         try:
             config.addTransport(
@@ -755,7 +799,12 @@ if __name__ == "__main__":
         comm_string_rewrite_observer,
         'rfc2576.processIncomingMsg:writable'
     )
-    
+
+    # # # # # # # # # # # #
+    # SNMPv3 setup
+    # # # # # # # # # # # #
+    config, snmp_engine=load_snmpv3_credentials(config, snmp_engine, tds.c_config) 
+
     # register snmp_engine_observer_cb for message arrival
     snmp_engine.observer.registerObserver(
         snmp_engine_observer_cb,
diff --git a/snmptrap/snmptrapd.sh b/snmptrap/snmptrapd.sh
index e8ef93e..57fd1b4 100644..100755
--- a/snmptrap/snmptrapd.sh
+++ b/snmptrap/snmptrapd.sh
@@ -21,18 +21,28 @@
 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
 #
 
-# get to where we are supposed to be for startup
-cd /opt/app/snmptrap/bin
+# basics
+current_cmd=`basename $0`
+current_module=`echo $current_cmd | cut -d"." -f1`
+
+# FMDL:: need to pick up these values from json config, but it isn't
+#        present at startup
+base_dir=/opt/app/snmptrap
+pid_file=${base_dir}/tmp/${current_module}.py.pid
+start_dir=${base_dir}/bin
 
 # include path to 3.6+ version of python that has required dependencies included
 export PATH=/opt/app/python-3.6.1/bin:$PATH
 
-# expand search for python modules to include ./mod in runtime dir
-export PYTHONPATH=./mod:./:$PYTHONPATH
-
 # set location of SSL certificates
 export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-bundle.crt
 
+# get to where we are supposed to be for startup
+cd /opt/app/snmptrap/bin
+
+# expand search for python modules to include ./mod in current/runtime dir
+export PYTHONPATH=./mod:./:$PYTHONPATH
+
 # PYTHONUNBUFFERED:
 #    set PYTHONUNBUFFERED to a non-empty string to avoid output buffering; 
 #    comment out for runtime environments/better performance!
@@ -42,14 +52,191 @@ export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-bundle.crt
 #
 export CBS_SIM_JSON=../etc/snmptrapd.json
 
-# want tracing?  Use this:
-# python -m trace --trackcalls snmptrapd.py -v
+# # # # # # # # # # 
+# log_msg - log messages to stdout in standard manner
+# # # # # # # # # # 
+log_msg()
+{
+   msg=$*
+
+   printf "`date +%Y-%m-%dT%H:%M:%S,%N | cut -c1-23` ${msg}"
+}
+
+# # # # # # # # # # 
+# Start the service
+# # # # # # # # # # 
+start_service()
+{
+   # Hints for startup modifications:
+   # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
+   # standard startup?  Use this:
+   cmd="python ./snmptrapd.py &"
+   # want tracing?  Use this:
+   #     "python -m trace --trackcalls snmptrapd.py &"
+   # unbuffered io for logs? Use this:
+   #     "python -u snmptrapd.py &"
+
+   cd ${start_dir}
+
+   # check for process already running
+   if [ -r ${pid_file} ]
+   then
+      pid=$(cat ${pid_file})
+      if ps -p ${pid} > /dev/null
+      then
+         printf "${current_module} already running - PID ${pid}\n"
+         return 0
+      fi
+   fi
+
+   # FMDL:: do this in snmptrapd.py at startup
+   # roll log if present at startup
+   # if [ -f ${LOGFILE} ]
+   # then
+      # mv -f ${LOGFILE} ${LOGFILE}.`date +%h-%m-%Y_%H:%M:%S`
+   # fi
+
+   log_msg "Starting ${current_module}...  "
+   eval ${cmd}
+   return_code=$?
+
+   if [ ${return_code} -eq 0 ]
+   then
+       log_msg "Started.\n"
+   else
+       log_msg "\nERROR!  Unable to start ${current_module}.  Check logs for details.\n"
+   fi
+
+   return ${return_code}
+
+}
+
+# # # # # # # # # # 
+# Stop the service
+# # # # # # # # # # 
+stop_service()
+{
+    if [ ! -r ${pid_file} ]
+    then
+        log_msg "PID file ${pid_file} does not exist or not readable - unable to stop specific instance of ${current_module}.\n"
+        log_msg "Diagnose further at command line as needed.\n"
+        return_code=1
+    else
+        pid=$(cat ${pid_file})
+        log_msg "Stopping ${current_module} PID ${pid}...\n"
+        kill ${pid}
+        if [ $? -ne 0 ]
+        then
+            log_msg "\nERROR while trying to terminate ${current_module} PID ${pid} (is it not running or owned by another userID?)"
+            log_msg "\nDiagnose further at command line as needed."
+            return_code=$?
+            if [ -w ${pid_file} ]
+            then
+                rm -f ${pid_file}
+            fi
+        else
+            log_msg "Stopped\n"
+            if [ -w ${pid_file} ]
+            then
+                rm -f ${pid_file}
+            fi
+            return_code=0
+        fi
+    fi
+
+    return ${return_code}
+}
+
+# # # # # # # # # # # # # # #
+# Check status of the service
+# # # # # # # # # # # # # # #
+status_service()
+{
+    if [ -r ${pid_file} ]
+    then
+        pid=$(cat ${pid_file})
+        pgrep -a python | grep ${current_module} | grep "^${pid}" > /dev/null
+        return_code=$?
+
+        if [ ${return_code} -eq 0 ]
+        then
+            log_msg "Status: ${current_module} running\n"
+            ps -p ${pid} -f | grep -v PID
+            return_code=0
+        else
+            log_msg "Status: ERROR! ${current_module} not running.\n"
+            return_code=1
+        fi
+   else
+        log_msg "PID file ${pid_file} does not exist or not readable - unable to check status of ${current_module}\n"
+        log_msg "Diagnose further at command line as needed.\n"
+        return 1
+    fi
+
+    return ${return_code}
+}
+
+# # # # # # # # # # # # # # # # #
+# Signal process to reload config
+# # # # # # # # # # # # # # # # #
+reload_cfg()
+{
+    if [ -r ${pid_file} ]
+    then
+       pid=$(cat ${pid_file})
+       ps -p ${pid} > /dev/null 2>&1
+       ret=$?
+       if [ ${ret} ]
+       then
+          log_msg "Signaling ${current_module} PID ${pid} to request/read updated configs...\n"
+          kill -USR1 ${pid}
+          return_code=$?
+          if [ ${return_code} -eq 0 ]
+          then
+              log_msg "...Signal complete.\n"
+          else
+              log_msg "\nERROR signaling ${current_module} - diagnose further at the command line.\n"
+          fi
+       else
+          log_msg "\nERROR: ${current_module} PID ${pid} does not appear to be running.\n"
+          return_code=1
+       fi
+    else
+       log_msg "\nERROR: ${current_module} pid_file ${pid_file} does not exist - unable to signal for config re-read.\n"
+       return_code=1
+    fi
 
-# want verbose logging?  Use this:
-# python snmptrapd.py -v
+    return ${return_code}
+}
 
-# standard startup?  Use this:
-# python snmptrapd.py
+# # # # # # # # # # # # #
+# M A I N
+# # # # # # # # # # # # #
 
-# unbuffered io for logs and verbose logging? Use this:
-python -u snmptrapd.py -v
+case "$1" in
+   "start")
+          start_service
+          exit $?
+          ;;
+   "stop") 
+          stop_service
+          exit $?
+          ;;
+   "restart")
+          stop_service
+          sleep 1
+          start_service
+          exit $?
+          ;;
+   "status") 
+          status_service
+          exit $?
+          ;;
+   "reloadCfg")
+          reload_cfg
+          exit $?
+          ;;
+   *)
+          printf "\nUsage: ${current_cmd} {start|stop|restart|status|rollLog|reloadCfg}\n"
+          exit 1
+   esac