From 678e65fa7938114bf7d66f212b16cef8633db214 Mon Sep 17 00:00:00 2001 From: Gokul Singaraju Date: Mon, 21 May 2018 15:32:52 -0400 Subject: Miss HB CBS integration Issue-ID: DCAEGEN2-279 Change-Id: I3f78e0870d4c620a304fb8d218f1d30187b4fbef Signed-off-by: Gokul Singaraju --- Dockerfile | 49 +++- bin/check_health.py | 66 ----- bin/run.py | 89 ------ etc/config.json | 26 ++ etc/config.yaml | 16 ++ miss_htbt_service.egg-info/SOURCES.txt | 15 +- miss_htbt_service.egg-info/top_level.txt | 1 + miss_htbt_service/__init__.py | 28 +- miss_htbt_service/check_health.py | 66 +++++ miss_htbt_service/get_logger.py | 40 +++ miss_htbt_service/htbtworker.py | 149 +++------- miss_htbt_service/misshtbt.sh | 56 ++++ miss_htbt_service/misshtbtd.py | 125 ++++++++ miss_htbt_service/mod/__init__.py | 21 ++ miss_htbt_service/mod/trapd_exit.py | 93 ++++++ miss_htbt_service/mod/trapd_get_cbs_config.py | 120 ++++++++ miss_htbt_service/mod/trapd_http_session.py | 58 ++++ miss_htbt_service/mod/trapd_io.py | 392 ++++++++++++++++++++++++++ miss_htbt_service/mod/trapd_runtime_pid.py | 94 ++++++ miss_htbt_service/mod/trapd_settings.py | 169 +++++++++++ mvn-phase-script.sh | 2 +- pom.xml | 2 +- requirements.txt | 3 +- setup.py | 18 +- tests/test_binding.py | 45 ++- tests/test_trapd_exit.py | 37 +++ tests/test_trapd_get_cbs_config.py | 70 +++++ tests/test_trapd_http_session.py | 20 ++ tests/test_trapd_runtime_pid.py | 49 ++++ tests/test_trapd_settings.py | 72 +++++ tox.ini | 13 +- 31 files changed, 1674 insertions(+), 330 deletions(-) delete mode 100755 bin/check_health.py delete mode 100755 bin/run.py create mode 100644 etc/config.json create mode 100644 etc/config.yaml create mode 100755 miss_htbt_service/check_health.py create mode 100755 miss_htbt_service/get_logger.py create mode 100755 miss_htbt_service/misshtbt.sh create mode 100755 miss_htbt_service/misshtbtd.py create mode 100755 miss_htbt_service/mod/__init__.py create mode 100755 miss_htbt_service/mod/trapd_exit.py create mode 100755 miss_htbt_service/mod/trapd_get_cbs_config.py create mode 100755 miss_htbt_service/mod/trapd_http_session.py create mode 100755 miss_htbt_service/mod/trapd_io.py create mode 100755 miss_htbt_service/mod/trapd_runtime_pid.py create mode 100755 miss_htbt_service/mod/trapd_settings.py create mode 100644 tests/test_trapd_exit.py create mode 100644 tests/test_trapd_get_cbs_config.py create mode 100644 tests/test_trapd_http_session.py create mode 100644 tests/test_trapd_runtime_pid.py create mode 100644 tests/test_trapd_settings.py diff --git a/Dockerfile b/Dockerfile index cb4bc48..b489714 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,53 @@ FROM python:3.6 MAINTAINER gs244f@att.com -ADD . /tmp +ENV INSROOT /opt/app +ENV APPUSER misshtbt +ENV APPDIR ${INSROOT}/${APPUSER} + +RUN useradd -d ${APPDIR} ${APPUSER} + +WORKDIR ${APPDIR} + +#ADD . /tmp +#RUN mkdir /tmp/config + +EXPOSE 10001 + +COPY ./miss_htbt_service/ ./bin/ +COPY ./etc/ ./etc/ +COPY requirements.txt ./ +COPY setup.py ./ #need pip > 8 to have internal pypi repo in requirements.txt RUN pip install --upgrade pip #do the install -WORKDIR /tmp +#WORKDIR /tmp RUN pip install pyyaml --upgrade RUN pip install -r requirements.txt RUN pip install -e . -RUN mkdir /tmp/config -#RUN echo 1.2.3.4 > /tmp/config/coll_ip.txt -#RUN echo 1234 > /tmp/config/coll_port.txt -#RUN echo 4.5.6.7 > /tmp/config/pol_ip.txt -#RUN echo 4567 > /tmp/config/pol_port.txt -EXPOSE 10001 +RUN mkdir -p ${APPDIR}/data \ + && mkdir -p ${APPDIR}/logs \ + && mkdir -p ${APPDIR}/tmp \ + && chown -R ${APPUSER}:${APPUSER} ${APPDIR} \ + && chmod a+w ${APPDIR}/data \ + && chmod a+w ${APPDIR}/logs \ + && chmod a+w ${APPDIR}/tmp \ + && chmod 500 ${APPDIR}/etc \ + && chmod 500 ${APPDIR}/bin/*.py \ + && chmod 500 ${APPDIR}/bin/*.sh \ + && chmod 500 ${APPDIR}/bin/*/*.py + +USER ${APPUSER} +VOLUME ${APPDIR}/logs + +CMD ["./bin/misshtbt.sh"] #ENV PYTHONPATH="/usr/local/lib/python3.6:/usr/local/lib/python3.6/site-packages:${PATH}" #ENV PYTHONPATH="/usr/local/lib/python3.6/site-packages:/usr/local/lib/python3.6" -ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages -#CMD run.py +#ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages:. #ENTRYPOINT ["/bin/python", "./bin/run.py"] #ENTRYPOINT ["/usr/bin/python","./bin/run.py" ] -ENTRYPOINT ["/usr/local/bin/python","./bin/run.py" ] -#ENTRYPOINT ["/bin/ls","-lR", "/usr/local"] +#ENTRYPOINT ["/usr/local/bin/python","./bin/misshtbtd.py" ] +#ENTRYPOINT ["/bin/ls","-lR", "."] diff --git a/bin/check_health.py b/bin/check_health.py deleted file mode 100755 index ae61881..0000000 --- a/bin/check_health.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# ============LICENSE_START======================================================= -# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= -# -# ECOMP is a trademark and service mark of AT&T Intellectual Property. -from http.server import HTTPServer, BaseHTTPRequestHandler -from urllib import parse - -#from BaseHTTPServer import BaseHTTPRequestHandler -#import urlparse -import json - -class GetHandler(BaseHTTPRequestHandler): - - def do_GET(self): - parsed_path = parse.urlparse(self.path) - message = '\n'.join([ - 'CLIENT VALUES:', - 'client_address=%s (%s)' % (self.client_address, self.address_string()), - 'command=%s' % self.command, - 'path=%s' % self.path, - 'real path=%s' % parsed_path.path, - 'query=%s' % parsed_path.query, - 'request_version=%s' % self.request_version, - '', - 'SERVER VALUES:', - 'server_version=%s' % self.server_version, - 'sys_version=%s' % self.sys_version, - 'protocol_version=%s' % self.protocol_version, - '', - ]) - self.send_response(200) - self.end_headers() - self.wfile.write(message) - return - - def do_POST(self): - content_len = int(self.headers.getheader('content-length')) - post_body = self.rfile.read(content_len) - self.send_response(200) - self.end_headers() - - data = json.loads(post_body) - - self.wfile.write(data['health']) - return - -if __name__ == '__main__': - from http.server import HTTPServer - #from BaseHTTPServer import HTTPServer - server = HTTPServer(("", 10001), GetHandler) - print('Starting server at http://localhost:10001') - server.serve_forever() diff --git a/bin/run.py b/bin/run.py deleted file mode 100755 index 311b44c..0000000 --- a/bin/run.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 - -# ============LICENSE_START======================================================= -# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= -# -# ECOMP is a trademark and service mark of AT&T Intellectual Property. -# -# Author Gokul Singaraju gs244f@att.com -# - -import os -import sys -import yaml -import multiprocessing -import logging -import subprocess -from miss_htbt_service import get_logger - -mr_url = 'http://mrrouter.onap.org:3904' -pol_url = 'http://mrrouter.onap.org:3904' -intopic = 'VESCOLL-VNFNJ-SECHEARTBEAT-OUTPUT' -outopic = 'POLICY-HILOTCA-EVENT-OUTPUT' - -#Checks heartbeat by calling worker thread -def checkhtbt(mr_url, intopic, pol_url, outopic, misshtbt,intvl): - print('Doing some work',mr_url, misshtbt,intvl,intopic,outopic) - subprocess.call(["/usr/bin/python","./miss_htbt_service/htbtworker.py" , mr_url , intopic, pol_url, outopic, str(misshtbt) , str(intvl) ]) - sys.stdout.flush() - return - -_logger = get_logger(__name__) - -#main functon which reads yaml config and invokes heartbeat -#monitoring -if __name__ == '__main__': - try: - print("Heartbeat Microservice ...") - if "INURL" in os.environ.keys(): - mr_url = os.environ['INURL'] - if "INTOPIC" in os.environ.keys(): - intopic = os.environ['INTOPIC'] - if "OUTURL" in os.environ.keys(): - pol_url = os.environ['OUTURL'] - if "OUTOPIC" in os.environ.keys(): - outopic = os.environ['OUTOPIC'] - print(outopic) - multiprocessing.log_to_stderr() - logger = multiprocessing.get_logger() - logger.setLevel(logging.INFO) - my_env = os.environ.copy() - my_env["PYTHONPATH"] = my_env["PYTHONPATH"]+"/usr/local/lib/python3.6:" - p = subprocess.Popen(['./bin/check_health.py'],stdout=subprocess.PIPE,env=my_env) - #print(p.communicate()) - with open("./miss_htbt_service/config/config.yaml", 'r') as ymlfile: - cfg = yaml.load(ymlfile) - # Put some initial values into the queue - for section in cfg: - print(section) - #print(cfg['global']) - #print(cfg['global']['message_router_url']) - jobs = [] - for vnf in (cfg['vnfs']): - print(cfg['vnfs'][vnf]) - #print(cfg['vnfs'][vnf][0]) - #print(cfg['vnfs'][vnf][1]) - #print(cfg['vnfs'][vnf][2]) - #Start Heartbeat monitoring process worker thread on VNFs configured - logger.info("Starting threads...") - p = multiprocessing.Process(target=checkhtbt, args=( mr_url, intopic, pol_url, outopic, cfg['vnfs'][vnf][0],cfg['vnfs'][vnf][1])) - jobs.append(p) - p.start() - for j in jobs: - j.join() - print('%s.exitcode = %s' % (j.name, j.exitcode)) - except Exception as e: - _logger.error("Fatal error. Could not start missing heartbeat service due to: {0}".format(e)) diff --git a/etc/config.json b/etc/config.json new file mode 100644 index 0000000..3f6e487 --- /dev/null +++ b/etc/config.json @@ -0,0 +1,26 @@ +{ + "heartbeat_config": { + "vnfs":[ + { "nfNamingCode": "VNFA", + "heartbeatcountmissed":3, "heartbeatinterval": 60, "closedLoopControlName":"ControlLoopEvent1"}, + { "nfNamingCode": "VNFB", + "heartbeatcountmissed":3, "heartbeatinterval": 60, "closedLoopControlName":"ControlLoopEvent1"}, + { "nfNamingCode": "VNFC", + "heartbeatcountmissed":3, "heartbeatinterval": 60, "closedLoopControlName":"ControlLoopEvent1"} + ] + }, + + "streams_publishes": { + "ves_heartbeat": { + "dmaap_info": {"topic_url": "http://message-router:3904/events/unauthenticated.DCAE_CL_OUTPUT/"}, + "type": "message_router" + } + }, + "streams_subscribes": { + "ves_heartbeat": { + "dmaap_info": {"topic_url": "http://message-router:3904/events/unauthenticated.SEC_HEARTBEAT_INPUT/"}, + "type": "message_router" + } + } +} + diff --git a/etc/config.yaml b/etc/config.yaml new file mode 100644 index 0000000..0dcc8bf --- /dev/null +++ b/etc/config.yaml @@ -0,0 +1,16 @@ +global: + host: localhost + message_router_url: http://msgrouter.att.com:3904 +# Missing heartbeats +# Heartbeat interval +# Input topic +# Output topic +# ClosedLoopControlName +vnfs: + vnfa: + - 3 + - 60 + - VESCOLL-VNFNJ-SECHEARTBEAT-OUTPUT + - DCAE-POLICY-HILOTCA-EVENT-OUTPUT + - ControlLoopEvent1 + diff --git a/miss_htbt_service.egg-info/SOURCES.txt b/miss_htbt_service.egg-info/SOURCES.txt index 3cceb86..d635623 100644 --- a/miss_htbt_service.egg-info/SOURCES.txt +++ b/miss_htbt_service.egg-info/SOURCES.txt @@ -1,9 +1,22 @@ README.md setup.py miss_htbt_service/__init__.py +miss_htbt_service/check_health.py +miss_htbt_service/get_logger.py miss_htbt_service/htbtworker.py +miss_htbt_service/misshtbtd.py miss_htbt_service.egg-info/PKG-INFO miss_htbt_service.egg-info/SOURCES.txt miss_htbt_service.egg-info/dependency_links.txt miss_htbt_service.egg-info/not-zip-safe -miss_htbt_service.egg-info/top_level.txt \ No newline at end of file +miss_htbt_service.egg-info/requires.txt +miss_htbt_service.egg-info/top_level.txt +miss_htbt_service/mod/__init__.py +miss_htbt_service/mod/trapd_exit.py +miss_htbt_service/mod/trapd_get_cbs_config.py +miss_htbt_service/mod/trapd_http_session.py +miss_htbt_service/mod/trapd_io.py +miss_htbt_service/mod/trapd_runtime_pid.py +miss_htbt_service/mod/trapd_settings.py +tests/__init__.py +tests/test_binding.py \ No newline at end of file diff --git a/miss_htbt_service.egg-info/top_level.txt b/miss_htbt_service.egg-info/top_level.txt index d08666e..9bad856 100644 --- a/miss_htbt_service.egg-info/top_level.txt +++ b/miss_htbt_service.egg-info/top_level.txt @@ -1 +1,2 @@ miss_htbt_service +tests diff --git a/miss_htbt_service/__init__.py b/miss_htbt_service/__init__.py index a18ed5c..8da7cd3 100644 --- a/miss_htbt_service/__init__.py +++ b/miss_htbt_service/__init__.py @@ -1,11 +1,11 @@ -# ============LICENSE_START======================================================= -# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -16,25 +16,7 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. -import os -import logging - -'''Configures the module root logger''' -root = logging.getLogger() -if root.handlers: - root.handlers.clear() -formatter = logging.Formatter('%(asctime)s | %(name)s | %(module)s | %(funcName)s | %(lineno)d | %(levelname)s | %(message)s') -handler = logging.StreamHandler() -handler.setFormatter(formatter) -root.addHandler(handler) -root.setLevel("DEBUG") - -class BadEnviornmentENVNotFound(Exception): - pass - -def get_logger(module=None): - '''Returns a module-specific logger or global logger if the module is None''' - return root if module is None else root.getChild(module) - +# empty __init__.py so that pytest can add correct path to coverage report, -- per pytest +# best practice guideline diff --git a/miss_htbt_service/check_health.py b/miss_htbt_service/check_health.py new file mode 100755 index 0000000..ae61881 --- /dev/null +++ b/miss_htbt_service/check_health.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# ============LICENSE_START======================================================= +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib import parse + +#from BaseHTTPServer import BaseHTTPRequestHandler +#import urlparse +import json + +class GetHandler(BaseHTTPRequestHandler): + + def do_GET(self): + parsed_path = parse.urlparse(self.path) + message = '\n'.join([ + 'CLIENT VALUES:', + 'client_address=%s (%s)' % (self.client_address, self.address_string()), + 'command=%s' % self.command, + 'path=%s' % self.path, + 'real path=%s' % parsed_path.path, + 'query=%s' % parsed_path.query, + 'request_version=%s' % self.request_version, + '', + 'SERVER VALUES:', + 'server_version=%s' % self.server_version, + 'sys_version=%s' % self.sys_version, + 'protocol_version=%s' % self.protocol_version, + '', + ]) + self.send_response(200) + self.end_headers() + self.wfile.write(message) + return + + def do_POST(self): + content_len = int(self.headers.getheader('content-length')) + post_body = self.rfile.read(content_len) + self.send_response(200) + self.end_headers() + + data = json.loads(post_body) + + self.wfile.write(data['health']) + return + +if __name__ == '__main__': + from http.server import HTTPServer + #from BaseHTTPServer import HTTPServer + server = HTTPServer(("", 10001), GetHandler) + print('Starting server at http://localhost:10001') + server.serve_forever() diff --git a/miss_htbt_service/get_logger.py b/miss_htbt_service/get_logger.py new file mode 100755 index 0000000..a18ed5c --- /dev/null +++ b/miss_htbt_service/get_logger.py @@ -0,0 +1,40 @@ +# ============LICENSE_START======================================================= +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import os +import logging + +'''Configures the module root logger''' +root = logging.getLogger() +if root.handlers: + root.handlers.clear() +formatter = logging.Formatter('%(asctime)s | %(name)s | %(module)s | %(funcName)s | %(lineno)d | %(levelname)s | %(message)s') +handler = logging.StreamHandler() +handler.setFormatter(formatter) +root.addHandler(handler) +root.setLevel("DEBUG") + +class BadEnviornmentENVNotFound(Exception): + pass + +def get_logger(module=None): + '''Returns a module-specific logger or global logger if the module is None''' + return root if module is None else root.getChild(module) + + + diff --git a/miss_htbt_service/htbtworker.py b/miss_htbt_service/htbtworker.py index 3bad8c9..347dbd6 100644 --- a/miss_htbt_service/htbtworker.py +++ b/miss_htbt_service/htbtworker.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # Copyright 2017 AT&T Intellectual Property, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,6 +41,8 @@ mr_url = 'http://mrrouter.onap.org:3904' pol_url = 'http://mrrouter.onap.org:3904' intopic = 'VESCOLL-VNFNJ-SECHEARTBEAT-OUTPUT' outopic = 'POLICY-HILOTCA-EVENT-OUTPUT' +nfc = "vVNF" +cl_loop = 'ControlLoopEvent1' periodic_scheduler = None # Checks for heartbeat event on periodic basis @@ -58,56 +60,17 @@ class PeriodicScheduler(object): def stop(self): list(map(self.scheduler.cancel, self.scheduler.queue)) -# Formats collector uri from config files of heat template -def get_collector_uri(): - """ - This method waterfalls reads an envioronmental variable called COLLECTOR_HOST - If that doesn't work, it raises an Exception - """ - with open('/tmp/config/coll_ip.txt', 'r') as myfile: - coll_ip=myfile.read().replace('\n', '') - myfile.close() - with open('/tmp/config/coll_port.txt', 'r') as myfile2: - coll_port=myfile2.read().replace('\n', '') - myfile2.close() - if coll_ip and coll_port: - # WARNING! TODO! Currently the env file does not include the port. - # But some other people think that the port should be a part of that. - # For now, I'm hardcoding 8500 until this gets resolved. - return "http://{0}:{1}".format(coll_ip, coll_port) - else: - raise BadEnviornmentENVNotFound("COLLECTOR_HOST") - -# Formats Policy uri from config files of heat template -def get_policy_uri(): - """ - This method waterfalls reads an envioronmental variable called POLICY_HOST - If that doesn't work, it raises an Exception - """ - with open('/tmp/config/coll_ip.txt', 'r') as myfile: - pol_ip=myfile.read().replace('\n', '') - myfile.close() - with open('/tmp/config/coll_port.txt', 'r') as myfile2: - pol_port=myfile2.read().replace('\n', '') - myfile2.close() - if pol_ip and pol_port : - # WARNING! TODO! Currently the env file does not include the port. - # But some other people think that the port should be a part of that. - # For now, I'm hardcoding 8500 until this gets resolved. - return "http://{0}:{1}".format(pol_ip,pol_port) - else: - raise BadEnviornmentENVNotFound("POLICY_HOST") - # Process the heartbeat event on input topic def periodic_event(): global periodic_scheduler - global mr_url, pol_url, missing_htbt, intvl, intopic, outopic + global mr_url, pol_url, missing_htbt, intvl, intopic, outopic, nfc, cl_loop ret = 0 #print("Args are :", locals()) - print("Checking..." , datetime.datetime.now()) + print("{0} Checking...".format(datetime.datetime.now())) #Read heartbeat - get_url = mr_url+'/events/'+intopic+'/DefaultGroup/1?timeout=15000' + #get_url = mr_url+'/events/'+intopic+'/DefaultGroup/1?timeout=15000' + get_url = mr_url+'/DefaultGroup/1?timeout=15000' print("Getting :"+get_url) try: res = requests.get(get_url) @@ -124,6 +87,8 @@ def periodic_event(): print("Line:"+line) jobj = json.loads(line) #print(jobj) + if( nfc != jobj['event']['commonEventHeader']['nfNamingCode']) : + continue srcid = (jobj['event']['commonEventHeader']['sourceId']) lastepo = (jobj['event']['commonEventHeader']['lastEpochMicrosec']) seqnum = (jobj['event']['commonEventHeader']['sequence']) @@ -139,35 +104,19 @@ def periodic_event(): heartflag[srcid] = sdiff; heartmsg[srcid] = jobj; else: - payload = json.dumps({"event": { - "commonEventHeader": { - "reportingEntityName": "VNFVM", - "reportingEntityName": "VNFVM", - "startEpochMicrosec": 1508641592248000, - "lastEpochMicrosec": 1508641592248000, - "eventId": "VNFVM_heartbeat", - "sequence": 1, - "priority": "Normal", - "sourceName": "VNFVM", - "domain": "heartbeat", - "eventName": "Heartbeat_Vnf", - "internalHeaderFields": { - "closedLoopFlag": "True", - "eventTag": "hp.Heartbeat Service.20171022.8447964515", - "collectorTimeStamp": "Sun, 10 22 2017 03:04:27 GMT", - "lastDatetime": "Sun, 22 Oct 2017 03:06:32 +0000", - "closedLoopControlName": "ControlLoopEvent1", - "firstDatetime": "Sun, 22 Oct 2017 03:06:32 +0000" - }, - "reportingEntityId": "cff8656d-0b42-4eda-ab5d-3d2b7f2d74c8", - "version": 3, - "sourceId": "cff8656d-0b42-4eda-ab5d-3d2b7f2d74c8" - } - } - }) + jobj["internalHeaderFields"] = json.dumps({ + "closedLoopFlag": "True", + "eventTag": "hp.Heartbeat Service.20171022.8447964515", + "collectorTimeStamp": "Sun, 10 22 2017 03:04:27 GMT", + "lastDatetime": "Sun, 22 Oct 2017 03:06:32 +0000", + "closedLoopControlName": cl_loop, + "firstDatetime": "Sun, 22 Oct 2017 03:06:32 +0000" + }); + heartmsg[srcid] = jobj; payload = heartmsg[srcid] print(payload) - psend_url = pol_url+'/events/'+outopic+'/DefaultGroup/1?timeout=15000' + #psend_url = pol_url+'/events/'+outopic+'/DefaultGroup/1?timeout=15000' + psend_url = pol_url+'/DefaultGroup/1?timeout=15000' print(psend_url) print("Heartbeat Dead raising alarm event "+psend_url) #Send response for policy on output topic @@ -191,34 +140,17 @@ def periodic_event(): print("Heartbeat Dead raise alarm event"+key) chkeys.append( key ) #print payload - payload = json.dumps({"event": { - "commonEventHeader": { - "reportingEntityName": "VNFVM", - "startEpochMicrosec": 1508641592248000, - "lastEpochMicrosec": 1508641592248000, - "eventId": "VNFVM_heartbeat", - "sequence": 1, - "priority": "Normal", - "sourceName": "VNFVM", - "domain": "heartbeat", - "eventName": "Heartbeat_Vnf", - "internalHeaderFields": { - "closedLoopFlag": "True", - "eventTag": "hp.Heartbeat Service.20171022.8447964515", - "collectorTimeStamp": "Sun, 10 22 2017 03:04:27 GMT", - "lastDatetime": "Sun, 22 Oct 2017 03:06:32 +0000", - "closedLoopControlName": "ControlLoopEvent1", - "firstDatetime": "Sun, 22 Oct 2017 03:06:32 +0000" - }, - "reportingEntityId": "cff8656d-0b42-4eda-ab5d-3d2b7f2d74c8", - "version": 3, - "sourceId": "cff8656d-0b42-4eda-ab5d-3d2b7f2d74c8" - } - } - }) + heartmsg[key]["internalHeaderFields"] = json.dumps({ + "closedLoopFlag": "True", + "eventTag": "hp.Heartbeat Service.20171022.8447964515", + "collectorTimeStamp": "Sun, 10 22 2017 03:04:27 GMT", + "lastDatetime": "Sun, 22 Oct 2017 03:06:32 +0000", + "closedLoopControlName": cl_loop, + "firstDatetime": "Sun, 22 Oct 2017 03:06:32 +0000" + }) payload = heartmsg[key] print(payload) - send_url = pol_url+'/events/'+outopic+'/DefaultGroup/1?timeout=15000' + send_url = pol_url+'/DefaultGroup/1?timeout=15000' print(send_url) r = requests.post(send_url, data=payload) print(r.status_code, r.reason) @@ -236,12 +168,12 @@ def periodic_event(): #test setup for coverage def test_setup(args): global mr_url, pol_url, missing_htbt, intvl, intopic, outopic - mr_url = get_collector_uri() - pol_url = get_policy_uri() missing_htbt = float(int(args[2])) intvl = float(int(args[3])) intopic = args[4] outopic = args[5] + mr_url = get_collector_uri()+'/events/'+intopic + pol_url = get_policy_uri()+'/events/'+outopic print ("Message router url %s " % mr_url) print ("Policy url %s " % pol_url) print ("Interval %s " % intvl) @@ -252,23 +184,26 @@ def test_setup(args): #Main invocation def main(args): global periodic_scheduler - global mr_url, pol_url, missing_htbt, intvl, intopic, outopic + global mr_url, pol_url, missing_htbt, intvl, intopic, outopic, nfc, cl_loop #mr_url = get_collector_uri() #pol_url = get_policy_uri() mr_url = args[0] intopic = args[1] pol_url = args[2] outopic = args[3] - missing_htbt = int(args[4]) - intvl = int(args[5]) + nfc = args[4] + missing_htbt = int(args[5]) + intvl = int(args[6]) + cl_loop = args[7] print ("Message router url %s " % mr_url) print ("Policy router url %s " % pol_url) print ("Interval %s " % intvl) - #intvl = 60 # every second - #Start periodic scheduler runs every interval - periodic_scheduler = PeriodicScheduler() - periodic_scheduler.setup(intvl, periodic_event,) # it executes the event just once - periodic_scheduler.run() # it starts the scheduler + if( cl_loop != "internal_test") : + #intvl = 60 # every second + #Start periodic scheduler runs every interval + periodic_scheduler = PeriodicScheduler() + periodic_scheduler.setup(intvl, periodic_event,) # it executes the event just once + periodic_scheduler.run() # it starts the scheduler if __name__ == "__main__": total = len(sys.argv) diff --git a/miss_htbt_service/misshtbt.sh b/miss_htbt_service/misshtbt.sh new file mode 100755 index 0000000..5b598b1 --- /dev/null +++ b/miss_htbt_service/misshtbt.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# + +# get to where we are supposed to be for startup +cd /opt/app/misshtbt/bin + +# include path to 3.6+ version of python that has required dependencies included +export PATH=/usr/local/lib/python3.6/bin:$PATH:/opt/app/misshtbt/bin + +# expand search for python modules to include ./mod in runtime dir +export PYTHONPATH=/usr/local/lib/python3.6/site-packages:./mod:./:$PYTHONPATH:/opt/app/misshtbt/bin + +# set location of SSL certificates +export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-bundle.crt + +# PYTHONUNBUFFERED: +# set PYTHONUNBUFFERED to a non-empty string to avoid output buffering; +# comment out for runtime environments/better performance! +# export PYTHONUNBUFFERED="True" + +# set location of config broker server overrride IF NEEDED +# +#export CBS_HTBT_JSON=../etc/config.json + +# want tracing? Use this: +# python -m trace --trackcalls misshtbtd.py -v + +# want verbose logging? Use this: +# python misshtbtd.py -v + +# standard startup? Use this: +# python misshtbtd.py + +# unbuffered io for logs and verbose logging? Use this: +python -u misshtbtd.py -v + diff --git a/miss_htbt_service/misshtbtd.py b/miss_htbt_service/misshtbtd.py new file mode 100755 index 0000000..1c89b2d --- /dev/null +++ b/miss_htbt_service/misshtbtd.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +# ============LICENSE_START======================================================= +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +# Author Gokul Singaraju gs244f@att.com +# + +import os +import sys +import json +import multiprocessing +import logging +import subprocess +import get_logger +from pathlib import Path + +import mod.trapd_settings as tds +from mod.trapd_runtime_pid import save_pid, rm_pid +from mod.trapd_get_cbs_config import get_cbs_config +#from mod.trapd_exit import cleanup_and_exit +from mod.trapd_http_session import init_session_obj + + +mr_url = 'http://mrrouter.onap.org:3904' +pol_url = 'http://mrrouter.onap.org:3904' +intopic = 'VESCOLL-VNFNJ-SECHEARTBEAT-OUTPUT' +outopic = 'POLICY-HILOTCA-EVENT-OUTPUT' + +#Checks heartbeat by calling worker thread +def checkhtbt(mr_url, intopic, pol_url, outopic, nfc, misshtbt,intvl, cl_loop): + print('Doing some work',mr_url, misshtbt,intvl,intopic,outopic) + my_file = Path("./miss_htbt_service/htbtworker.py") + if my_file.is_file(): + subprocess.call(["python","./miss_htbt_service/htbtworker.py" , mr_url , intopic, pol_url, outopic, nfc, str(misshtbt) , str(intvl), cl_loop ]) + else: + subprocess.call(["python","/opt/app/misshtbt/bin/htbtworker.py" , mr_url , intopic, pol_url, outopic, nfc, str(misshtbt) , str(intvl), cl_loop ]) + sys.stdout.flush() + return + +_logger = get_logger.get_logger(__name__) + +#main functon which reads yaml config and invokes heartbeat +#monitoring +if __name__ == '__main__': + try: + print("Heartbeat Microservice ...") + if "INURL" in os.environ.keys(): + mr_url = os.environ['INURL'] + if "INTOPIC" in os.environ.keys(): + intopic = os.environ['INTOPIC'] + if "OUTURL" in os.environ.keys(): + pol_url = os.environ['OUTURL'] + if "OUTOPIC" in os.environ.keys(): + outopic = os.environ['OUTOPIC'] + print(outopic) + multiprocessing.log_to_stderr() + logger = multiprocessing.get_logger() + logger.setLevel(logging.INFO) + my_env = os.environ.copy() + my_env["PYTHONPATH"] = my_env["PYTHONPATH"]+":/usr/local/lib/python3.6"+":./miss_htbt_service/" + my_env["PATH"] = my_env["PATH"]+":./bin/:./miss_htbt_service/" + p = subprocess.Popen(['check_health.py'],stdout=subprocess.PIPE,stderr=subprocess.STDOUT,env=my_env) + #print(p.communicate()) + jsfile='empty' + + # re-request config from config binding service + # (either broker, or json file override) + if get_cbs_config(): + current_runtime_config_file_name = tds.c_config['files.runtime_base_dir'] + "../etc/download.json" + msg = "current config logged to : %s" % current_runtime_config_file_name + logger.error(msg) + print(msg) + with open(current_runtime_config_file_name, 'w') as outfile: + json.dump(tds.c_config, outfile) + else: + msg = "CBS Config not available using local config" + logger.error(msg) + print(msg) + my_file = Path("./etc/config.json") + if my_file.is_file(): + jsfile = "./etc/config.json" + else: + jsfile = "../etc/config.json" + + print("opening %s " % jsfile) + with open(jsfile, 'r') as outfile: + cfg = json.load(outfile) + # Put some initial values into the queue + mr_url = cfg['streams_subscribes']['ves_heartbeat']['dmaap_info']['topic_url'] + pol_url = cfg['streams_publishes']['ves_heartbeat']['dmaap_info']['topic_url'] + jobs = [] + print(cfg['heartbeat_config']) + for vnf in (cfg['heartbeat_config']['vnfs']): + print(vnf) + nfc = vnf['nfNamingCode'] + missed = vnf['heartbeatcountmissed'] + intvl = vnf['heartbeatinterval'] + clloop = vnf['closedLoopControlName'] + print('{0} {1} {2} {3}'.format(nfc,missed,intvl,clloop)) + #Start Heartbeat monitoring process worker thread on VNFs configured + logger.info("Starting threads...") + p = multiprocessing.Process(target=checkhtbt, args=( mr_url, intopic, pol_url, outopic, nfc, missed, intvl, clloop)) + jobs.append(p) + p.start() + for j in jobs: + j.join() + print('%s.exitcode = %s' % (j.name, j.exitcode)) + except Exception as e: + _logger.error("Fatal error. Could not start missing heartbeat service due to: {0}".format(e)) diff --git a/miss_htbt_service/mod/__init__.py b/miss_htbt_service/mod/__init__.py new file mode 100755 index 0000000..1875bf6 --- /dev/null +++ b/miss_htbt_service/mod/__init__.py @@ -0,0 +1,21 @@ +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + + +# empty __init__.py so that pytest can add correct path to coverage report, -- per pytest +# best practice guideline diff --git a/miss_htbt_service/mod/trapd_exit.py b/miss_htbt_service/mod/trapd_exit.py new file mode 100755 index 0000000..6247f4b --- /dev/null +++ b/miss_htbt_service/mod/trapd_exit.py @@ -0,0 +1,93 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +""" +trapc_exit_snmptrapd is responsible for removing any existing runtime PID +file, and exiting with the provided (param 1) exit code +""" + +__docformat__ = 'restructuredtext' + +import sys +import os +import string +from mod.trapd_runtime_pid import save_pid, rm_pid + +prog_name = os.path.basename(__file__) + + +# # # # # # # # # # # # # +# fx: cleanup_and_exit +# - remove pid file +# - exit with supplied return code +# # # # # # # # # # # # # +def cleanup_and_exit(_loc_exit_code, _pid_file_name): + """ + Remove existing PID file, and exit with provided exit code + :Parameters: + _loc_exit_code + value to return to calling shell upon exit + _pid_file_name + name of file that contains current process ID (for + removal) + :Exceptions: + none + :Keywords: + runtime PID exit + :Variables: + _num_params + number of parameters passed to module + """ + + # _num_params = len(locals()) + + if _pid_file_name is not None: + rc = rm_pid(_pid_file_name) + sys.exit(_loc_exit_code) + +# # # # # # # # # # # # # +# fx: cleanup_and_exit +# - remove pid file +# - exit with supplied return code +# # # # # # # # # # # # # +def cleanup(_loc_exit_code, _pid_file_name): + """ + Remove existing PID file, and exit with provided exit code + :Parameters: + _loc_exit_code + value to return to calling shell upon exit + _pid_file_name + name of file that contains current process ID (for + removal) + :Exceptions: + none + :Keywords: + runtime PID exit + :Variables: + _num_params + number of parameters passed to module + """ + + # _num_params = len(locals()) + + if _pid_file_name is not None: + rc = rm_pid(_pid_file_name) + + diff --git a/miss_htbt_service/mod/trapd_get_cbs_config.py b/miss_htbt_service/mod/trapd_get_cbs_config.py new file mode 100755 index 0000000..c108107 --- /dev/null +++ b/miss_htbt_service/mod/trapd_get_cbs_config.py @@ -0,0 +1,120 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +""" +Look for CBS broker and return application config; if not present, look for +env variable that specifies JSON equiv of CBS config (typically used for +testing purposes) +""" + +__docformat__ = 'restructuredtext' + +import json +import os +import sys +import string +import time +import traceback +import collections + +import mod.trapd_settings as tds +from onap_dcae_cbs_docker_client.client import get_config +from mod.trapd_exit import cleanup,cleanup_and_exit +from mod.trapd_io import stdout_logger + +prog_name = os.path.basename(__file__) + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# function: trapd_get_config_sim +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + + +def get_cbs_config(): + """ + Get config values from CBS or JSON file (fallback) + :Parameters: + none + :Exceptions: + """ + + tds.c_config = {} + + # See if we are in a config binding service (CBS) /controller environment + try: + tds.c_config = get_config() + if tds.c_config == {}: + msg = "Unable to fetch CBS config or it is erroneously empty - trying override/simulator config" + stdout_logger(msg) + + # if no CBS present, default to JSON config specified via CBS_HTBT_JSON env var + except Exception as e: + msg = "ONAP controller not present, trying json config override via CBS_HTBT_JSON env variable" + stdout_logger(msg) + + try: + _cbs_sim_json_file = os.getenv("CBS_HTBT_JSON", "None") + except Exception as e: + msg = "CBS_HTBT_JSON not defined - FATAL ERROR, exiting" + stdout_logger(msg) + cleanup(1,None) + return False + + if _cbs_sim_json_file == "None": + msg = "CBS_HTBT_JSON not defined - FATAL ERROR, exiting" + stdout_logger(msg) + cleanup(1,None) + return False + else: + msg = ("ONAP controller override specified via CBS_HTBT_JSON: %s" % + _cbs_sim_json_file) + stdout_logger(msg) + try: + tds.c_config = json.load(open(_cbs_sim_json_file)) + except Exception as e: + msg = "Unable to load CBS_HTBT_JSON " + _cbs_sim_json_file + \ + " (invalid json?) - FATAL ERROR, exiting" + stdout_logger(msg) + cleanup_and_exit(1,None) + + # recalc timeout, set default if not present + try: + tds.timeout_seconds = tds.c_config['publisher.http_timeout_milliseconds'] / 1000.0 + except Exception as e: + tds.timeout_seconds = 1.5 + + # recalc seconds_between_retries, set default if not present + try: + tds.seconds_between_retries = tds.c_config['publisher.http_milliseconds_between_retries'] / 1000.0 + except Exception as e: + tds.seconds_between_retries = .750 + + # recalc min_severity_to_log, set default if not present + try: + tds.minimum_severity_to_log = tds.c_config['files.minimum_severity_to_log'] + except Exception as e: + tds.minimum_severity_to_log = 3 + + try: + tds.publisher_retries = tds.c_config['publisher.http_retries'] + except Exception as e: + tds.publisher_retries = 3 + + return True diff --git a/miss_htbt_service/mod/trapd_http_session.py b/miss_htbt_service/mod/trapd_http_session.py new file mode 100755 index 0000000..b34c19d --- /dev/null +++ b/miss_htbt_service/mod/trapd_http_session.py @@ -0,0 +1,58 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +""" +trapd_http_session establishes an http session for future use in publishing +messages to the dmaap cluster. +""" + +__docformat__ = 'restructuredtext' + +import os +import requests +import traceback + +prog_name = os.path.basename(__file__) + + +# # # # # # # # # # # # # +# fx: init_session_obj +# # # # # # # # # # # # # +def init_session_obj(): + """ + Initializes and returns a http request session object for later use + :Parameters: + none + :Exceptions: + session object creation + this function will throw an exception if unable to create + a new session object + :Keywords: + http request session + :Variables: + none + """ + + try: + _loc_session = requests.Session() + except Exception as e: + return None + + return _loc_session diff --git a/miss_htbt_service/mod/trapd_io.py b/miss_htbt_service/mod/trapd_io.py new file mode 100755 index 0000000..c89eaa3 --- /dev/null +++ b/miss_htbt_service/mod/trapd_io.py @@ -0,0 +1,392 @@ +# ============LICENSE_START=======================================================) +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +""" +""" + +__docformat__ = 'restructuredtext' + +# basics +import datetime +import errno +import inspect +import json +import logging +import logging.handlers +import os +import sys +import string +import time +import traceback +import unicodedata + +# dcae_snmptrap +import mod.trapd_settings as tds +from mod.trapd_exit import cleanup_and_exit + +prog_name = os.path.basename(__file__) + + +# # # # # # # # # # # # # # # # # # # +# fx: roll_all_logs -> roll all logs to timestamped backup +# # # # # # # # # # ## # # # # # # # + + +def roll_all_logs(): + """ + roll all active logs to timestamped version, open new one + based on frequency defined in files.roll_frequency + """ + + # first roll all the eelf files + # NOTE: this will go away when onap logging is standardized/available + try: + # open various ecomp logs - if any fails, exit + for fd in [tds.eelf_error_fd, tds.eelf_debug_fd, tds.eelf_audit_fd, + tds.eelf_metrics_fd, tds.arriving_traps_fd, tds.json_traps_fd]: + fd.close() + + roll_file(tds.eelf_error_file_name) + roll_file(tds.eelf_debug_file_name) + roll_file(tds.eelf_audit_file_name) + roll_file(tds.eelf_metrics_file_name) + + except Exception as e: + msg = "Error closing logs: " + str(e) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + reopened_successfully = open_eelf_logs() + if not reopened_successfully: + msg = "Error re-opening EELF logs during roll-over to timestamped versions - EXITING" + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + # json log + roll_file(tds.json_traps_filename) + + try: + tds.json_traps_fd = open_file(tds.json_traps_filename) + except Exception as e: + msg = ("Error opening json_log %s : %s" % + (json_traps_filename, str(e))) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + # arriving trap log + roll_file(tds.arriving_traps_filename) + + try: + tds.arriving_traps_fd = open_file(tds.arriving_traps_filename) + except Exception as e: + msg = ("Error opening arriving traps %s : %s" % + (arriving_traps_filename, str(e))) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + +# # # # # # # # # # # # # # # # # # # +# fx: setup_ecomp_logs -> log in eelf format until standard +# is released for python via LOG-161 +# # # # # # # # # # ## # # # # # # # + + +def open_eelf_logs(): + """ + open various (multiple ???) logs + """ + + try: + # open various ecomp logs - if any fails, exit + + tds.eelf_error_file_name = ( + tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_error']) + tds.eelf_error_fd = open_file(tds.eelf_error_file_name) + + except Exception as e: + msg = "Error opening eelf error log : " + str(e) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + try: + tds.eelf_debug_file_name = ( + tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_debug']) + tds.eelf_debug_fd = open_file(tds.eelf_debug_file_name) + + except Exception as e: + msg = "Error opening eelf debug log : " + str(e) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + try: + tds.eelf_audit_file_name = ( + tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_audit']) + tds.eelf_audit_fd = open_file(tds.eelf_audit_file_name) + except Exception as e: + msg = "Error opening eelf audit log : " + str(e) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + try: + tds.eelf_metrics_file_name = ( + tds.c_config['files.eelf_base_dir'] + "/" + tds.c_config['files.eelf_metrics']) + tds.eelf_metrics_fd = open_file(tds.eelf_metrics_file_name) + except Exception as e: + msg = "Error opening eelf metric log : " + str(e) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + return True + +# # # # # # # # # # # # # # # # # # # +# fx: roll_log_file -> move provided filename to timestamped version +# # # # # # # # # # ## # # # # # # # + + +def roll_file(_loc_file_name): + """ + move active file to timestamped archive + """ + + _file_name_suffix = "%s" % (datetime.datetime.fromtimestamp(time.time()). + fromtimestamp(time.time()). + strftime('%Y-%m-%dT%H:%M:%S')) + + _loc_file_name_bak = _loc_file_name + '.' + _file_name_suffix + + # roll existing file if present + if os.path.isfile(_loc_file_name): + try: + os.rename(_loc_file_name, _loc_file_name_bak) + return True + except Exception as e: + _msg = ("ERROR: Unable to rename %s to %s" + % (_loc_file_name, + _loc_file_name_bak)) + ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_CRIT, + tds.CODE_GENERAL, _msg) + return False + + return False + +# # # # # # # # # # # # # +# fx: open_log_file +# # # # # # # # # # # # # + + +def open_file(_loc_file_name): + """ + open _loc_file_name, return file handle + """ + + try: + # open append mode just in case so nothing is lost, but should be + # non-existent file + _loc_fd = open(_loc_file_name, 'a') + return _loc_fd + except Exception as e: + msg = "Error opening " + _loc_file_name + " append mode - " + str(e) + stdout_logger(msg) + cleanup_and_exit(1, tds.pid_file_name) + + +# # # # # # # # # # # # # +# fx: close_file +# # # # # # # # # # # # # + """ + close _loc_file_name, return True with success, False otherwise + """ + + +def close_file(_loc_fd, _loc_filename): + + try: + _loc_fd.close() + return True + except Exception as e: + msg = "Error closing %s : %s - results indeterminate" % ( + _loc_filename, str(e)) + ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_FATAL, tds.CODE_GENERAL, msg) + return False + +# # # # # # # # # # # # # # # # # # # +# fx: ecomp_logger -> log in eelf format until standard +# is released for python via LOG-161 +# # # # # # # # # # ## # # # # # # # + +def ecomp_logger(_log_type, _sev, _error_code, _msg): + """ + Log to ecomp-style logfiles. Logs include: + + Note: this will be updated when https://jira.onap.org/browse/LOG-161 + is closed/available; until then, we resort to a generic format with + valuable info in "extra=" field (?) + + :Parameters: + _msg - + :Exceptions: + none + :Keywords: + eelf logging + :Log Styles: + + :error.log: + + if CommonLogger.verbose: print("using CommonLogger.ErrorFile") + self._logger.log(50, '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + % (requestID, threadID, serviceName, partnerName, targetEntity, targetServiceName, + errorCategory, errorCode, errorDescription, detailMessage)) + + error.log example: + + 2018-02-20T07:21:34,007+00:00||MainThread|snmp_log_monitor||||FATAL|900||Tue Feb 20 07:21:11 UTC 2018 CRITICAL: [a0cae74e-160e-11e8-8f9f-0242ac110002] ALL publish attempts failed to DMAPP server: dcae-mrtr-zltcrdm5bdce1.1dff83.rdm5b.tci.att.com, topic: DCAE-COLLECTOR-UCSNMP, 339 trap(s) not published in epoch_serno range: 15191112530000 - 15191112620010 + + :debug.log: + + if CommonLogger.verbose: print("using CommonLogger.DebugFile") + self._logger.log(50, '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + % (requestID, threadID, serverName, serviceName, instanceUUID, upperLogLevel, + severity, serverIPAddress, server, IPAddress, className, timer, detailMessage)) + + debug.log example: + + none available + + :audit.log: + + if CommonLogger.verbose: print("using CommonLogger.AuditFile") + endAuditTime, endAuditMsec = self._getTime() + if self._begTime is not None: + d = {'begtime': self._begTime, 'begmsecs': self._begMsec, 'endtime': endAuditTime, + 'endmsecs': endAuditMsec} + else: + d = {'begtime': endAuditTime, 'begmsecs': endAuditMsec, 'endtime': endAuditTime, + 'endmsecs': endAuditMsec} + + self._logger.log(50, '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + % (requestID, serviceInstanceID, threadID, serverName, serviceName, partnerName, + statusCode, responseCode, responseDescription, instanceUUID, upperLogLevel, + severity, serverIPAddress, timer, server, IPAddress, className, unused, + processKey, customField1, customField2, customField3, customField4, + detailMessage), extra=d) + + + :metrics.log: + + self._logger.log(50,'%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + % (requestID, serviceInstanceID, threadID, serverName, serviceName, partnerName, + targetEntity, targetServiceName, statusCode, responseCode, responseDescription, + instanceUUID, upperLogLevel, severity, serverIPAddress, timer, server, + IPAddress, + className, unused, processKey, targetVirtualEntity, customField1, customField2, + customField3, customField4, detailMessage), extra=d) + + metrics.log example: + + none available + + + """ + + unused = "" + + # above were various attempts at setting time string found in other + # libs; instead, let's keep it real: + t_out = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S,%f")[:-3] + calling_fx = inspect.stack()[1][3] + + # DLFM: this entire module is a hack to override concept of prog logging + # written across multiple files (???), making diagnostics IMPOSSIBLE! + # Hoping to leverage ONAP logging libraries & standards when available + + # catch invalid log type + if _log_type < 1 or _log_type > 5: + msg = ("INVALID log type: %s " % _log_type) + _out_rec = ("%s|%s|%s|%s|%s|%s|%s|%s|%s" + % (calling_fx, "snmptrapd", unused, unused, unused, tds.SEV_TYPES[_sev], _error_code, unused, (msg + _msg))) + try: + tds.eelf_error_fd.write('%s|%s\n' % (t_out, str(_out_rec))) + except Exception as e: + stdout_logger(str(_out_rec)) + + return False + + if _sev >= tds.minimum_severity_to_log: + # log to appropriate eelf log (different files ??) + if _log_type == tds.LOG_TYPE_ERROR: + _out_rec = ('%s|%s|%s|%s|%s|%s|%s|%s|%s' + % (calling_fx, "snmptrapd", unused, unused, unused, tds.SEV_TYPES[_sev], _error_code, unused, _msg)) + try: + tds.eelf_error_fd.write('%s|%s\n' % (t_out, str(_out_rec))) + except Exception as e: + stdout_logger(str(_out_rec)) + elif _log_type == tds.LOG_TYPE_AUDIT: + # log message in AUDIT format + _out_rec = ('%s|%s|%s|%s|%s|%s|%s|%s|%s' + % (calling_fx, "snmptrapd", unused, unused, unused, tds.SEV_TYPES[_sev], _error_code, unused, _msg)) + try: + tds.eelf_audit_fd.write('%s|%s\n' % (t_out, str(_out_rec))) + except Exception as e: + stdout_logger(str(_out_rec)) + elif _log_type == tds.LOG_TYPE_METRICS: + # log message in METRICS format + _out_rec = ('%s|%s|%s|%s|%s|%s|%s|%s|%s' + % (calling_fx, "snmptrapd", unused, unused, unused, tds.SEV_TYPES[_sev], _error_code, unused, _msg)) + try: + tds.eelf_metrics_fd.write('%s|%s\n' % (t_out, str(_out_rec))) + except Exception as e: + stdout_logger(str(_out_rec)) + + # DEBUG *AND* others - there *MUST BE* a single time-sequenced log for diagnostics! + # DLFM: too much I/O !!! + # always write to debug; we need ONE logfile that has time-sequence full view !!! + # log message in DEBUG format + _out_rec = ("%s|%s|%s|%s|%s|%s|%s|%s|%s" + % (calling_fx, "snmptrapd", unused, unused, unused, tds.SEV_TYPES[_sev], _error_code, unused, _msg)) + try: + tds.eelf_debug_fd.write('%s|%s\n' % (t_out, str(_out_rec))) + except Exception as e: + stdout_logger(str(_out_rec)) + + return True + +# # # # # # # # # # # # # +# fx: stdout_logger +# # # # # # # # # # # # # + + +def stdout_logger(_msg): + """ + Log info/errors to stdout. This is done: + - for critical runtime issues + + :Parameters: + _msg + message to print + :Exceptions: + none + :Keywords: + log stdout + :Variables: + """ + + t_out = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S,%f")[:-3] + + print('%s %s' % (t_out, _msg)) diff --git a/miss_htbt_service/mod/trapd_runtime_pid.py b/miss_htbt_service/mod/trapd_runtime_pid.py new file mode 100755 index 0000000..c6ef76e --- /dev/null +++ b/miss_htbt_service/mod/trapd_runtime_pid.py @@ -0,0 +1,94 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +""" +trapd_runtime_pid maintains a 'PID file' (file that contains the +PID of currently running trap receiver) +""" + +__docformat__ = 'restructuredtext' + +import logging +import os +import string +import time +import traceback + +prog_name = os.path.basename(__file__) + + +# # # # # # # # # # # # # +# fx: save_pid - save PID of running process +# # # # # # # # # # # # # +def save_pid(_pid_file_name): + """ + Save the current process ID in a file for external + access. + :Parameters: + none + :Exceptions: + file open + this function will catch exception of unable to + open/create _pid_file_name + :Keywords: + pid /var/run + """ + + try: + pid_fd = open(_pid_file_name, 'w') + pid_fd.write('%d' % os.getpid()) + pid_fd.close() + except IOError: + print("IOError saving PID file %s :" % _pid_file_name) + return False + # except: + # print("Error saving PID file %s :" % _pid_file_name) + # return False + else: + # print("Runtime PID file: %s" % _pid_file_name) + return True + + +# # # # # # # # # # # # # +# fx: rm_pid - remove PID of running process +# # # # # # # # # # # # # +def rm_pid(_pid_file_name): + """ + Remove the current process ID file before exiting. + :Parameters: + none + :Exceptions: + file open + this function will catch exception of unable to find or remove + _pid_file_name + :Keywords: + pid /var/run + """ + + try: + if os.path.isfile(_pid_file_name): + os.remove(_pid_file_name) + return True + else: + return False + + except IOError: + print("Error removing Runtime PID file: %s" % _pid_file_name) + return False diff --git a/miss_htbt_service/mod/trapd_settings.py b/miss_htbt_service/mod/trapd_settings.py new file mode 100755 index 0000000..be87e26 --- /dev/null +++ b/miss_htbt_service/mod/trapd_settings.py @@ -0,0 +1,169 @@ +# ============LICENSE_START=======================================================) +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +""" +""" + +__docformat__ = 'restructuredtext' + + +def init(): + + # + # consul config or simulated via json file + global c_config + c_config = None + # + + # + # + # dns_cache_ip_to_name + # key [ip address] -> fqdn + # dns_cache_ip_expires + # key [ip address] -> epoch time this entry expires and must + # be reloaded + global dns_cache_ip_to_name + dns_cache_ip_to_name = {} + global dns_cache_ip_expires + dns_cache_ip_expires = {} + # + + # + global eelf_error_file_name + eelf_error_file_name = "" + global eelf_error_fd + eelf_error_fd = None + + global eelf_debug_file_name + eelf_debug_file_name = "" + global eelf_debug_fd + eelf_debug_fd = None + + global eelf_audit_file_name + eelf_audit_file_name = "" + global eelf_audit_fd + eelf_audit_fd = None + + global eelf_metrics_file_name + eelf_metrics_file_name = "" + global eelf_metrics_fd + eelf_metrics_fd = None + + global last_minute + last_minute = 0 + global last_hour + last_hour = 0 + global last_day + last_day = 0 + # + + # + + # + global traps_in_minute + traps_in_minute = 0 + global last_epoch_second + last_epoch_second = 0 + global traps_since_last_publish + traps_since_last_publish = 0 + global last_pub_time + last_pub_time = 0 + global milliseconds_since_last_publish + milliseconds_since_last_publish = 0 + global timeout_seconds + timeout_seconds = 1.5 + global seconds_between_retries + seconds_between_retries = 2 + global publisher_retries + publisher_retries = 2 + # + + # + global http_requ_session + http_requ_session = None + # + + # + global json_traps_filename + json_log_filename = "" + global json_traps_fd + json_fd = None + # + + # + global arriving_traps_filename + arriving_traps_filename = "" + global arriving_traps_fd + arriving_traps_fd = None + # + + # + global pid_file_name + pid_file_name = "" + + # + global LOG_TYPES + global LOG_TYPE_NONE + global LOG_TYPE_ERROR + global LOG_TYPE_DEBUG + global LOG_TYPE_AUDIT + global LOG_TYPE_METRICS + LOG_TYPES = ["none", "ERROR", "DEBUG", "AUDIT", "METRICS"] + LOG_TYPE_NONE = 0 + LOG_TYPE_ERROR = 1 + LOG_TYPE_DEBUG = 2 + LOG_TYPE_AUDIT = 3 + LOG_TYPE_METRICS = 4 + + global SEV_TYPES + global SEV_NONE + global SEV_DETAILED + global SEV_INFO + global SEV_WARN + global SEV_CRIT + global SEV_FATAL + SEV_TYPES = ["none", "DETAILED", "INFO", "WARN", "CRITICAL", "FATAL"] + SEV_NONE = 0 + SEV_DETAILED = 1 + SEV_INFO = 2 + SEV_WARN = 3 + SEV_CRIT = 4 + SEV_FATAL = 5 + + global CODE_GENERAL + CODE_GENERAL = "100" + + global minimum_severity_to_log + minimum_severity_to_log = 3 + + # diff --git a/mvn-phase-script.sh b/mvn-phase-script.sh index 83b76dd..6b47ba5 100755 --- a/mvn-phase-script.sh +++ b/mvn-phase-script.sh @@ -138,7 +138,7 @@ run_tox_test() rm -rf ./venv-tox ./.tox virtualenv ./venv-tox source ./venv-tox/bin/activate - pip install pip==9.0.3 + pip install pip==10.0.1 pip install --upgrade argparse pip install tox==2.9.1 pip freeze diff --git a/pom.xml b/pom.xml index bd1efd5..0bec0e1 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property. py python **/**.py - tests/*,setup.py,bin/* + tests/*,setup.py ${project.artifactId}-${project.version} diff --git a/requirements.txt b/requirements.txt index 768a335..d94b4c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ request==1.0.1 -requests==2.18.2 +requests==2.18.3 +onap_dcae_cbs_docker_client==1.0.1 six==1.10.0 PyYAML==3.12 httplib2==0.9.2 diff --git a/setup.py b/setup.py index e5d3ceb..504435a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,11 @@ # ECOMP is a trademark and service mark of AT&T Intellectual Property. import os +import string +import sys +import setuptools from setuptools import setup, find_packages + #from pip.req import parse_requirements try: # for pip >= 10 from pip._internal.req import parse_requirements @@ -33,7 +37,19 @@ setup( name='miss_htbt_service', description='Missing heartbeat microservice to communicate with policy-engine', version='2.0.0', - packages=find_packages(exclude=["tests.*", "tests"]), + #packages=find_packages(exclude=["tests.*", "tests"]), + packages=find_packages(), + install_requires=[ +"request==1.0.1", +"requests==2.18.3", +"onap_dcae_cbs_docker_client==1.0.1", +"six==1.10.0", +"PyYAML==3.12", +"httplib2==0.9.2", +"HTTPretty==0.8.14", +"pyOpenSSL==17.5.0", +"Wheel==0.31.0" + ], author = "Gokul Singaraju", author_email = "gs244f@att.com", license = "", diff --git a/tests/test_binding.py b/tests/test_binding.py index f0faac0..24c7b61 100644 --- a/tests/test_binding.py +++ b/tests/test_binding.py @@ -23,11 +23,16 @@ import httpretty #import miss_htbt_service from miss_htbt_service import htbtworker #from miss_htbt_service.htbtworker import get_collector_uri,get_policy_uri +import subprocess import pytest import json import base64 import errno import imp +import time +from onap_dcae_cbs_docker_client.client import get_config + + MODULE_EXTENSIONS = ('.py', '.pyc', '.pyo') def package_contents(package_name): @@ -44,14 +49,14 @@ def package_contents(package_name): ##### #mr_url = 'http://127.0.0.1:3904' -mr_url = 'http://mrrouter.att.com:3904' +mr_url = 'http://mrrouter.onap.org:3904' intopic = 'VESCOLL-VNFNJ-SECHEARTBEAT-OUTPUT' outopic = 'POLICY-HILOTCA-EVENT-OUTPUT' @httpretty.activate def test_resolve_all(monkeypatch): #htbtmsg = "Find the best daily deals" - htbtmsg = '{"event":{"commonEventHeader":{"startEpochMicrosec":1518616063564475,"sourceId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","eventId":"10048640","reportingEntityId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","priority":"Normal","version":3,"reportingEntityName":"SWMSVM","sequence":10048640,"domain":"heartbeat","lastEpochMicrosec":1518616063564476,"eventName":"Heartbeat_vMrf","sourceName":"SWMSVM","nfNamingCode":"vMRF"}}}' + htbtmsg = '{"event":{"commonEventHeader":{"startEpochMicrosec":1518616063564475,"sourceId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","eventId":"10048640","reportingEntityId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","priority":"Normal","version":3,"reportingEntityName":"TESTVM","sequence":10048640,"domain":"heartbeat","lastEpochMicrosec":1518616063564476,"eventName":"Heartbeat_vVnf","sourceName":"TESTVM","nfNamingCode":"vVNF"}}}' send_url = mr_url+'/events/'+intopic+'/DefaultGroup/1?timeout=15000' print(send_url) httpretty.register_uri(httpretty.GET, send_url, body=htbtmsg) @@ -60,33 +65,25 @@ def test_resolve_all(monkeypatch): print(response) print(response.text) assert(response.text == htbtmsg) - try: - os.makedirs('/tmp/config') - except OSError as e: - if e.errno != errno.EEXIST: - raise - with open("/tmp/config/coll_ip.txt", "w") as file: - #file.write('127.0.0.1') - file.write('mytest.onap.org') - file.close() - with open("/tmp/config/coll_port.txt", "w") as file2: - file2.write('3904') - file2.close() - #print(package_contents('miss_htbt_service')) - #response = requests.get(send_url) - #print(response) - #print(response.text) - #assert(response.text == htbtmsg) - htbtmsg = json.dumps({"event":{"commonEventHeader":{"startEpochMicrosec":1518616063564475,"sourceId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","eventId":"10048640","reportingEntityId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","priority":"Normal","version":3,"reportingEntityName":"SWMSVM","sequence":10048640,"domain":"heartbeat","lastEpochMicrosec":1518616063564476,"eventName":"Heartbeat_vMrf","sourceName":"SWMSVM","nfNamingCode":"vMRF"}}}) - send_url = htbtworker.get_collector_uri()+'/events/'+intopic+'/DefaultGroup/1?timeout=15000' + htbtmsg = json.dumps({"event":{"commonEventHeader":{"startEpochMicrosec":1518616063564475,"sourceId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","eventId":"10048640","reportingEntityId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","priority":"Normal","version":3,"reportingEntityName":"TESTVM","sequence":10048640,"domain":"heartbeat","lastEpochMicrosec":1518616063564476,"eventName":"Heartbeat_vVnf","sourceName":"TESTVM","nfNamingCode":"vVNF"}}}) + send_url = mr_url+'/events/'+intopic+'/DefaultGroup/1?timeout=15000' print("Send URL : "+send_url) httpretty.register_uri(httpretty.GET, send_url, body=htbtmsg, content_type="application/json") - pol_url = htbtworker.get_policy_uri()+'/events/'+outopic+'/DefaultGroup/1?timeout=15000' - pol_body = json.dumps({"event":{"commonEventHeader":{"startEpochMicrosec":1518616063564475,"sourceId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","eventId":"10048640","reportingEntityId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","priority":"Normal","version":3,"reportingEntityName":"SWMSVM","sequence":10048640,"domain":"heartbeat","lastEpochMicrosec":1518616063564476,"eventName":"Heartbeat_vMrf","sourceName":"SWMSVM","nfNamingCode":"vMRF"}}}) + pol_url = mr_url+'/events/'+outopic+'/DefaultGroup/1?timeout=15000' + pol_body = json.dumps({"event":{"commonEventHeader":{"startEpochMicrosec":1518616063564475,"sourceId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","eventId":"10048640","reportingEntityId":"587c14b3-72c0-4581-b5cb-6567310b9bb7","priority":"Normal","version":3,"reportingEntityName":"TESTVM","sequence":10048640,"domain":"heartbeat","lastEpochMicrosec":1518616063564476,"eventName":"Heartbeat_vVnf","sourceName":"TESTVM","nfNamingCode":"vVNF"}}}) print("Policy URL : "+pol_url) httpretty.register_uri(httpretty.POST, pol_url, body=pol_body, status=200, content_type='text/json') - htbtworker.test_setup([send_url,send_url,3,60,intopic,outopic]) + htbtworker.main([send_url,intopic,send_url,outopic,"vVNF",3,60,"internal_test"]) ret = htbtworker.periodic_event() print("Returned",ret) assert(ret == 1) +def test_full(): + p = subprocess.Popen(['./miss_htbt_service/misshtbtd.py'],stdout=subprocess.PIPE) + time.sleep(30) + r = requests.get('http://127.0.0.1:10001') + print(r.status_code) + assert(r.status_code == 200) + #r = requests.post('http://127.0.0.1:10001',data={'number': 12524, 'health': 'good', 'action': 'show'}) + #print(r.status_code) + #assert(r.status_code == 200) diff --git a/tests/test_trapd_exit.py b/tests/test_trapd_exit.py new file mode 100644 index 0000000..594624f --- /dev/null +++ b/tests/test_trapd_exit.py @@ -0,0 +1,37 @@ +import pytest +import unittest +import trapd_exit + +pid_file="/tmp/test_pid_file" +pid_file_dne="/tmp/test_pid_file_NOT" + +class test_cleanup_and_exit(unittest.TestCase): + """ + Test the cleanup_and_exit mod + """ + + def test_normal_exit(self): + """ + Test normal exit works as expected + """ + open(pid_file, 'w') + + with pytest.raises(SystemExit) as pytest_wrapped_sys_exit: + result = trapd_exit.cleanup_and_exit(0,pid_file) + assert pytest_wrapped_sys_exit.type == SystemExit + assert pytest_wrapped_sys_exit.value.code == 0 + + # compare = str(result).startswith("SystemExit: 0") + # self.assertEqual(compare, True) + + def test_abnormal_exit(self): + """ + Test exit with missing PID file exits non-zero + """ + with pytest.raises(SystemExit) as pytest_wrapped_sys_exit: + result = trapd_exit.cleanup_and_exit(0,pid_file_dne) + assert pytest_wrapped_sys_exit.type == SystemExit + assert pytest_wrapped_sys_exit.value.code == 1 + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_trapd_get_cbs_config.py b/tests/test_trapd_get_cbs_config.py new file mode 100644 index 0000000..bd8d082 --- /dev/null +++ b/tests/test_trapd_get_cbs_config.py @@ -0,0 +1,70 @@ +import pytest +import unittest +import os + +from onap_dcae_cbs_docker_client.client import get_config +from trapd_exit import cleanup_and_exit +from trapd_io import stdout_logger, ecomp_logger +import trapd_settings as tds +import trapd_get_cbs_config + +class test_get_cbs_config(unittest.TestCase): + """ + Test the trapd_get_cbs_config mod + """ + + pytest_json_data = "{ \"snmptrap.version\": \"1.3.0\", \"snmptrap.title\": \"ONAP SNMP Trap Receiver\" , \"protocols.transport\": \"udp\", \"protocols.ipv4_interface\": \"0.0.0.0\", \"protocols.ipv4_port\": 6164, \"protocols.ipv6_interface\": \"::1\", \"protocols.ipv6_port\": 6164, \"cache.dns_cache_ttl_seconds\": 60, \"publisher.http_timeout_milliseconds\": 1500, \"publisher.http_retries\": 3, \"publisher.http_milliseconds_between_retries\": 750, \"publisher.http_primary_publisher\": \"true\", \"publisher.http_peer_publisher\": \"unavailable\", \"publisher.max_traps_between_publishes\": 10, \"publisher.max_milliseconds_between_publishes\": 10000, \"streams_publishes\": { \"sec_measurement\": { \"type\": \"message_router\", \"aaf_password\": \"aaf_password\", \"dmaap_info\": { \"location\": \"mtl5\", \"client_id\": \"111111\", \"client_role\": \"com.att.dcae.member\", \"topic_url\": null }, \"aaf_username\": \"aaf_username\" }, \"sec_fault_unsecure\": { \"type\": \"message_router\", \"aaf_password\": null, \"dmaap_info\": { \"location\": \"mtl5\", \"client_id\": null, \"client_role\": null, \"topic_url\": \"http://uebsb93kcdc.it.att.com:3904/events/ONAP-COLLECTOR-SNMPTRAP\" }, \"aaf_username\": null } }, \"files.runtime_base_dir\": \"/tmp/opt/app/snmptrap\", \"files.log_dir\": \"logs\", \"files.data_dir\": \"data\", \"files.pid_dir\": \"/tmp/opt/app/snmptrap/tmp\", \"files.arriving_traps_log\": \"snmptrapd_arriving_traps.log\", \"files.snmptrapd_diag\": \"snmptrapd_prog_diag.log\", \"files.traps_stats_log\": \"snmptrapd_stats.csv\", \"files.perm_status_file\": \"snmptrapd_status.log\", \"files.eelf_base_dir\": \"/tmp/opt/app/snmptrap/logs\", \"files.eelf_error\": \"error.log\", \"files.eelf_debug\": \"debug.log\", \"files.eelf_audit\": \"audit.log\", \"files.eelf_metrics\": \"metrics.log\", \"files.roll_frequency\": \"hour\", \"files.minimum_severity_to_log\": 2, \"trap_def.1.trap_oid\" : \".1.3.6.1.4.1.74.2.46.12.1.1\", \"trap_def.1.trap_category\": \"DCAE-SNMP-TRAPS\", \"trap_def.2.trap_oid\" : \"*\", \"trap_def.2.trap_category\": \"DCAE-SNMP-TRAPS\", \"stormwatch.1.stormwatch_oid\" : \".1.3.6.1.4.1.74.2.46.12.1.1\", \"stormwatch.1.low_water_rearm_per_minute\" : \"5\", \"stormwatch.1.high_water_arm_per_minute\" : \"100\", \"stormwatch.2.stormwatch_oid\" : \".1.3.6.1.4.1.74.2.46.12.1.2\", \"stormwatch.2.low_water_rearm_per_minute\" : \"2\", \"stormwatch.2.high_water_arm_per_minute\" : \"200\", \"stormwatch.3.stormwatch_oid\" : \".1.3.6.1.4.1.74.2.46.12.1.2\", \"stormwatch.3.low_water_rearm_per_minute\" : \"2\", \"stormwatch.3.high_water_arm_per_minute\" : \"200\" }" + + # create copy of snmptrapd.json for pytest + pytest_json_config = "/tmp/opt/app/miss_htbt_service/etc/config.json" + with open(pytest_json_config, 'w') as outfile: + outfile.write(pytest_json_data) + + + def test_cbs_env_present(self): + """ + Test that CONSUL_HOST env variable exists but fails to + respond + """ + os.environ.update(CONSUL_HOST='nosuchhost') + # del os.environ['CBS_HTBT_JSON'] + # result = trapd_get_cbs_config.get_cbs_config() + # print("result: %s" % result) + # compare = str(result).startswith("{'snmptrap': ") + # self.assertEqual(compare, False) + + with pytest.raises(Exception) as pytest_wrapped_sys_exit: + result = trapd_get_cbs_config.get_cbs_config() + assert pytest_wrapped_sys_exit.type == SystemExit + # assert pytest_wrapped_sys_exit.value.code == 1 + + + def test_cbs_override_env_invalid(self): + """ + """ + os.environ.update(CBS_HTBT_JSON='/tmp/opt/app/miss_htbt_service/etc/nosuchfile.json') + # result = trapd_get_cbs_config.get_cbs_config() + # print("result: %s" % result) + # compare = str(result).startswith("{'snmptrap': ") + # self.assertEqual(compare, False) + + with pytest.raises(SystemExit) as pytest_wrapped_sys_exit: + result = trapd_get_cbs_config.get_cbs_config() + assert pytest_wrapped_sys_exit.type == SystemExit + assert pytest_wrapped_sys_exit.value.code == 1 + + + def test_cbs_fallback_env_present(self): + """ + Test that CBS fallback env variable exists and we can get config + from fallback env var + """ + os.environ.update(CBS_HTBT_JSON='/tmp/opt/app/miss_htbt_service/etc/config.json') + result = trapd_get_cbs_config.get_cbs_config() + print("result: %s" % result) + # compare = str(result).startswith("{'snmptrap': ") + # self.assertEqual(compare, True) + self.assertEqual(result, True) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_trapd_http_session.py b/tests/test_trapd_http_session.py new file mode 100644 index 0000000..8f61d08 --- /dev/null +++ b/tests/test_trapd_http_session.py @@ -0,0 +1,20 @@ +import pytest +import unittest +import trapd_http_session + +class test_init_session_obj(unittest.TestCase): + """ + Test the init_session_obj mod + """ + + def test_correct_usage(self): + """ + Test that attempt to create http session object works + """ + result = trapd_http_session.init_session_obj() + compare = str(result).startswith("