From 8ea70b60efac67f8b4ad4be8b8018a946f551d06 Mon Sep 17 00:00:00 2001 From: Satoshi Fujii Date: Wed, 13 Oct 2021 09:16:09 +0000 Subject: Fix pod become unready state The original implementation used Popen but its stdout pipe became clogged because the captured output was not processed properly. As a result, readiness check failed due to health-check server not responding and the pod became unready state. Signed-off-by: Satoshi Fujii Issue-ID: DCAEGEN2-2832 Change-Id: I98259e25fe40d5374a2c4236fc20bd631673f15e --- miss_htbt_service/misshtbtd.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'miss_htbt_service/misshtbtd.py') diff --git a/miss_htbt_service/misshtbtd.py b/miss_htbt_service/misshtbtd.py index 0ebffe7..ff2f3fe 100644 --- a/miss_htbt_service/misshtbtd.py +++ b/miss_htbt_service/misshtbtd.py @@ -39,6 +39,8 @@ import yaml import socket import os.path as path from pathlib import Path + +import check_health import htbtworker as heartbeat import get_logger from mod import trapd_settings as tds @@ -48,7 +50,6 @@ hb_properties_file = path.abspath(path.join(__file__, "../config/hbproperties.ya ABSOLUTE_PATH1 = path.abspath(path.join(__file__, "../htbtworker.py")) ABSOLUTE_PATH2 = path.abspath(path.join(__file__, "../db_monitoring.py")) -ABSOLUTE_PATH3 = path.abspath(path.join(__file__, "../check_health.py")) ABSOLUTE_PATH4 = path.abspath(path.join(__file__, "../cbs_polling.py")) @@ -332,9 +333,13 @@ _logger = get_logger.get_logger(__name__) def main(): + hc_proc = multiprocessing.Process(target=check_health.start_health_check_server) try: - subprocess.Popen([ABSOLUTE_PATH3], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) _logger.info("MSHBD:Execution Started") + # Start health check server + hc_proc.start() + _logger.info("MSHBD: Started health check server. PID=%d", hc_proc.pid) + job_list = [] pid_current = os.getpid() jsfile = fetch_json_file() @@ -444,6 +449,11 @@ def main(): msg = "Fatal error. Could not start missing heartbeat service due to: {0}".format(e) _logger.error(msg) + finally: + # Stop health check server + if hc_proc.pid is not None: + hc_proc.terminate() + hc_proc.join() if __name__ == '__main__': -- cgit 1.2.3-korg