Add support for query_range in promql_api

Added support for query_range in promql_api

Issue-ID: ONAPARC-452
Signed-off-by: Rajamohan Raj <rajamohan.raj@intel.com>
Change-Id: If4870c8a8ea2fd4e6b23237cb8fd0f34a17fe3e0

3 files changed, 157 insertions, 27 deletions

diff --git a/vnfs/DAaaS/lib/promql_api/README.md b/vnfs/DAaaS/lib/promql_api/README.md
index 89f996af..594e1279 100644
--- a/vnfs/DAaaS/lib/promql_api/README.md
+++ b/vnfs/DAaaS/lib/promql_api/README.md
@@ -1,47 +1,106 @@
 ## What does this API do ?
-This api as of now provides a function which takes in a list of 'QUERY_STRINGs' with respect to prometheus
-and returns the corresponding result_sets in a list.
 
-For eg:
-If the QUERY_STRING is
+This API has support for two PROMQL functions as of now :
+
+1. 'query' 
+2. 'query_range'
+
+You can directly call the above functions with required parameters, and API shall
+give out a result set.
+
+## How to use this API ?
+
+### Using 'query'
+
+```
+1. Copy the directory 'promql_api' to your working directory. 
+```
 
 ```
+2. Import the API function: query
+from promql_api.prom_ql_api import query
+```
+
+```
+3. have a global or local variable as 'QUERY_STRING'
 QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])']
 ```
 
-The return is:
+```
+4. Store the result set in a list:
+list_of_result_sets = query(QUERY_STRING)
+```
 
 ```
+The return is :
+
 [{'metric': {'cpufreq': '1',
              'endpoint': 'collectd-prometheus',
              'exported_instance': 'otconap7',
-             'instance': '172.25.103.1:9103',
+             'instance': 'ZZ.QQQ.XXX.YY:9103',
              'job': 'collectd',
              'namespace': 'edge1',
              'pod': 'plundering-liger-collectd-wz7xg',
              'service': 'collectd'},
   'value': [1559177169.415, '119727200']}]
 ```
-
-## How to use this API ?
+### Using 'query_range'
 
 ```
 1. Copy the directory 'promql_api' to your working directory. 
 ```
 
 ```
-2. Import the API function: query
-from promql_api.prom_ql_api import query
+2. Import the API function: query_range
+from promql_api.prom_ql_api import query_range
 ```
 
 ```
-3. have a global or local variable as 'QUERY_STRING'
-QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])']
+3. Its the calling application's responsibility to pass on the correct set of paramters. Form a map of required parameters:
+
+map_of_parameters = {'query': 'up', 'start': '2019-06-19T20:10:30.781Z', 'end': '2019-06-19T20:10:45.781Z', 'step': '15s'}
 ```
 
 ```
 4. Store the result set in a list:
-list_of_result_sets = query(QUERY_STRING)
+list_of_result_sets = query_range(map_of_parameters)
+```
+
+```
+The return is:
+
+[[{'metric': {'__name__': 'up',
+              'endpoint': 'cadvisor-prometheus',
+              'instance': 'ZZ.QQQ.XXX.YY:8080',
+              'job': 'cadvisor',
+              'namespace': 'default',
+              'pod': 'cp-cadvisor-69287',
+              'service': 'cadvisor'},
+   'values': [[1560975030.781, '1'], [1560975045.781, '1']]},
+  {'metric': {'__name__': 'up',
+              'endpoint': 'collectd-prometheus',
+              'instance': 'ZZ.QQQ.XXX.YY:9103',
+              'job': 'collectd',
+              'namespace': 'default',
+              'pod': 'vcmts-stats-daemonset-lsqjk',
+              'service': 'collectd'},
+   'values': [[1560975030.781, '1'], [1560975045.781, '1']]},
+  {'metric': {'__name__': 'up',
+              'endpoint': 'metrics',
+              'instance': 'ZZ.QQQ.XXX.YY:9100',
+              'job': 'cp-prometheus-node-exporter',
+              'namespace': 'default',
+              'pod': 'cp-prometheus-node-exporter-sbbb9',
+              'service': 'cp-prometheus-node-exporter'},
+   'values': [[1560975030.781, '1'], [1560975045.781, '1']]},
+  {'metric': {'__name__': 'up',
+              'endpoint': 'metrics',
+              'instance': 'ZZ.QQQ.XXX.YY:9100',
+              'job': 'cp-prometheus-node-exporter',
+              'namespace': 'default',
+              'pod': 'cp-prometheus-node-exporter-mcqtp',
+              'service': 'cp-prometheus-node-exporter'},
+   'values': [[1560975030.781, '1'], [1560975045.781, '1']]}]]
 ```
 
 ## Troubleshooting tips
@@ -58,11 +117,11 @@ list_of_result_sets = query(QUERY_STRING)
  05-30-2019 08:47:53PM ::prom_ql_api.py :: load_and_validate_env_vars :: INFO :: Loading the env variables ...
  05-30-2019 08:47:53PM ::prom_ql_api.py :: load_and_validate_env_vars :: ERROR :: Env var: DATA_ENDPOINT not         found !
  05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: Forming the get request ...
- 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: API request::: URL: http://172.25.103.1:30090/api/v1/    query
+ 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: API request::: URL: http://ZZ.QQQ.XXX.YY:30090/api/v1/    query
  05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: API request::: params: {'query':                         'irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])'}
- 05-30-2019 08:47:53PM ::connectionpool.py :: _new_conn :: DEBUG :: Starting new HTTP connection (1): 172.25.103.1   05-30-2019 08:47:53PM ::connectionpool.py :: _make_request :: DEBUG :: http://172.25.103.1:30090 "GET /api/v1/      query?query=irate%28collectd_cpufreq%7Bexported_instance%3D%22otconap7%22%2Ccpufreq%3D%221%22%7D%5B2m%5D%29 HTTP/1. 1" 200 370
+ 05-30-2019 08:47:53PM ::connectionpool.py :: _new_conn :: DEBUG :: Starting new HTTP connection (1): ZZ.QQQ.XXX.YY   05-30-2019 08:47:53PM ::connectionpool.py :: _make_request :: DEBUG :: http://ZZ.QQQ.XXX.YY:30090 "GET /api/v1/      query?query=irate%28collectd_cpufreq%7Bexported_instance%3D%22otconap7%22%2Ccpufreq%3D%221%22%7D%5B2m%5D%29 HTTP/1. 1" 200 370
  05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: ::::::::::RESULTS:::::::::::::                           irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])
- 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: {'metric': {'cpufreq': '1', 'endpoint': 'collectd-       prometheus', 'exported_instance': 'otconap7', 'instance': '172.25.103.1:9103', 'job': 'collectd', 'namespace':      'edge1', 'pod': 'plundering-liger-collectd-wz7xg', 'service': 'collectd'}, 'value': [1559249299.084, '236300']}
+ 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: {'metric': {'cpufreq': '1', 'endpoint': 'collectd-       prometheus', 'exported_instance': 'otconap7', 'instance': 'ZZ.QQQ.XXX.YY:9103', 'job': 'collectd', 'namespace':      'edge1', 'pod': 'plundering-liger-collectd-wz7xg', 'service': 'collectd'}, 'value': [1559249299.084, '236300']}
  ```
 
  * Tested Error scenario: Configure QUERY_STRING as :
@@ -71,11 +130,11 @@ list_of_result_sets = query(QUERY_STRING)
  ```
  O/P :
  ```
- Check logs..HTTP error occurred: 400 Client Error: Bad Request for url: http://172.25.103.1:30090/api/v1/query?query=collectd_cpu_percent%7Bjob%3D%22collectd%22+exported_instance%3D%22an11-31%22%7D%5B1m%5D
+ Check logs..HTTP error occurred: 400 Client Error: Bad Request for url: http://ZZ.QQQ.XXX.YY:30090/api/v1/query?query=collectd_cpu_percent%7Bjob%3D%22collectd%22+exported_instance%3D%22an11-31%22%7D%5B1m%5D
 [{'metric': {'cpufreq': '1',
              'endpoint': 'collectd-prometheus',
              'exported_instance': 'otconap7',
-             'instance': '172.25.103.1:9103',
+             'instance': 'ZZ.QQQ.XXX.YY:9103',
              'job': 'collectd',
              'namespace': 'edge1',
              'pod': 'plundering-liger-collectd-wz7xg',
diff --git a/vnfs/DAaaS/lib/promql_api/prom_ql_api.py b/vnfs/DAaaS/lib/promql_api/prom_ql_api.py
index 8a5ce508..1dc578e9 100644
--- a/vnfs/DAaaS/lib/promql_api/prom_ql_api.py
+++ b/vnfs/DAaaS/lib/promql_api/prom_ql_api.py
@@ -23,7 +23,8 @@ import requests
 from requests.exceptions import HTTPError
 
 
-API_VERSION = '/api/v1/query'
+QUERY_API_VERSION = '/api/v1/query'
+QUERY_RANGE_API_VERSION = '/api/v1/query_range'
 LIST_OF_ENV_VARIABLES = ["DATA_ENDPOINT"]
 MAP_ENV_VARIABLES = dict()
 LOG = logging.getLogger(__name__)
@@ -34,7 +35,7 @@ def set_log_config():
                     datefmt='%m-%d-%Y %I:%M:%S%p',
                     level=logging.DEBUG,
                     filename='promql_api.log',
-                    filemode='w')
+                    filemode='a')
     LOG.info("Set the log configs.")
 
 
@@ -45,7 +46,7 @@ def load_and_validate_env_vars(list_of_env_vars):
             LOG.info("Found env variable: {} ".format(env_var.upper()))
             MAP_ENV_VARIABLES[env_var.upper()] = environ.get(env_var)
         else:
-            #MAP_ENV_VARIABLES['DATA_ENDPOINT']='http://127.0.0.1:30090' # to be deleted
+            #MAP_ENV_VARIABLES['DATA_ENDPOINT']='http://127.0.0.1:9090' # to be deleted
             LOG.error("Env var: {} not found ! ".format(env_var.upper()))
             raise KeyError("Env variable: {} not found ! ".format(env_var.upper()))
 
@@ -74,7 +75,7 @@ def query(QUERY_STRING):
     params_map = {}
     list_of_result_sets = []
     list_of_substrings.append(MAP_ENV_VARIABLES['DATA_ENDPOINT'])
-    list_of_substrings.append(API_VERSION)
+    list_of_substrings.append(QUERY_API_VERSION)
     url = ''.join(list_of_substrings)
 
     for each_query_string in QUERY_STRING:
@@ -111,3 +112,60 @@ def query(QUERY_STRING):
                     LOG.info(each_result)
                 list_of_result_sets.append(results)
     return list_of_result_sets
+
+
+def validate_parameters(map_of_parameters):
+    for k,v in map_of_parameters.items():
+        if k not in ['query', 'start', 'end', 'step', 'timeout']:
+            LOG.error('Parameter : \'{}\' not supported by query_range'.format(k))
+            LOG.info('Valid parameters :: \'query\', \'start\', \'end\', \'step\', \'timeout\'')
+            raise Exception('Parameter : \'{}\' not supported by query_range. Check logs'.format(k))
+        if not isinstance(k,str):
+            LOG.error(':: Key Paramter : \'{}\' NOT a string! Keys should be string ::'.format(k))
+            raise Exception(':: Key Paramter : \'{}\' NOT a string! Keys should be string ::'.format(k))
+        if not isinstance(v,str):
+            LOG.error(':: Value Paramter of key: \'{}\' NOT a string! Values should be string ::'.format(k))
+            raise Exception(':: Value Paramter of key: \'{}\' NOT a string! Values should be string ::'.format(k))
+    return True
+
+        
+def query_range(map_of_parameters):
+    list_of_result_sets = []
+    set_log_config()
+    if validate_parameters(map_of_parameters):
+        LOG.info(':::Validation of map_of_parameters done::')
+    load_and_validate_env_vars(LIST_OF_ENV_VARIABLES)
+    LOG.info("Forming the query_range request ...")
+    
+    list_of_substrings = []
+    list_of_substrings.append(MAP_ENV_VARIABLES['DATA_ENDPOINT'])
+    list_of_substrings.append(QUERY_RANGE_API_VERSION)
+    url = ''.join(list_of_substrings)
+
+    try:
+        LOG.info('API request::: URL: {} '.format(url))
+        LOG.info('API request::: params: {} '.format(map_of_parameters))
+        response = requests.get(url, params=map_of_parameters)
+        response.raise_for_status() # This might raise HTTPError which is handled in except block
+    except HTTPError as http_err:
+        if response.json()['status'] == "error":
+            LOG.error("::::ERROR OCCURED::::")
+            LOG.error("::::ERROR TYPE:::: {}".format(response.json()['errorType']))
+            LOG.error("::::ERROR:::: {}".format(response.json()['error']))
+        print(f'Check logs..HTTP error occurred: {http_err}')
+    except Exception as err:
+            print(f'Check logs..Other error occurred: {err}')
+    else:
+        if response.json()['status'] == "error":
+            LOG.error("::::ERROR OCCURED!::::")
+            LOG.error("::::ERROR TYPE:::: {}".format(response.json()['errorType']))
+            LOG.error("::::ERROR:::: {}".format(response.json()['error']))
+            list_of_result_sets.append(response.json()['error'])
+            list_of_result_sets.append(dict({'error':response.json()['error'],
+                                                'errorType' : response.json()['errorType']}))
+        else:
+            results = response.json()['data']['result']
+            LOG.info('::::::::::RESULTS OF QUERY_RANGE::::::::::::: {}'.format(map_of_parameters))
+            list_of_result_sets.append(results)
+
+    return list_of_result_sets
\ No newline at end of file
diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py b/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py
index fbbf497e..2c85f1f8 100644
--- a/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py
+++ b/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py
@@ -17,19 +17,32 @@
 
 
 from promql_api.prom_ql_api import query
+from promql_api.prom_ql_api import query_range
 import pprint
 
 QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])']
+MAP_OF_PARAMETERS = {'query': 'up', 'start': '2019-06-19T20:10:30.781Z', 'end': '2019-06-19T20:10:45.781Z', 'step': '15s'}
 
 #Other examples
 #QUERY_STRING = [ 'irate(http_requests_total{code="200"}[1m])', 'collectd_cpu_percent{job="collectd", exported_instance="otconap7"}[1m]' ]
 #QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])', 'go_info']
+#MAP_OF_PARAMETERS = {'query': 'up', 'start': '2019-06-19T20:10:30.781Z', 'end': '2019-06-19T20:10:45.781Z', 'step': '15s', 'timeout':'600s'}
+
+def demo_query():
+        list_of_result_sets = query(QUERY_STRING)
+        if list_of_result_sets:
+                for each_result in list_of_result_sets:
+                    pprint.pprint(each_result)
+
+def demo_query_range():
+        list_of_result_sets = query_range(MAP_OF_PARAMETERS)
+        pprint.pprint(list_of_result_sets)
+
 
 def main():
-    list_of_result_sets = query(QUERY_STRING)
-    if list_of_result_sets:
-        for each_result in list_of_result_sets:
-                pprint.pprint(each_result)
+        demo_query()
+        demo_query_range()
 
+    
 if __name__ == "__main__":
-     main()
+        main()