From f604d490d97e3b580029bd4cfade324174555dc2 Mon Sep 17 00:00:00 2001 From: Petr Ospalý Date: Mon, 4 Mar 2019 06:56:33 +0100 Subject: Add default logging settings for docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Default configuration of logging for docker daemon. - New ansible module for generic handling of JSON files. - New setting in ansible.cfg: jinja2_native = True To preserve double-quotes in json values (OOM-1698). Issue-ID: OOM-1681 Change-Id: I8f8e19ebc290fd48a63146e96f418b98344e4433 Signed-off-by: Petr Ospalý --- ansible/docker/Dockerfile | 1 + ansible/library/json_add.py | 90 --------- ansible/library/json_mod.py | 328 +++++++++++++++++++++++++++++++++ ansible/roles/docker/defaults/main.yml | 4 + ansible/roles/docker/tasks/main.yml | 14 +- 5 files changed, 345 insertions(+), 92 deletions(-) delete mode 100644 ansible/library/json_add.py create mode 100644 ansible/library/json_mod.py create mode 100644 ansible/roles/docker/defaults/main.yml (limited to 'ansible') diff --git a/ansible/docker/Dockerfile b/ansible/docker/Dockerfile index 8056b9fc..ca6dbfb2 100644 --- a/ansible/docker/Dockerfile +++ b/ansible/docker/Dockerfile @@ -25,6 +25,7 @@ RUN apk --no-cache update \ ansible==$ansible_version \ jmespath \ netaddr \ + jsonpointer \ && apk del build-dependencies && rm -rf /var/cache/apk/* && rm -rf /root/.cache ENV ANSIBLE_HOST_KEY_CHECKING false diff --git a/ansible/library/json_add.py b/ansible/library/json_add.py deleted file mode 100644 index 6aad2d7c..00000000 --- a/ansible/library/json_add.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/python - -from ansible.module_utils.basic import AnsibleModule -import json -import os - -DOCUMENTATION=""" ---- -module: json_add -descritption: - - This module will search top level objects in json and adds specified - value into list for specified key. - - If file does not exists module will create it automatically. - -options: - path: - required: true - aliases=[name, destfile, dest] - description: - - The json file to modify. - key: - required: true - description: - - Top level object. - value: - required: true - description: - - Value to add to specified key. -""" - -def load_json(path): - if os.path.exists(path): - with open(path, 'r') as f: - return json.load(f) - else: - return {} - -def value_is_set(path, key, value, json_obj): - return value in json_obj.get(key, []) - -def insert_to_json(path, key, value, check_mode=False): - json_obj = load_json(path) - if not value_is_set(path, key, value, json_obj): - if not check_mode: - json_obj.setdefault(key, []).append(value) - store_json(path, json_obj) - return True, 'Value %s added to %s.' % (value, key) - else: - return False, '' - -def store_json(path, json_obj): - with open(path, 'w') as f: - json.dump(json_obj, f, indent=4) - -def check_file_attrs(module, changed, message, diff): - file_args = module.load_file_common_arguments(module.params) - if module.set_fs_attributes_if_different(file_args, False, diff=diff): - - if changed: - message += ' ' - changed = True - message += 'File attributes changed.' - - return changed, message - -def run_module(): - module = AnsibleModule( - argument_spec=dict( - path=dict(type='path', required=True, aliases=['name', 'destfile', 'dest']), - key=dict(type='str', required=True), - value=dict(type='str', required=True), - ), - add_file_common_args=True, - supports_check_mode=True - ) - params = module.params - path = params['path'] - key = params['key'] - value = params['value'] - try: - changed, msg = insert_to_json(path, key, value, module.check_mode) - fs_diff = {} - changed, msg = check_file_attrs(module, changed, msg, fs_diff) - module.exit_json(changed=changed, msg=msg, file_attr_diff=fs_diff) - except IOError as e: - module.fail_json(msg=e.msg) - -if __name__ == '__main__': - run_module() - diff --git a/ansible/library/json_mod.py b/ansible/library/json_mod.py new file mode 100644 index 00000000..1a95c75b --- /dev/null +++ b/ansible/library/json_mod.py @@ -0,0 +1,328 @@ +#!/usr/bin/python + +from ansible.module_utils.basic import AnsibleModule + +import os +import copy +import json + +try: + import jsonpointer +except ImportError: + jsonpointer = None + +DOCUMENTATION = """ +--- +module: json_mod +short_description: Modifies json data inside a file +description: + - This module modifies a file containing a json. + - It is leveraging jsonpointer module implementing RFC6901: + https://pypi.org/project/jsonpointer/ + https://tools.ietf.org/html/rfc6901 + - If the file does not exist the module will create it automatically. + +options: + path: + description: + - The json file to modify. + required: true + aliases: + - name + - destfile + - dest + key: + description: + - Pointer to the key inside the json object. + - You can leave out the leading slash '/'. It will be prefixed by the + module for convenience ('key' equals '/key'). + - Empty key '' designates the whole JSON document (RFC6901) + - Key '/' is valid too and it translates to '' ("": "some value"). + - The last object in the pointer can be missing but the intermediary + objects must exist. + required: true + value: + description: + - Value to be added/changed for the key specified by pointer. + - In the case of 'state = absent' the module will delete those elements + described in the value. If the whole key/value should be deleted then + value must be set to the empty string '' ! + required: true + state: + description: + - It states either that the combination of key and value should be + present or absent. + - If 'present' then the exact results depends on 'action' argument. + - If 'absent' and key does not exists - no change, if does exist but + 'value' is unapplicable (old value is dict, but new is not), then the + module will raise error. Special 'value' for state 'absent' is an empty + string '' (read above). If 'value' is applicable (both key and value is + dict or list) then it will remove only those explicitly named elements. + Please beware that if you want to remove key/value pairs from dict then + you must provide as 'value' a valid dict - that means key/value pair(s) + in curls {}. Here you can use just some dummy value like "". The values + can differ, the key/value pair will be deleted if key matches. + For example to delete key "xyz" from json object, you must provide + 'value' similar to this: { "key": ""} + required: false + default: present + choices: + - present + - absent + action: + description: + - It modifies a presence of the key/value pair when state is 'present' + otherwise is ignored. + - 'add' is default and means that combination of key/value will be added + if not already there. If there is already an old value then it is + expected that the old value and the new value are of the same type. + Otherwise the module will fail. By the same type we mean that both of + them are either scalars (strings, numbers), lists or dicts. + - In the case of scalar values everything is simple - if there is already + a value, nothing happens. + - In the case of lists the module ensures that all components of the new + value list are present in the result - it will extend an old value list + with the elements of the new value list. + - In the case of dicts the missing key/value pairs are added but those + already present are preserved - it will NOT overwrite old values. + - 'Update' is identical to 'add', but it WILL overwrite old values. For + list values this has no meaning, so it behaves like add - it simply + merges two lists (extends the old with new). + - 'replace' will (re)create key/value combination from scratch - it means + that the old value is completely discarded if there is any. + required: false + default: add + choices: + - add + - update + - replace +""" + + +def load_json(path): + if os.path.exists(path): + with open(path, 'r') as f: + return json.load(f) + else: + return {} + + +def store_json(path, json_data): + with open(path, 'w') as f: + json.dump(json_data, f, indent=4) + f.write("\n") + + +def modify_json(json_data, pointer, json_value, state='present', action='add'): + is_root = False # special treatment - we cannot modify reference in place + key_exists = False + + try: + value = json.loads(json_value) + except Exception: + value = None + + if state == 'present': + if action not in ['add', 'update', 'replace']: + raise ValueError + elif state == 'absent': + pass + else: + raise ValueError + + # we store the original json document to compare it later + original_json_data = copy.deepcopy(json_data) + + try: + target = jsonpointer.resolve_pointer(json_data, pointer) + if pointer == '': + is_root = True + key_exists = True + except jsonpointer.JsonPointerException: + key_exists = False + + if key_exists: + if state == "present": + if action == "add": + if isinstance(target, dict) and isinstance(value, dict): + # we keep old values and only append new ones + value.update(target) + result = jsonpointer.set_pointer(json_data, + pointer, + value, + inplace=(not is_root)) + if is_root: + json_data = result + elif isinstance(target, list) and isinstance(value, list): + # we just append new items to the list + for item in value: + if item not in target: + target.append(item) + elif ((not isinstance(target, dict)) and + (not isinstance(target, list))): + # 'add' does not overwrite + pass + else: + raise ValueError + elif action == "update": + if isinstance(target, dict) and isinstance(value, dict): + # we append new values and overwrite the old ones + target.update(value) + elif isinstance(target, list) and isinstance(value, list): + # we just append new items to the list - same as with 'add' + for item in value: + if item not in target: + target.append(item) + elif ((not isinstance(target, dict)) and + (not isinstance(target, list))): + # 'update' DOES overwrite + if value is not None: + result = jsonpointer.set_pointer(json_data, + pointer, + value) + elif target != json_value: + result = jsonpointer.set_pointer(json_data, + pointer, + json_value) + else: + raise ValueError + else: + raise ValueError + elif action == "replace": + # simple case when we don't care what was there before (almost) + if value is not None: + result = jsonpointer.set_pointer(json_data, + pointer, + value, + inplace=(not is_root)) + else: + result = jsonpointer.set_pointer(json_data, + pointer, + json_value, + inplace=(not is_root)) + if is_root: + json_data = result + else: + raise ValueError + elif state == "absent": + # we will delete the elements in the object or object itself + if is_root: + if json_value == '': + # we just return empty json + json_data = {} + elif isinstance(target, dict) and isinstance(value, dict): + for key in value: + target.pop(key, None) + else: + raise ValueError + else: + # we must take a step back in the pointer, so we can edit it + ppointer = pointer.split('/') + to_delete = ppointer.pop() + ppointer = '/'.join(ppointer) + ptarget = jsonpointer.resolve_pointer(json_data, ppointer) + if (((not isinstance(target, dict)) and + (not isinstance(target, list)) and + json_value == '') or + (isinstance(target, dict) or + isinstance(target, list)) and + json_value == ''): + # we simply delete the key with it's value (whatever it is) + ptarget.pop(to_delete, None) + target = ptarget # piece of self-defense + elif isinstance(target, dict) and isinstance(value, dict): + for key in value: + target.pop(key, None) + elif isinstance(target, list) and isinstance(value, list): + for item in value: + try: + target.remove(item) + except ValueError: + pass + else: + raise ValueError + else: + raise ValueError + else: + # the simplest case - nothing was there before and pointer is not root + # because in that case we would have key_exists = true + if state == 'present': + if value is not None: + result = jsonpointer.set_pointer(json_data, + pointer, + value) + else: + result = jsonpointer.set_pointer(json_data, + pointer, + json_value) + + if json_data != original_json_data: + changed = True + else: + changed = False + + if changed: + msg = "JSON object '%s' was updated" % pointer + else: + msg = "No change to JSON object '%s'" % pointer + + return json_data, changed, msg + + +def main(): + module = AnsibleModule( + argument_spec=dict( + path=dict(type='path', required=True, + aliases=['name', 'destfile', 'dest']), + key=dict(type='str', required=True), + value=dict(type='str', required=True), + state=dict(default='present', choices=['present', 'absent']), + action=dict(required=False, default='add', + choices=['add', + 'update', + 'replace']), + ), + supports_check_mode=True + ) + + if jsonpointer is None: + module.fail_json(msg='jsonpointer module is not available') + + path = module.params['path'] + pointer = module.params['key'] + value = module.params['value'] + state = module.params['state'] + action = module.params['action'] + + if pointer == '' or pointer == '/': + pass + elif not pointer.startswith("/"): + pointer = "/" + pointer + + try: + json_data = load_json(path) + except Exception as err: + module.fail_json(msg=str(err)) + + try: + json_data, changed, msg = modify_json(json_data, + pointer, + value, + state, + action) + except jsonpointer.JsonPointerException as err: + module.fail_json(msg=str(err)) + except ValueError as err: + module.fail_json(msg="Wrong usage of state, action and/or key/value") + + try: + if not module.check_mode and changed: + store_json(path, json_data) + except IOError as err: + module.fail_json(msg=str(err)) + + module.exit_json(changed=changed, msg=msg) + + +if __name__ == '__main__': + main() diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml new file mode 100644 index 00000000..1922f64b --- /dev/null +++ b/ansible/roles/docker/defaults/main.yml @@ -0,0 +1,4 @@ +--- +docker: + log_max_size: 100m + log_max_file: 3 diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index 09e790a4..16b7002f 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -16,11 +16,21 @@ path: /etc/docker state: directory +- name: Setup docker container logging settings + json_mod: + path: /etc/docker/daemon.json + key: '' # the whole JSON document per https://tools.ietf.org/html/rfc6901 + # "value" must be wrapped in single quote "'" with extra space in front of "{" (ansible workaround) + # reference: https://stackoverflow.com/questions/31969872 + value: ' { "log-driver": "json-file", "log-opts": { "max-size": "{{ docker.log_max_size }}", "max-file": "{{ docker.log_max_file }}" } }' + - name: Setup docker dns settings - json_add: + json_mod: path: /etc/docker/daemon.json key: dns - value: "{{ hostvars[groups.infrastructure[0]].cluster_ip }}" + # "value" must be wrapped in single quote "'" with extra space in front of "[" (ansible workaround) + # reference: https://stackoverflow.com/questions/31969872 + value: ' [ "{{ hostvars[groups.infrastructure[0]].cluster_ip }}" ]' notify: - Restart Docker -- cgit 1.2.3-korg