diff options
| -rw-r--r-- | ansible/docker/Dockerfile | 1 | ||||
| -rw-r--r-- | ansible/library/json_add.py | 90 | ||||
| -rw-r--r-- | ansible/library/json_mod.py | 328 | ||||
| -rw-r--r-- | ansible/roles/certificates/tasks/generate-certificates.yml | 34 | ||||
| -rw-r--r-- | ansible/roles/certificates/templates/v3.ext.j2 | 9 | ||||
| -rw-r--r-- | ansible/roles/docker/defaults/main.yml | 4 | ||||
| -rw-r--r-- | ansible/roles/docker/tasks/main.yml | 14 | ||||
| -rwxr-xr-x | build/package.sh | 29 | ||||
| -rw-r--r-- | docs/BuildGuide.rst | 2 | ||||
| -rw-r--r-- | patches/casablanca.patch | 40 |
10 files changed, 403 insertions, 148 deletions
diff --git a/ansible/docker/Dockerfile b/ansible/docker/Dockerfile index 8056b9fc..ca6dbfb2 100644 --- a/ansible/docker/Dockerfile +++ b/ansible/docker/Dockerfile @@ -25,6 +25,7 @@ RUN apk --no-cache update \ ansible==$ansible_version \ jmespath \ netaddr \ + jsonpointer \ && apk del build-dependencies && rm -rf /var/cache/apk/* && rm -rf /root/.cache ENV ANSIBLE_HOST_KEY_CHECKING false diff --git a/ansible/library/json_add.py b/ansible/library/json_add.py deleted file mode 100644 index 6aad2d7c..00000000 --- a/ansible/library/json_add.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/python - -from ansible.module_utils.basic import AnsibleModule -import json -import os - -DOCUMENTATION=""" ---- -module: json_add -descritption: - - This module will search top level objects in json and adds specified - value into list for specified key. - - If file does not exists module will create it automatically. - -options: - path: - required: true - aliases=[name, destfile, dest] - description: - - The json file to modify. - key: - required: true - description: - - Top level object. - value: - required: true - description: - - Value to add to specified key. -""" - -def load_json(path): - if os.path.exists(path): - with open(path, 'r') as f: - return json.load(f) - else: - return {} - -def value_is_set(path, key, value, json_obj): - return value in json_obj.get(key, []) - -def insert_to_json(path, key, value, check_mode=False): - json_obj = load_json(path) - if not value_is_set(path, key, value, json_obj): - if not check_mode: - json_obj.setdefault(key, []).append(value) - store_json(path, json_obj) - return True, 'Value %s added to %s.' % (value, key) - else: - return False, '' - -def store_json(path, json_obj): - with open(path, 'w') as f: - json.dump(json_obj, f, indent=4) - -def check_file_attrs(module, changed, message, diff): - file_args = module.load_file_common_arguments(module.params) - if module.set_fs_attributes_if_different(file_args, False, diff=diff): - - if changed: - message += ' ' - changed = True - message += 'File attributes changed.' - - return changed, message - -def run_module(): - module = AnsibleModule( - argument_spec=dict( - path=dict(type='path', required=True, aliases=['name', 'destfile', 'dest']), - key=dict(type='str', required=True), - value=dict(type='str', required=True), - ), - add_file_common_args=True, - supports_check_mode=True - ) - params = module.params - path = params['path'] - key = params['key'] - value = params['value'] - try: - changed, msg = insert_to_json(path, key, value, module.check_mode) - fs_diff = {} - changed, msg = check_file_attrs(module, changed, msg, fs_diff) - module.exit_json(changed=changed, msg=msg, file_attr_diff=fs_diff) - except IOError as e: - module.fail_json(msg=e.msg) - -if __name__ == '__main__': - run_module() - diff --git a/ansible/library/json_mod.py b/ansible/library/json_mod.py new file mode 100644 index 00000000..1a95c75b --- /dev/null +++ b/ansible/library/json_mod.py @@ -0,0 +1,328 @@ +#!/usr/bin/python + +from ansible.module_utils.basic import AnsibleModule + +import os +import copy +import json + +try: + import jsonpointer +except ImportError: + jsonpointer = None + +DOCUMENTATION = """ +--- +module: json_mod +short_description: Modifies json data inside a file +description: + - This module modifies a file containing a json. + - It is leveraging jsonpointer module implementing RFC6901: + https://pypi.org/project/jsonpointer/ + https://tools.ietf.org/html/rfc6901 + - If the file does not exist the module will create it automatically. + +options: + path: + description: + - The json file to modify. + required: true + aliases: + - name + - destfile + - dest + key: + description: + - Pointer to the key inside the json object. + - You can leave out the leading slash '/'. It will be prefixed by the + module for convenience ('key' equals '/key'). + - Empty key '' designates the whole JSON document (RFC6901) + - Key '/' is valid too and it translates to '' ("": "some value"). + - The last object in the pointer can be missing but the intermediary + objects must exist. + required: true + value: + description: + - Value to be added/changed for the key specified by pointer. + - In the case of 'state = absent' the module will delete those elements + described in the value. If the whole key/value should be deleted then + value must be set to the empty string '' ! + required: true + state: + description: + - It states either that the combination of key and value should be + present or absent. + - If 'present' then the exact results depends on 'action' argument. + - If 'absent' and key does not exists - no change, if does exist but + 'value' is unapplicable (old value is dict, but new is not), then the + module will raise error. Special 'value' for state 'absent' is an empty + string '' (read above). If 'value' is applicable (both key and value is + dict or list) then it will remove only those explicitly named elements. + Please beware that if you want to remove key/value pairs from dict then + you must provide as 'value' a valid dict - that means key/value pair(s) + in curls {}. Here you can use just some dummy value like "". The values + can differ, the key/value pair will be deleted if key matches. + For example to delete key "xyz" from json object, you must provide + 'value' similar to this: { "key": ""} + required: false + default: present + choices: + - present + - absent + action: + description: + - It modifies a presence of the key/value pair when state is 'present' + otherwise is ignored. + - 'add' is default and means that combination of key/value will be added + if not already there. If there is already an old value then it is + expected that the old value and the new value are of the same type. + Otherwise the module will fail. By the same type we mean that both of + them are either scalars (strings, numbers), lists or dicts. + - In the case of scalar values everything is simple - if there is already + a value, nothing happens. + - In the case of lists the module ensures that all components of the new + value list are present in the result - it will extend an old value list + with the elements of the new value list. + - In the case of dicts the missing key/value pairs are added but those + already present are preserved - it will NOT overwrite old values. + - 'Update' is identical to 'add', but it WILL overwrite old values. For + list values this has no meaning, so it behaves like add - it simply + merges two lists (extends the old with new). + - 'replace' will (re)create key/value combination from scratch - it means + that the old value is completely discarded if there is any. + required: false + default: add + choices: + - add + - update + - replace +""" + + +def load_json(path): + if os.path.exists(path): + with open(path, 'r') as f: + return json.load(f) + else: + return {} + + +def store_json(path, json_data): + with open(path, 'w') as f: + json.dump(json_data, f, indent=4) + f.write("\n") + + +def modify_json(json_data, pointer, json_value, state='present', action='add'): + is_root = False # special treatment - we cannot modify reference in place + key_exists = False + + try: + value = json.loads(json_value) + except Exception: + value = None + + if state == 'present': + if action not in ['add', 'update', 'replace']: + raise ValueError + elif state == 'absent': + pass + else: + raise ValueError + + # we store the original json document to compare it later + original_json_data = copy.deepcopy(json_data) + + try: + target = jsonpointer.resolve_pointer(json_data, pointer) + if pointer == '': + is_root = True + key_exists = True + except jsonpointer.JsonPointerException: + key_exists = False + + if key_exists: + if state == "present": + if action == "add": + if isinstance(target, dict) and isinstance(value, dict): + # we keep old values and only append new ones + value.update(target) + result = jsonpointer.set_pointer(json_data, + pointer, + value, + inplace=(not is_root)) + if is_root: + json_data = result + elif isinstance(target, list) and isinstance(value, list): + # we just append new items to the list + for item in value: + if item not in target: + target.append(item) + elif ((not isinstance(target, dict)) and + (not isinstance(target, list))): + # 'add' does not overwrite + pass + else: + raise ValueError + elif action == "update": + if isinstance(target, dict) and isinstance(value, dict): + # we append new values and overwrite the old ones + target.update(value) + elif isinstance(target, list) and isinstance(value, list): + # we just append new items to the list - same as with 'add' + for item in value: + if item not in target: + target.append(item) + elif ((not isinstance(target, dict)) and + (not isinstance(target, list))): + # 'update' DOES overwrite + if value is not None: + result = jsonpointer.set_pointer(json_data, + pointer, + value) + elif target != json_value: + result = jsonpointer.set_pointer(json_data, + pointer, + json_value) + else: + raise ValueError + else: + raise ValueError + elif action == "replace": + # simple case when we don't care what was there before (almost) + if value is not None: + result = jsonpointer.set_pointer(json_data, + pointer, + value, + inplace=(not is_root)) + else: + result = jsonpointer.set_pointer(json_data, + pointer, + json_value, + inplace=(not is_root)) + if is_root: + json_data = result + else: + raise ValueError + elif state == "absent": + # we will delete the elements in the object or object itself + if is_root: + if json_value == '': + # we just return empty json + json_data = {} + elif isinstance(target, dict) and isinstance(value, dict): + for key in value: + target.pop(key, None) + else: + raise ValueError + else: + # we must take a step back in the pointer, so we can edit it + ppointer = pointer.split('/') + to_delete = ppointer.pop() + ppointer = '/'.join(ppointer) + ptarget = jsonpointer.resolve_pointer(json_data, ppointer) + if (((not isinstance(target, dict)) and + (not isinstance(target, list)) and + json_value == '') or + (isinstance(target, dict) or + isinstance(target, list)) and + json_value == ''): + # we simply delete the key with it's value (whatever it is) + ptarget.pop(to_delete, None) + target = ptarget # piece of self-defense + elif isinstance(target, dict) and isinstance(value, dict): + for key in value: + target.pop(key, None) + elif isinstance(target, list) and isinstance(value, list): + for item in value: + try: + target.remove(item) + except ValueError: + pass + else: + raise ValueError + else: + raise ValueError + else: + # the simplest case - nothing was there before and pointer is not root + # because in that case we would have key_exists = true + if state == 'present': + if value is not None: + result = jsonpointer.set_pointer(json_data, + pointer, + value) + else: + result = jsonpointer.set_pointer(json_data, + pointer, + json_value) + + if json_data != original_json_data: + changed = True + else: + changed = False + + if changed: + msg = "JSON object '%s' was updated" % pointer + else: + msg = "No change to JSON object '%s'" % pointer + + return json_data, changed, msg + + +def main(): + module = AnsibleModule( + argument_spec=dict( + path=dict(type='path', required=True, + aliases=['name', 'destfile', 'dest']), + key=dict(type='str', required=True), + value=dict(type='str', required=True), + state=dict(default='present', choices=['present', 'absent']), + action=dict(required=False, default='add', + choices=['add', + 'update', + 'replace']), + ), + supports_check_mode=True + ) + + if jsonpointer is None: + module.fail_json(msg='jsonpointer module is not available') + + path = module.params['path'] + pointer = module.params['key'] + value = module.params['value'] + state = module.params['state'] + action = module.params['action'] + + if pointer == '' or pointer == '/': + pass + elif not pointer.startswith("/"): + pointer = "/" + pointer + + try: + json_data = load_json(path) + except Exception as err: + module.fail_json(msg=str(err)) + + try: + json_data, changed, msg = modify_json(json_data, + pointer, + value, + state, + action) + except jsonpointer.JsonPointerException as err: + module.fail_json(msg=str(err)) + except ValueError as err: + module.fail_json(msg="Wrong usage of state, action and/or key/value") + + try: + if not module.check_mode and changed: + store_json(path, json_data) + except IOError as err: + module.fail_json(msg=str(err)) + + module.exit_json(changed=changed, msg=msg) + + +if __name__ == '__main__': + main() diff --git a/ansible/roles/certificates/tasks/generate-certificates.yml b/ansible/roles/certificates/tasks/generate-certificates.yml index ac8fe1e3..9bf75fff 100644 --- a/ansible/roles/certificates/tasks/generate-certificates.yml +++ b/ansible/roles/certificates/tasks/generate-certificates.yml @@ -66,25 +66,19 @@ extended_key_usage: - serverAuth subject_alt_name: - "{{ simulated_hosts | map('regex_replace', '(.*)', 'DNS:\\1') | list }}" + "{{ all_simulated_hosts | map('regex_replace', '(.*)', 'DNS:\\1') | list }}" -- name: Generate v3 extension config file - template: - src: v3.ext.j2 - dest: "{{ certificates_local_dir }}/v3.ext" - -# Signing certificate is added to Ansible in version 2.7 (release date 04.10.2018) -# Currently using 2.6.3 - name: Sign Nexus certificate - command: > - openssl - x509 - -req - -in "{{ certificates_local_dir }}/nexus_server.csr" - -extfile "{{ certificates_local_dir }}/v3.ext" - -CA "{{ certificates_local_dir }}/rootCA.crt" - -CAkey "{{ certificates_local_dir }}/rootCA.key" - -CAcreateserial - -out "{{ certificates_local_dir }}/nexus_server.crt" - -days 3650 - -sha256 + openssl_certificate: + provider: ownca + path: "{{ certificates_local_dir }}/nexus_server.crt" + csr_path: "{{ certificates_local_dir }}/nexus_server.csr" + ownca_path: "{{ certificates_local_dir }}/rootCA.crt" + ownca_privatekey_path: "{{ certificates_local_dir }}/rootCA.key" + key_usage: + - digitalSignature + - nonRepudiation + - keyEncipherment + - dataEncipherment + subject_alt_name: + "{{ all_simulated_hosts | map('regex_replace', '(.*)', 'DNS:\\1') | list }}" diff --git a/ansible/roles/certificates/templates/v3.ext.j2 b/ansible/roles/certificates/templates/v3.ext.j2 deleted file mode 100644 index 7be946fd..00000000 --- a/ansible/roles/certificates/templates/v3.ext.j2 +++ /dev/null @@ -1,9 +0,0 @@ -authorityKeyIdentifier=keyid,issuer -basicConstraints=CA:FALSE -keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment -subjectAltName = @alt_names - -[alt_names] -{% for name in all_simulated_hosts -%} - DNS.{{ loop.index }} = {{ name }} -{% endfor %} diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml new file mode 100644 index 00000000..1922f64b --- /dev/null +++ b/ansible/roles/docker/defaults/main.yml @@ -0,0 +1,4 @@ +--- +docker: + log_max_size: 100m + log_max_file: 3 diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index 09e790a4..16b7002f 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -16,11 +16,21 @@ path: /etc/docker state: directory +- name: Setup docker container logging settings + json_mod: + path: /etc/docker/daemon.json + key: '' # the whole JSON document per https://tools.ietf.org/html/rfc6901 + # "value" must be wrapped in single quote "'" with extra space in front of "{" (ansible workaround) + # reference: https://stackoverflow.com/questions/31969872 + value: ' { "log-driver": "json-file", "log-opts": { "max-size": "{{ docker.log_max_size }}", "max-file": "{{ docker.log_max_file }}" } }' + - name: Setup docker dns settings - json_add: + json_mod: path: /etc/docker/daemon.json key: dns - value: "{{ hostvars[groups.infrastructure[0]].cluster_ip }}" + # "value" must be wrapped in single quote "'" with extra space in front of "[" (ansible workaround) + # reference: https://stackoverflow.com/questions/31969872 + value: ' [ "{{ hostvars[groups.infrastructure[0]].cluster_ip }}" ]' notify: - Restart Docker diff --git a/build/package.sh b/build/package.sh index cd2531c4..c527db2e 100755 --- a/build/package.sh +++ b/build/package.sh @@ -33,8 +33,15 @@ crash () { usage () { echo "Usage:" - echo " ./$(basename $0) <project_name> <version> <packaging_target_dir> [--conf <file>]" - echo "Example: ./$(basename $0) myproject 1.0.1 /tmp/package --conf ~/myproject.conf" + echo " ./$(basename $0) <project_name> <version> <packaging_target_dir> [--conf <file>] [--force]" + echo "" + echo "Options:" + echo " --force Remove packaging_target_dir if exists prior to script execution" + echo " --conf Custom configuration file path for script" + echo "" + echo "Example:" + echo " ./$(basename $0) myproject 1.0.1 /tmp/package --conf ~/myproject.conf" + echo "" echo "packaging_target_dir will be created if does not exist. All tars will be produced into it." } @@ -194,17 +201,20 @@ APPLICATION_FILES_IN_PACKAGE="ansible/application" HELM_CHARTS_DIR_IN_PACKAGE="${APPLICATION_FILES_IN_PACKAGE}/helm_charts" if [ "$#" -lt 3 ]; then - echo "Missing some mandatory parameter!" + echo "Missing some mandatory arguments!" usage exit 1 fi CONF_FILE="" +FORCE_REMOVE=0 for arg in "$@"; do shift case "$arg" in -c|--conf) CONF_FILE="$1" ;; + --force) + FORCE_REMOVE=1 ;; *) set -- "$@" "$arg" esac @@ -224,10 +234,17 @@ pushd ${LOCAL_PATH} # checking bash capability of parsing arrays whotest[0]='test' || (crash 3 "Arrays not supported in this version of bash.") +# Prepare output directory for our packaging +# Check target dir exists and is not empty +if [ -d ${PACKAGING_TARGET_DIR} ] && [ "$(ls -A ${PACKAGING_TARGET_DIR})" ]; then + if [ ${FORCE_REMOVE} -eq 0 ]; then + crash 1 "Target directory not empty. Use --force to overwrite it." + else + rm -rf ${PACKAGING_TARGET_DIR} + fi +fi -# Prepare output directory for our packaging and create all tars - -rm -rf ${PACKAGING_TARGET_DIR} +# Create all tars build_sw_artifacts create_sw_package create_resource_package diff --git a/docs/BuildGuide.rst b/docs/BuildGuide.rst index 96c1fbf2..8857945c 100644 --- a/docs/BuildGuide.rst +++ b/docs/BuildGuide.rst @@ -62,7 +62,7 @@ Then it is necessary to clone all installer and build related repositories and p # prepare the onap build directory structure cd /tmp - git clone -b casablanca https://gerrit.onap.org/r/oom/offline-installer + git clone https://gerrit.onap.org/r/oom/offline-installer onap-offline cd onap-offline Part 2. Download artifacts for offline installer diff --git a/patches/casablanca.patch b/patches/casablanca.patch index e0ea0ec5..7f9d2068 100644 --- a/patches/casablanca.patch +++ b/patches/casablanca.patch @@ -259,7 +259,7 @@ index 6564e798..53c8b600 100644 imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} name: {{ include "common.name" . }}-readiness +{{ include "common.update-system-ca-store-ubuntu" . | indent 6 }} -+{{ include "policy.update-policy-keystore" . | indent 6 }} ++{{ include "policy.update-policy-truststore" . | indent 6 }} containers: - name: {{ include "common.name" . }} image: "{{ include "common.repository" . }}/{{ .Values.image }}" @@ -269,7 +269,7 @@ index 6564e798..53c8b600 100644 volumeMounts: +{{ include "common.cacert-mount-ubuntu" . | indent 10 }} +{{ include "common.system-ca-store-mount-ubuntu" . | indent 10 }} -+{{ include "policy.keystore-mount" . | indent 10 }} ++{{ include "policy.truststore-mount" . | indent 10 }} - mountPath: /etc/localtime name: localtime readOnly: true @@ -279,7 +279,7 @@ index 6564e798..53c8b600 100644 volumes: +{{ include "common.cacert-volume" . | indent 8 }} +{{ include "common.system-ca-store-volume" . | indent 8 }} -+{{ include "policy.keystore-storage-volume" . | indent 8 }} ++{{ include "policy.truststore-storage-volume" . | indent 8 }} - name: localtime hostPath: path: /etc/localtime @@ -337,46 +337,46 @@ index 00000000..81ba09bc +# +# COPYRIGHT NOTICE ENDS HERE + -+#This template creates a volume for storing policy-keystore with imported ca. ++#This template creates a volume for storing policy-truststore with imported ca. +#It is ignored if cacert was not given. -+{{- define "policy.keystore-storage-volume" }} ++{{- define "policy.truststore-storage-volume" }} +{{- if .Values.global.cacert }} -+- name: keystore-storage ++- name: truststore-storage + emptyDir: +{{- end }} +{{- end }} + -+#This template mounts policy-keystore in appropriate place for policy components to take it. ++#This template mounts policy-truststore in appropriate place for policy components to take it. +#It is ignored if cacert is not given. -+{{- define "policy.keystore-mount" }} ++{{- define "policy.truststore-mount" }} +{{- if .Values.global.cacert }} -+- mountPath: "/tmp/policy-install/config/policy-keystore" -+ name: keystore-storage -+ subPath: policy-keystore ++- mountPath: "/tmp/policy-install/config/policy-truststore" ++ name: truststore-storage ++ subPath: policy-truststore +{{- end }} +{{- end }} + -+#This will extract a policy keystore and then import ++#This will extract a policy truststore and then import +#the root cacert of offline nexus into it. -+#This template expects a volume named keystore-storage where policy-keystore will be put. ++#This template expects a volume named truststore-storage where policy-truststore will be put. +#It also expects volume named cacert where the file "certificate" will contain the cert to import. +#Template is ignored if ca certificate not given. -+{{- define "policy.update-policy-keystore" }} ++{{- define "policy.update-policy-truststore" }} +{{- if .Values.global.cacert }} +- command: + - "/bin/bash" + - "-c" + - | + set -e -+ tar -xzf base-*.tar.gz etc/ssl/policy-keystore -+ cp etc/ssl/policy-keystore keystore-storage/ -+ keytool -import -keystore keystore-storage/policy-keystore -storepass "Pol1cy_0nap" -noprompt -file /usr/local/share/ca-certificates/cacert.crt -+ name: update-policy-keystore ++ tar -xzf base-*.tar.gz etc/ssl/policy-truststore ++ cp etc/ssl/policy-truststore truststore-storage/ ++ keytool -import -keystore truststore-storage/policy-truststore -storepass "Pol1cy_0nap" -noprompt -file /usr/local/share/ca-certificates/cacert.crt ++ name: update-policy-truststore + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + image: {{ include "common.repository" . }}/{{ .Values.image }} + volumeMounts: -+ - mountPath: "/tmp/policy-install/keystore-storage" -+ name: keystore-storage ++ - mountPath: "/tmp/policy-install/truststore-storage" ++ name: truststore-storage +{{ include "common.cacert-mount-ubuntu" . | indent 2 }} +{{- end }} +{{- end }} |