Merge "Upgrade to common-packer"

15 files changed, 122 insertions, 254 deletions

diff --git a/jjb/ci-management/ci-management-packer-jobs.yaml b/jjb/ci-management/ci-management-packer-jobs.yaml
new file mode 100644
index 000000000..f78186112
--- /dev/null
+++ b/jjb/ci-management/ci-management-packer-jobs.yaml
@@ -0,0 +1,71 @@
+---
+- project:
+    name: packer-builder-jobs
+    jobs:
+      - gerrit-packer-merge
+
+    project: ci-management
+    project-name: ci-management
+    branch: master
+    archive-artifacts: '**/*.log'
+
+    build-node: centos7-builder-2c-1g
+    build-timeout: 90
+
+    platforms:
+      - centos-7
+      - ubuntu-16.04
+    templates: builder
+
+- project:
+    name: packer-docker-jobs
+    jobs:
+      - gerrit-packer-merge
+
+    project: ci-management
+    project-name: ci-management
+    branch: master
+    archive-artifacts: '**/*.log'
+
+    build-node: centos7-builder-2c-1g
+    build-timeout: 60
+
+    platforms:
+      - centos-7
+      - ubuntu-16.04
+    templates: docker
+
+- project:
+    name: packer-memcached-jobs
+    jobs:
+      - gerrit-packer-merge
+
+    project: ci-management
+    project-name: ci-management
+    branch: master
+    archive-artifacts: '**/*.log'
+
+    build-node: centos7-builder-2c-1g
+    build-timeout: 90
+
+    platforms:
+      - centos-7
+      - ubuntu-16.04
+    templates: memcached
+
+- project:
+    name: packer-redis-jobs
+    jobs:
+      - gerrit-packer-merge
+
+    project: ci-management
+    project-name: ci-management
+    branch: master
+    archive-artifacts: '**/*.log'
+
+    build-node: centos7-builder-2c-1g
+
+    platforms:
+      - centos-7
+      - ubuntu-16.04
+    templates: redis
diff --git a/jjb/ci-management/ci-management.yaml b/jjb/ci-management/ci-management.yaml
index 16ae88bc1..184bf894d 100644
--- a/jjb/ci-management/ci-management.yaml
+++ b/jjb/ci-management/ci-management.yaml
@@ -12,29 +12,6 @@
     build-node: 'centos7-builder-2c-1g'
     jjb-version: 2.0.0
 
-- project:
-    name: packer-jobs
-    # packer jobs templates are defined in global-jjb
-    jobs:
-      - '{project-name}-packer-jobs'
-    project: 'ci-management'
-    project-name: 'ci-management'
-    branch: master
-    archive-artifacts: '**/*.log'
-    build-node: 'ubuntu1604-builder-4c-4g'
-    platforms:
-      - centos
-      - ubuntu-16.04
-    templates:
-      - builder
-      - docker
-      - memcached
-      - redis
-    exclude:
-      # only build the docker image on ubuntu at present
-      - platforms: centos
-        templates: docker
-
 - job-template:
     name: '{project}-verify-jjb'
 
diff --git a/jjb/global-defaults.yaml b/jjb/global-defaults.yaml
index 64f6a74eb..51cf6c6dd 100644
--- a/jjb/global-defaults.yaml
+++ b/jjb/global-defaults.yaml
@@ -14,6 +14,9 @@
     # lf-infra defaults
     lftools-version: <1.0.0
 
+    # packer_version
+    packer-version: 1.1.3
+
     # default gerrit server definition
     server-name: 'Primary'
     gerrit-server-name: 'Primary'
diff --git a/jjb/global-jjb b/jjb/global-jjb
-Subproject aa205517366205dd017d0801bd0cfe104e742c1
+Subproject 90e4e825b3d0ac5959122c50e457419ad0cfe13
diff --git a/packer/README.markdown b/packer/README.markdown
index 4972affae..1ef176a62 100644
--- a/packer/README.markdown
+++ b/packer/README.markdown
@@ -12,10 +12,10 @@ You'll need to [install Packer][2], of course.
 Open-O's Packer configuration is divided into build-specific variables,
 output-specific templates and a set of shared provisioning scripts. To do a
 specific build, combine the template for the desired output artifact type with
-a variable file. To build a new basebuild instance the following would be done:
+a variable file. To build a new builder instance the following would be done:
 
 ```
-packer build -var-file=vars/cloud-env.json -var-file=vars/centos.json templates/basebuild.json
+packer build -var-file=vars/cloud-env.json -var-file=vars/centos.json templates/builder.json
 ```
 
 **NOTE:** vars/cloud-env.json is a gitignored file as it contains private
diff --git a/packer/provision/docker.sh b/packer/provision/docker.sh
deleted file mode 100644
index 9d34eb079..000000000
--- a/packer/provision/docker.sh
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/bin/bash
-
-# vim: ts=4 sw=4 sts=4 et tw=72 :
-
-# force any errors to cause the script and job to end in failure
-set -xeu -o pipefail
-
-rh_systems() {
-    # Assumes that python is already installed by basebuild
-
-    # Install dependencies for robotframework and robotframework-sshlibrary
-    yum install -y yum-utils unzip sshuttle nc libffi-devel openssl-devel
-
-    # Install docker
-    yum install -y docker supervisor bridge-utils
-    systemctl enable docker
-
-    # configure docker networking so that it does not conflict with LF
-    # internal networks
-    cat <<EOL > /etc/sysconfig/docker-network
-# /etc/sysconfig/docker-network
-DOCKER_NETWORK_OPTIONS='--bip=10.250.0.254/24'
-EOL
-    # configure docker daemon to listen on port 5555 enabling remote
-    # managment
-    sed -i -e "s#='--selinux-enabled'#='--selinux-enabled -H unix:///var/run/docker.sock -H tcp://0.0.0.0:5555'#g" /etc/sysconfig/docker
-
-    # docker group doesn't get created by default for some reason
-    groupadd docker
-}
-
-ubuntu_docker_mtu_fix(){
-    echo "---> Fixing docker's mtu settings"
-    systemctl stop docker
-    cat <<'EOL' > /etc/systemd/system/docker.service
-[Unit]
-Description=Docker Application Container Engine
-Documentation=https://docs.docker.com
-After=network.target docker.socket
-Requires=docker.socket
-
-[Service]
-Type=notify
-# the default is not to use systemd for cgroups because the delegate issues still
-# exists and systemd currently does not support the cgroup feature set required
-# for containers run by docker
-ExecStart=
-ExecStart=/usr/bin/dockerd --mtu 1454 -H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock
-ExecReload=/bin/kill -s HUP $MAINPID
-# Having non-zero Limit*s causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
-LimitNOFILE=infinity
-LimitNPROC=infinity
-LimitCORE=infinity
-# Uncomment TasksMax if your systemd version supports it.
-# Only systemd 226 and above support this version.
-TasksMax=infinity
-TimeoutStartSec=0
-# set delegate yes so that systemd does not reset the cgroups of docker containers
-Delegate=yes
-# kill only the docker process, not all processes in the cgroup
-KillMode=process
-
-[Install]
-WantedBy=multi-user.target
-EOL
-    systemctl start docker
-    echo "---> Checking MTU"
-    docker network inspect bridge
-    echo "---> MTU set to 1454"
-}
-
-ubuntu_systems() {
-    # Assumes that python is already installed by basebuild
-
-    # Install Python3.6
-    sudo add-apt-repository -y ppa:jonathonf/python-3.6
-    sudo apt-get update
-    sudo apt-get install -y python3.6 python3.6-dev
-
-    # Install dependencies for robotframework and robotframework-sshlibrary
-    apt install -y unzip sshuttle netcat libffi-dev libssl-dev
-    wget https://github.com/mozilla/geckodriver/releases/download/v0.18.0/geckodriver-v0.18.0-linux64.tar.gz
-    tar xvzf geckodriver-v0.18.0-linux64.tar.gz -C /usr/local/bin
-    export PATH=$PATH:/usr/local/bin
-
-    # Install docker
-    apt install -y docker.io
-
-    # Fixing Docker MTU settings
-    ubuntu_docker_mtu_fix
-}
-
-all_systems() {
-    # Install docker-compose
-    curl -o /usr/local/bin/docker-compose -L "https://github.com/docker/compose/releases/download/1.15.0/docker-compose-$(uname -s)-$(uname -m)"
-    chmod +x /usr/local/bin/docker-compose
-    docker-compose -v
-}
-
-echo "---> Detecting OS"
-ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]')
-
-case "${ORIGIN}" in
-    fedora|centos|redhat)
-        echo "---> RH type system detected"
-        rh_systems
-    ;;
-    ubuntu)
-        echo "---> Ubuntu system detected"
-        ubuntu_systems
-    ;;
-    *)
-        echo "---> Unknown operating system"
-    ;;
-esac
-
-# execute steps for all systems
-all_systems
diff --git a/packer/provision/local-builder.yaml b/packer/provision/local-builder.yaml
new file mode 100644
index 000000000..a3f2c0300
--- /dev/null
+++ b/packer/provision/local-builder.yaml
@@ -0,0 +1,23 @@
+---
+- import_playbook: ../common-packer/provision/baseline.yaml
+
+- hosts: all
+  become_user: root
+  become_method: sudo
+
+  roles:
+    - protobuf-install
+
+  tasks:
+    - name: Install Docker-Compose
+      shell: |
+        virtualenv $WORKSPACE/venv
+        source $WORKSPACE/venv/bin/activate
+        PYTHON="$WORKSPACE/venv/bin/python"
+        $PYTHON -m pip install --upgrade pip
+        $PYTHON -m pip install docker-compose
+
+  post_tasks:
+    - name: System Reseal
+      script: ../common-packer/provision/system-reseal.sh
+      become: true
diff --git a/packer/provision/local-docker.yaml b/packer/provision/local-docker.yaml
new file mode 100644
index 000000000..fce32a404
--- /dev/null
+++ b/packer/provision/local-docker.yaml
@@ -0,0 +1,17 @@
+---
+- import_playbook: ../common-packer/provision/docker.yaml
+
+- hosts: all
+  become_user: root
+  become_method: sudo
+
+  pre_tasks:
+    - include_role: name=system-update
+
+  roles:
+    - protobuf-install
+
+  post_tasks:
+    - name: System Reseal
+      script: ../common-packer/provision/system-reseal.sh
+      become: true
diff --git a/packer/templates/basebuild.json b/packer/templates/basebuild.json
deleted file mode 100644
index ff84a72d6..000000000
--- a/packer/templates/basebuild.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "variables": {
-    "stack_tenant": null,
-    "stack_user": null,
-    "stack_pass": null,
-    "stack_network": null,
-    "base_image": null,
-    "cloud_user": null,
-    "distro": null,
-    "cloud_user_data": null
-  },
-  "builders": [
-    {
-      "type": "openstack",
-      "identity_endpoint": "https://auth.vexxhost.net/v2.0/",
-      "tenant_name": "{{user `stack_tenant`}}",
-      "username": "{{user `stack_user`}}",
-      "password": "{{user `stack_pass`}}",
-      "region": "ca-ymq-1",
-      "ssh_username": "{{user `cloud_user`}}",
-      "image_name": "{{user `distro`}} - basebuild - {{isotime \"20060102-1504\"}}",
-      "source_image_name": "{{user `base_image`}}",
-      "flavor": "v1-standard-1",
-      "availability_zone": "ca-ymq-2",
-      "networks": [
-        "{{user `stack_network`}}"
-      ],
-      "user_data_file": "{{user `cloud_user_data`}}"
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "shell",
-      "inline": [
-        "mkdir -p /tmp/packer"
-      ]
-    },
-    {
-      "type": "file",
-      "source": "provision/basebuild/",
-      "destination": "/tmp/packer"
-    },
-    {
-      "type": "shell",
-      "scripts": [
-        "provision/baseline.sh",
-        "provision/basebuild.sh",
-        "provision/system_reseal.sh"
-      ],
-      "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi"
-    }
-  ]
-}
diff --git a/packer/templates/builder.json b/packer/templates/builder.json
new file mode 120000
index 000000000..bd227be50
--- /dev/null
+++ b/packer/templates/builder.json
@@ -0,0 +1 @@
+../common-packer/templates/builder.json
\ No newline at end of file
diff --git a/packer/templates/docker.json b/packer/templates/docker.json
index 1c8d3ec56..edea6d285 100644..120000
--- a/packer/templates/docker.json
+++ b/packer/templates/docker.json
@@ -1,54 +1 @@
-{
-  "variables": {
-    "stack_tenant": null,
-    "stack_user": null,
-    "stack_pass": null,
-    "stack_network": null,
-    "base_image": null,
-    "cloud_user": null,
-    "distro": null,
-    "cloud_user_data": null
-  },
-  "builders": [
-    {
-      "type": "openstack",
-      "identity_endpoint": "https://auth.vexxhost.net/v2.0/",
-      "tenant_name": "{{user `stack_tenant`}}",
-      "username": "{{user `stack_user`}}",
-      "password": "{{user `stack_pass`}}",
-      "region": "ca-ymq-1",
-      "ssh_username": "{{user `cloud_user`}}",
-      "image_name": "{{user `distro`}} - docker - {{isotime \"20060102-1504\"}}",
-      "source_image_name": "{{user `base_image`}}",
-      "flavor": "v1-standard-1",
-      "availability_zone": "ca-ymq-2",
-      "networks": [
-        "{{user `stack_network`}}"
-      ],
-      "user_data_file": "{{user `cloud_user_data`}}"
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "shell",
-      "inline": [
-        "mkdir -p /tmp/packer"
-      ]
-    },
-    {
-      "type": "file",
-      "source": "provision/basebuild/",
-      "destination": "/tmp/packer"
-    },
-    {
-      "type": "shell",
-      "scripts": [
-        "provision/baseline.sh",
-        "provision/basebuild.sh",
-        "provision/docker.sh",
-        "provision/system_reseal.sh"
-      ],
-      "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi"
-    }
-  ]
-}
+../common-packer/templates/docker.json
\ No newline at end of file
diff --git a/packer/vars/centos.json b/packer/vars/centos.json
index c5b8c79f2..366202c7a 100644
--- a/packer/vars/centos.json
+++ b/packer/vars/centos.json
@@ -1,6 +1,6 @@
 {
   "base_image": "CentOS 7.1611 (2017-01-30) - LF upload",
-  "cloud_user": "centos",
+  "ssh_user": "centos",
   "distro": "CentOS 7",
   "cloud_user_data": "provision/rh-user_data.sh"
 }
diff --git a/packer/vars/cloud-env.json.example b/packer/vars/cloud-env.json.example
index 3a2ea9d2b..451c5dab9 100644
--- a/packer/vars/cloud-env.json.example
+++ b/packer/vars/cloud-env.json.example
@@ -3,4 +3,5 @@
   "stack_user": "USERID",
   "stack_pass": "PASSWD",
   "stack_network": "NETID",
+  "ssh_proxy_host": ""
 }
diff --git a/packer/vars/ubuntu-14.04.json b/packer/vars/ubuntu-14.04.json
index dd81e2594..337ee1f1c 100644
--- a/packer/vars/ubuntu-14.04.json
+++ b/packer/vars/ubuntu-14.04.json
@@ -1,6 +1,6 @@
 {
   "base_image": "Ubuntu 14.04 (2017-02-01) - LF upload",
-  "cloud_user": "ubuntu",
   "distro": "Ubuntu 14.04",
+  "ssh_user": "ubuntu",
   "cloud_user_data": "provision/null_data.sh"
 }
diff --git a/packer/vars/ubuntu-16.04.json b/packer/vars/ubuntu-16.04.json
index 12f9430ed..eadf41863 100644
--- a/packer/vars/ubuntu-16.04.json
+++ b/packer/vars/ubuntu-16.04.json
@@ -1,6 +1,6 @@
 {
   "base_image": "Ubuntu 16.04 (2017-02-01) - LF upload",
-  "cloud_user": "ubuntu",
   "distro": "Ubuntu 16.04",
+  "ssh_user": "ubuntu",
   "cloud_user_data": "provision/null_data.sh"
 }