From 2de95edebaa92072aa857ca70961cd7bdaf7a77d Mon Sep 17 00:00:00 2001 From: Michael Hwang Date: Wed, 1 Aug 2018 16:22:13 -0400 Subject: Do first pass on dcae onboarding vagrant * Provide Vagrantfile etc to standup a VirtualBox image that runs a minimal DCAE runtime for onboarding * Add README with directions on how to do above * Make minor fixes to the dcae-cli for issues discovered from testing above Issue-ID: DCAEGEN2-574 Change-Id: I99ea595df9e5e2b9a872b864e553caec1f020669 Signed-off-by: Michael Hwang --- dcae-cli/ChangeLog.md | 6 ++ dcae-cli/dcae_cli/_version.py | 2 +- dcae-cli/dcae_cli/util/docker_util.py | 15 +---- dcae-cli/requirements.txt | 30 ++++++---- dcae-cli/setup.py | 5 +- vagrant/README.md | 101 ++++++++++++++++++++++++++++++++++ vagrant/Vagrantfile | 72 ++++++++++++++++++++++++ vagrant/consul/config.json | 11 ++++ vagrant/docker-compose.yaml | 85 ++++++++++++++++++++++++++++ vagrant/provision.sh | 55 ++++++++++++++++++ 10 files changed, 355 insertions(+), 27 deletions(-) create mode 100644 vagrant/README.md create mode 100644 vagrant/Vagrantfile create mode 100644 vagrant/consul/config.json create mode 100644 vagrant/docker-compose.yaml create mode 100644 vagrant/provision.sh diff --git a/dcae-cli/ChangeLog.md b/dcae-cli/ChangeLog.md index e3aaac4..aa845bd 100644 --- a/dcae-cli/ChangeLog.md +++ b/dcae-cli/ChangeLog.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [2.10.2] + +* Fix dependency conflict with python-consul +* Fix psycopg2 warning by requiring psycopg2-binary +* Use Docker client from env when checking images + ## [2.10.1] * Fix DCAEGEN2-402 diff --git a/dcae-cli/dcae_cli/_version.py b/dcae-cli/dcae_cli/_version.py index be3a135..ac49ad3 100644 --- a/dcae-cli/dcae_cli/_version.py +++ b/dcae-cli/dcae_cli/_version.py @@ -19,4 +19,4 @@ # ECOMP is a trademark and service mark of AT&T Intellectual Property. # -*- coding: utf-8 -*- -__version__ = "2.10.1" +__version__ = "2.10.2" diff --git a/dcae-cli/dcae_cli/util/docker_util.py b/dcae-cli/dcae_cli/util/docker_util.py index a0499d6..7ae933f 100644 --- a/dcae-cli/dcae_cli/util/docker_util.py +++ b/dcae-cli/dcae_cli/util/docker_util.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -83,21 +83,10 @@ def get_docker_client(profile, logins=[]): except: raise DockerError('Could not connect to the Docker daemon. Is it running?') -def _get_docker_client(client_funcs=(docker.Client, docker.from_env)): - '''Returns a docker client object''' - for func in client_funcs: - try: - client = func(version='auto') - client.ping() - return client - except: - continue - raise DockerError('Could not connect to the Docker daemon. Is it running?') - def image_exists(image): '''Returns True if the image exists locally''' - client = _get_docker_client() + client = docker.from_env(version="auto") return True if client.images(image) else False diff --git a/dcae-cli/requirements.txt b/dcae-cli/requirements.txt index 03244e7..4ca45e2 100644 --- a/dcae-cli/requirements.txt +++ b/dcae-cli/requirements.txt @@ -1,13 +1,21 @@ -# TODO: May need to add the following line -python-consul==0.6.1 -six==1.10.0 -SQLAlchemy==1.1.4 -SQLAlchemy-Utils==0.32.11 -click==6.6 -jsonschema==2.5.1 +certifi==2018.4.16 +chardet==3.0.4 +click==6.7 docker-py==1.10.6 -terminaltables==3.1.0 +docker-pycreds==0.3.0 +genson==1.0.1 +idna==2.7 +jsonschema==2.6.0 +-e git+ssh://researchmike@gerrit.onap.org:29418/dcaegen2/platform/cli@4f039724360fbf13a8295c2074180fa5c9c0320b#egg=onap_dcae_cli&subdirectory=dcae-cli onap-dcae-discovery-client==2.1.0 -onap-dcae-dockering==1.3.0 -psycopg2==2.7.1 -genson==0.2.2 +onap-dcae-dockering==1.4.0 +psycopg2==2.7.5 +psycopg2-binary==2.7.5 +python-consul==0.7.2 +requests==2.19.1 +six==1.11.0 +SQLAlchemy==1.2.10 +SQLAlchemy-Utils==0.33.3 +terminaltables==3.1.0 +urllib3==1.23 +websocket-client==0.48.0 diff --git a/dcae-cli/setup.py b/dcae-cli/setup.py index 9cfdd3f..d5f5dcc 100644 --- a/dcae-cli/setup.py +++ b/dcae-cli/setup.py @@ -1,7 +1,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ setup( dcae_cli=dcae_cli.cli:cli """, setup_requires=['pytest-runner'], - install_requires=['python-consul', + install_requires=['python-consul<1.0.0', 'six', 'sqlalchemy', 'SQLAlchemy-Utils', @@ -50,6 +50,7 @@ setup( 'jsonschema', 'terminaltables', 'psycopg2', + 'psycopg2-binary', 'genson', 'onap-dcae-discovery-client>=2.0.0', 'onap-dcae-dockering>=1.0.0,<2.0.0' diff --git a/vagrant/README.md b/vagrant/README.md new file mode 100644 index 0000000..7f4cf3a --- /dev/null +++ b/vagrant/README.md @@ -0,0 +1,101 @@ +# Vagrant for DCAE runtime + +This project provides component developers the means to stand up their own local DCAE runtime environment to do local onboarding through the dcae-cli. [Vagrant](https://www.vagrantup.com/) is the technology used and is required to the provided Vagrantfile. + +## Pre-req + +1. Install Docker +2. Install Vagrant +3. Install VirtualBox +4. Add the following to your `/etc/hosts` (or equivalent): + +``` +127.0.0.1 vagrant-dcae +``` + +## Run + +Clone the [dcaegen2.platform.cli](https://gerrit.onap.org/r/#/admin/projects/dcaegen2/platform/cli) project. + +``` +cd vagrant +vagrant up +``` + +Open the following [Consul link](http://vagrant-dcae:8500/ui) to verify that Consul is running and review the listed services by clicking on each: + +* [`config_binding_service`](http://vagrant-dcae:8500/ui/#/vagrant-dcae/services/config_binding_service) +* [`onboardingdb`](http://vagrant-dcae:8500/ui/#/vagrant-dcae/services/onboardingdb) + +NOTE: The vagrant DCAE runtime requires several ports on the host machine. See the [Vagrantfile](Vagrantfile) for all the lines that contain `forwarded_port`. + +## Post run + +Post run involves setting up the dcae-cli to work with the newly instantiated local DCAE runtime environment. + +### Install + +First install [dcae-cli](https://pypi.org/project/onap-dcae-cli/). + +Even if you have already installed dcae-cli before, please take the time now to upgrade: + +``` +pip install --upgrade onap-dcae-cli +``` + +### Configure + +Type the following command and you will be taken through a series of prompts to configure dcae-cli. For the most part, use the responses shown below except for the *user id*: + +``` +$ dcae_cli --reinit +Warning! Reinitializing your dcae-cli configuration +Please enter the remote server url: https://git.onap.org/dcaegen2/platform/cli/plain +Could not download initial configuration from remote server. Attempt manually setting up? [y/N]: y +Please enter your user id: +Now we need to set up access to the onboarding catalog +Please enter the onboarding catalog hostname: vagrant-dcae +Please enter the onboarding catalog user: postgres +Please enter the onboarding catalog password: onap123 +Could not download initial profiles from remote server. Set empty default? [y/N]: y +Reinitialize done +``` + +#### Profile + +Next you will need to setup your default profile. + +Running the following, you should see the following: + +``` +$ dcae_cli profiles show default +{ + "cdap_broker": "cdap_broker", + "config_binding_service": "config_binding_service", + "consul_host": "", + "docker_host": "" +} +``` + +You need to set `consul_host` and `docker_host` by running the following: + +``` +$ dcae_cli profiles set default consul_host vagrant-dcae +$ dcae_cli profiles set default docker_host vagrant-dcae:2376 +``` + +Repeating `dcae_cli profiles show default` should show those parameters now filled. + +### Run + +You must point your Docker client to the Docker engine running in the VirtualBox instance by doing the following: + +``` +export DOCKER_HOST="tcp://vagrant-dcae:2376" +``` + +You must have this environment variable set everytime before you run the dcae-cli. + +## TODO + +* There is a known issue where Consul does not recover after doing a `vagrant reload`. It get diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile new file mode 100644 index 0000000..a7b293f --- /dev/null +++ b/vagrant/Vagrantfile @@ -0,0 +1,72 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + # use official ubuntu image for virtualbox + config.vm.provider "virtualbox" do |vb, override| + override.vm.box = "ubuntu/xenial64" + override.vm.synced_folder ".", "/srv/dcae-onboarding" + vb.customize ["modifyvm", :id, "--memory", "1024"] + vb.customize ["modifyvm", :id, "--cpus", "2"] + end + + # use third party image and NFS sharing for lxc + config.vm.provider "lxc" do |_, override| + override.vm.box = "generic/ubuntu1604" + override.vm.synced_folder ".", "/srv/dcae-onboarding", :type => "nfs" + end + + # use third party image and NFS sharing for libvirt + config.vm.provider "libvirt" do |_, override| + override.vm.box = "generic/ubuntu1604" + override.vm.synced_folder ".", "/srv/dcae-onboarding", :type => "nfs" + end + + # configure shared package cache if possible + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.enable :apt + config.cache.scope = :box + end + + # port forward for webrick on 3000 + # Access to docker engine + config.vm.network :forwarded_port, :guest => 2376, :host => 2376 + # Access to consul + config.vm.network :forwarded_port, :guest => 8500, :host => 8500 + # Access to config binding + config.vm.network :forwarded_port, :guest => 10000, :host => 10000 + # Access to onboarding db + config.vm.network :forwarded_port, :guest => 5432, :host => 5432 + # Access to local docker registry + config.vm.network :forwarded_port, :guest => 8443, :host => 8443 + + # Access to ephemeral ports which docker engine uses when using the + # -P option when doing "docker run". The range comes from + # /proc/sys/net/ipv4/ip_local_port_range and its a small slice of the range. + (32768..32775).each do |ephemeral_port| + config.vm.network :forwarded_port, :guest => ephemeral_port, :host => ephemeral_port + end + + # provision using a simple shell script + config.vm.provision :shell, :path => "provision.sh" +end diff --git a/vagrant/consul/config.json b/vagrant/consul/config.json new file mode 100644 index 0000000..0e223ab --- /dev/null +++ b/vagrant/consul/config.json @@ -0,0 +1,11 @@ +{ + "client_addr" : "0.0.0.0", + "bootstrap_expect" : 1, + "datacenter": "vagrant-dcae", + "http_api_response_headers": { + "Access-Control-Allow-Origin" : "*" + }, + "server": true, + "ui": true, + "log_level": "info" +} diff --git a/vagrant/docker-compose.yaml b/vagrant/docker-compose.yaml new file mode 100644 index 0000000..e6ecd4c --- /dev/null +++ b/vagrant/docker-compose.yaml @@ -0,0 +1,85 @@ +version: '2.0' +services: + + dockerregistry: + image: "registry:2" + container_name: "dockerregistry" + restart: "always" + hostname: "dockerregistry" + environment: + - "REGISTRY_HTTP_ADDR=0.0.0.0:8443" + - "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt" + - "REGISTRY_HTTP_TLS_KEY=/certs/domain.key" + ports: + - "8443:8443" + volumes: + - /certs:/certs + depends_on: + - "registrator" + + onboardingdb: + image: "postgres:9.5" + container_name: "onboardingdb" + restart: "always" + hostname: "onboardingdb" + environment: + - "POSTGRES_USER=postgres" + - "POSTGRES_PASSWORD=onap123" + ports: + - "5432:5432" + depends_on: + - "registrator" + labels: + - "SERVICE_5432_NAME=onboardingdb" + - "SERVICE_5432_CHECK_TCP=true" + - "SERVICE_5432_CHECK_INTERVAL=15s" + - "SERVICE_5432_CHECK_INITIAL_STATUS=passing" + + consul: + image: "consul:0.8.3" + container_name: "consul" + restart: "always" + hostname: "consul" + ports: + - "8500:8500" + environment: + - "DOCKER_HOST=tcp://${MYIP}:2376" + volumes: + - ${PWD}/consul:/opt/consul + command: "agent -bind 0.0.0.0 -config-file /opt/consul/config.json" + labels: + - "SERVICE_8500_NAME=consul" + - "SERVICE_8500_CHECK_HTTP=/v1/agent/services" + - "SERVICE_8500_CHECK_INTERVAL=15s" + - "SERVICE_8500_CHECK_INITIAL_STATUS=passing" + + registrator: + image: "onapdcae/registrator:v7" + container_name: "registrator" + restart: "always" + hostname: "consul" + environment: + - "CONSUL_HOST=consul" + - "EXTERNAL_IP=${MYIP}" + volumes: + - /var/run/docker.sock:/tmp/docker.sock + depends_on: + - "consul" + + config-binding-service: + image: "nexus3.onap.org:10001/onap/org.onap.dcaegen2.platform.configbinding:2.1.5" + container_name: "config_binding_service" + restart: "always" + hostname: "config-binding-service" + environment: + - "CONSUL_HOST=consul" + ports: + - "10000:10000" + depends_on: + - "registrator" + labels: + - "SERVICE_10000_NAME=config_binding_service" + - "SERVICE_10000_CHECK_HTTP=/healthcheck" + - "SERVICE_10000_CHECK_INTERVAL=15s" + - "SERVICE_10000_CHECK_INITIAL_STATUS=passing" + - "SERVICE_443_IGNORE=true" diff --git a/vagrant/provision.sh b/vagrant/provision.sh new file mode 100644 index 0000000..f4345e3 --- /dev/null +++ b/vagrant/provision.sh @@ -0,0 +1,55 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +#!/usr/bin/env bash + +# abort on error +set -e + +# Update and upgrade apt packages +apt-get update +apt-get upgrade -y + +# Hostname vagrant-dcae will be used to refer to this vagrant machine. This is +# important for certificates. +echo '127.0.0.1 vagrant-dcae' >>/etc/hosts + +# Generate self-signed certs for docker registry +mkdir /certs +openssl req \ + -newkey rsa:4096 -nodes -sha256 -keyout /certs/domain.key \ + -x509 -days 365 -out /certs/domain.crt \ + -subj "/C=US/ST=New York/L=New York/O=ONAP/CN=vagrant-dcae" + +# Install and setup docker +apt-get install -y -q docker.io docker-compose +echo 'DOCKER_OPTS="--raw-logs -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock"' >>/etc/default/docker +systemctl restart docker + +export MYIP=$(ip address show enp0s3 | grep 'inet ' | sed -e 's/^.*inet //' -e 's/\/.*$//' | tr -d '\n') +cd /srv/dcae-onboarding + +docker-compose up -d +# REVIEW: There seems to be a race condition where registrator must fully +# come up before the config binding service should start. Put in a arbitrary +# sleep then restarting containers as a work around. +sleep 10 +docker restart config_binding_service +docker restart onboardingdb -- cgit 1.2.3-korg