From 8c7373d8c5c432bc0aca3c508edc2948659f3136 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorek Date: Tue, 7 Jan 2020 15:28:55 +0100 Subject: Add support for using SSH This patch changes "create_config.py" script interface and its behaviour: * by default anonymous Gerrit is used for repository cloning and code URLS, * external git mirror for code URLs might be provided but there is no default one, * SSH credentials might be provided for repository cloning. Cloning repository using SSH is required due to current ONAP infrastructure HTTPS bandwidth constraints. User wishing to use this setup should provide two environmental variables: 1) API_USER - Gerrit username for cloning repositories via SSH 2) API_KEY - Gerrit API key (User menu -> Settings -> HTTP Credentials) Be aware that a new RSA key will be generated and added to the user's Gerrit account for purposes of Hound code search operation inside a VM. Do remember to delete it (User menu -> Settings -> SSH keys) after evaluating this environment. Issue-ID: ONAPARC-540 Change-Id: I3e98dfa8582322d5b3bbbbf377d748faea0da57b Signed-off-by: Pawel Wieczorek --- bootstrap/codesearch/Vagrantfile | 38 +++++++++++++++++++++++++++++++--- bootstrap/codesearch/create_config.py | 39 +++++++++++++++++------------------ 2 files changed, 54 insertions(+), 23 deletions(-) (limited to 'bootstrap') diff --git a/bootstrap/codesearch/Vagrantfile b/bootstrap/codesearch/Vagrantfile index 0407a57b5..774a02134 100644 --- a/bootstrap/codesearch/Vagrantfile +++ b/bootstrap/codesearch/Vagrantfile @@ -5,6 +5,12 @@ host_ip = "192.168.121.1" synced_folder = "/vagrant" houndd_bin = "${HOME}/go/bin/houndd" houndd_config = "${HOME}/config.json" +key_file = "${HOME}/.ssh/id_rsa" +api_base = "https://gerrit.onap.org/r" +api_user = ENV.fetch('API_USER') { |user| abort("missing env: #{user}") } +api_key = ENV.fetch('API_KEY') { |key| abort("missing env: #{key}") } +onap_git = "git.onap.org" +gerrit_port = "29418" $replace_dns = <<-SCRIPT HOST_IP="$1" @@ -12,11 +18,35 @@ $replace_dns = <<-SCRIPT echo nameserver "$HOST_IP" | tee /etc/resolv.conf SCRIPT +$generate_key = <<-SCRIPT + KEY_FILE="$1" + echo "Generating SSH key (${KEY_FILE})" + ssh-keygen -q -b 4096 -t rsa -f "$KEY_FILE" -N "" +SCRIPT + +$upload_key = <<-SCRIPT + KEY_FILE="$1" + API_BASE="$2" + echo "Uploading SSH pubkey (${KEY_FILE}.pub) for user: ${API_USER}" + curl -sS \ + -u "${API_USER}:${API_KEY}" \ + -d "@${KEY_FILE}.pub" \ + -H "Content-Type: text/plain" \ + -X POST "${API_BASE}/a/accounts/${API_USER}/sshkeys" +SCRIPT + Vagrant.configure("2") do |config| config.vm.box = "generic/ubuntu1804" config.vm.synced_folder ".", synced_folder, type: "rsync", rsync__exclude: "Vagrantfile" config.vm.network "forwarded_port", guest: 6080, host: 6080 config.vm.provision "replace_dns", type: :shell, run: "always", inline: $replace_dns, args: host_ip + config.vm.provision "generate_key", type: :shell, privileged: false, inline: $generate_key, args: key_file + config.vm.provision "upload_key", type: :shell do |s| + s.privileged = false + s.inline = $upload_key + s.args = [key_file, api_base] + s.env = {'API_USER': api_user, 'API_KEY': api_key} + end config.vm.provision "dependencies", type: :shell, inline: <<-SHELL export DEBIAN_FRONTEND=noninteractive apt-get update @@ -26,9 +56,11 @@ Vagrant.configure("2") do |config| export GOPATH="${HOME}/go" go get -u github.com/hound-search/hound/cmds/... SHELL - config.vm.provision "generate_config", type: :shell, privileged: false, inline: <<-SHELL - python3 #{synced_folder}/create_config.py > #{houndd_config} - SHELL + config.vm.provision "generate_config", type: :shell do |s| + s.privileged = false + s.inline = "python3 #{synced_folder}/create_config.py --ssh ${1} ${2} --git ${3} > #{houndd_config}" + s.args = [api_user, gerrit_port, onap_git] + end config.vm.provision "run_codesearch", type: :shell, privileged: false, inline: <<-SHELL tmux new -d -s codesearch #{houndd_bin} -conf #{houndd_config} SHELL diff --git a/bootstrap/codesearch/create_config.py b/bootstrap/codesearch/create_config.py index aef99e778..115b4672b 100755 --- a/bootstrap/codesearch/create_config.py +++ b/bootstrap/codesearch/create_config.py @@ -8,8 +8,8 @@ import json import urllib.request import sys -DEFAULT_GERRIT = "https://gerrit.onap.org/r" -DEFAULT_GIT = "https://git.onap.org" +DEFAULT_GERRIT = "gerrit.onap.org" +API_PREFIX = "/r" API_PROJECTS = "/projects/" MAGIC_PREFIX = ")]}'" @@ -20,7 +20,7 @@ GIT_ANCHOR = "#n{line}" def get_projects_list(gerrit): """Request list of all available projects from ONAP Gerrit.""" - resp = urllib.request.urlopen(gerrit + API_PROJECTS) + resp = urllib.request.urlopen("https://{}{}{}".format(gerrit, API_PREFIX, API_PROJECTS)) resp_body = resp.read() no_magic = resp_body[len(MAGIC_PREFIX):] @@ -30,23 +30,23 @@ def get_projects_list(gerrit): return projects.keys() -def create_repos_list(projects, gitweb, git, gerrit): +def create_repos_list(projects, gerrit, ssh, git): """Create a map of all projects to their repositories' URLs.""" - gerrit_project_url = "{}/{}.git" - gitweb_code_url = "{}/gitweb?p={}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}" - - git_project_url = "{}/{}" - git_code_url = "{url}/tree/{path}{anchor}" + gerrit_url = "https://{}{}".format(gerrit, API_PREFIX) + gerrit_project_url = "{}/{{}}.git".format(gerrit_url) + gitweb_code_url = "{}/gitweb?p={{}}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}".format(gerrit_url) repos_list = {} for project in projects: - if gitweb: - project_url = gerrit_project_url.format(gerrit, project) - code_url = gitweb_code_url.format(gerrit, project) - anchor = GITWEB_ANCHOR - else: - project_url = git_project_url.format(git, project) - code_url = git_code_url + project_url = gerrit_project_url.format(project) + code_url = gitweb_code_url.format(project) + anchor = GITWEB_ANCHOR + + if ssh and len(ssh) == 2: + user, port = ssh[0], ssh[1] + project_url = "ssh://{}@{}:{}/{}.git".format(user, gerrit, port, project) + if git: + code_url = "https://{}/{}/tree/{{path}}{{anchor}}".format(git, project) anchor = GIT_ANCHOR repos_list[project] = { @@ -64,9 +64,8 @@ def parse_arguments(): """Return parsed command-line arguments.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--gerrit', help='Gerrit address', default=DEFAULT_GERRIT) - access = parser.add_mutually_exclusive_group() - access.add_argument('--gitweb', help='use Gerrit\'s gitweb (bool)', action='store_true') - access.add_argument('--git', help='use external git (address)', default=DEFAULT_GIT) + parser.add_argument('--ssh', help='SSH information: user, port', nargs=2) + parser.add_argument('--git', help='external git address') return parser.parse_args() @@ -76,7 +75,7 @@ def main(): arguments = parse_arguments() projects = get_projects_list(arguments.gerrit) - repos = create_repos_list(projects, arguments.gitweb, arguments.git, arguments.gerrit) + repos = create_repos_list(projects, arguments.gerrit, arguments.ssh, arguments.git) config = { "max-concurrent-indexers": 2, "dbpath": "data", -- cgit 1.2.3-korg