From 8c7373d8c5c432bc0aca3c508edc2948659f3136 Mon Sep 17 00:00:00 2001
From: Pawel Wieczorek
Date: Tue, 7 Jan 2020 15:28:55 +0100
Subject: Add support for using SSH
This patch changes "create_config.py" script interface and its
behaviour:
* by default anonymous Gerrit is used for repository cloning and code
URLS,
* external git mirror for code URLs might be provided but there is no
default one,
* SSH credentials might be provided for repository cloning.
Cloning repository using SSH is required due to current ONAP
infrastructure HTTPS bandwidth constraints.
User wishing to use this setup should provide two environmental
variables:
1) API_USER - Gerrit username for cloning repositories via SSH
2) API_KEY - Gerrit API key (User menu -> Settings -> HTTP Credentials)
Be aware that a new RSA key will be generated and added to the user's
Gerrit account for purposes of Hound code search operation inside a VM.
Do remember to delete it (User menu -> Settings -> SSH keys) after
evaluating this environment.
Issue-ID: ONAPARC-540
Change-Id: I3e98dfa8582322d5b3bbbbf377d748faea0da57b
Signed-off-by: Pawel Wieczorek
---
bootstrap/codesearch/Vagrantfile | 38 +++++++++++++++++++++++++++++++---
bootstrap/codesearch/create_config.py | 39 +++++++++++++++++------------------
2 files changed, 54 insertions(+), 23 deletions(-)
(limited to 'bootstrap/codesearch')
diff --git a/bootstrap/codesearch/Vagrantfile b/bootstrap/codesearch/Vagrantfile
index 0407a57b5..774a02134 100644
--- a/bootstrap/codesearch/Vagrantfile
+++ b/bootstrap/codesearch/Vagrantfile
@@ -5,6 +5,12 @@ host_ip = "192.168.121.1"
synced_folder = "/vagrant"
houndd_bin = "${HOME}/go/bin/houndd"
houndd_config = "${HOME}/config.json"
+key_file = "${HOME}/.ssh/id_rsa"
+api_base = "https://gerrit.onap.org/r"
+api_user = ENV.fetch('API_USER') { |user| abort("missing env: #{user}") }
+api_key = ENV.fetch('API_KEY') { |key| abort("missing env: #{key}") }
+onap_git = "git.onap.org"
+gerrit_port = "29418"
$replace_dns = <<-SCRIPT
HOST_IP="$1"
@@ -12,11 +18,35 @@ $replace_dns = <<-SCRIPT
echo nameserver "$HOST_IP" | tee /etc/resolv.conf
SCRIPT
+$generate_key = <<-SCRIPT
+ KEY_FILE="$1"
+ echo "Generating SSH key (${KEY_FILE})"
+ ssh-keygen -q -b 4096 -t rsa -f "$KEY_FILE" -N ""
+SCRIPT
+
+$upload_key = <<-SCRIPT
+ KEY_FILE="$1"
+ API_BASE="$2"
+ echo "Uploading SSH pubkey (${KEY_FILE}.pub) for user: ${API_USER}"
+ curl -sS \
+ -u "${API_USER}:${API_KEY}" \
+ -d "@${KEY_FILE}.pub" \
+ -H "Content-Type: text/plain" \
+ -X POST "${API_BASE}/a/accounts/${API_USER}/sshkeys"
+SCRIPT
+
Vagrant.configure("2") do |config|
config.vm.box = "generic/ubuntu1804"
config.vm.synced_folder ".", synced_folder, type: "rsync", rsync__exclude: "Vagrantfile"
config.vm.network "forwarded_port", guest: 6080, host: 6080
config.vm.provision "replace_dns", type: :shell, run: "always", inline: $replace_dns, args: host_ip
+ config.vm.provision "generate_key", type: :shell, privileged: false, inline: $generate_key, args: key_file
+ config.vm.provision "upload_key", type: :shell do |s|
+ s.privileged = false
+ s.inline = $upload_key
+ s.args = [key_file, api_base]
+ s.env = {'API_USER': api_user, 'API_KEY': api_key}
+ end
config.vm.provision "dependencies", type: :shell, inline: <<-SHELL
export DEBIAN_FRONTEND=noninteractive
apt-get update
@@ -26,9 +56,11 @@ Vagrant.configure("2") do |config|
export GOPATH="${HOME}/go"
go get -u github.com/hound-search/hound/cmds/...
SHELL
- config.vm.provision "generate_config", type: :shell, privileged: false, inline: <<-SHELL
- python3 #{synced_folder}/create_config.py > #{houndd_config}
- SHELL
+ config.vm.provision "generate_config", type: :shell do |s|
+ s.privileged = false
+ s.inline = "python3 #{synced_folder}/create_config.py --ssh ${1} ${2} --git ${3} > #{houndd_config}"
+ s.args = [api_user, gerrit_port, onap_git]
+ end
config.vm.provision "run_codesearch", type: :shell, privileged: false, inline: <<-SHELL
tmux new -d -s codesearch #{houndd_bin} -conf #{houndd_config}
SHELL
diff --git a/bootstrap/codesearch/create_config.py b/bootstrap/codesearch/create_config.py
index aef99e778..115b4672b 100755
--- a/bootstrap/codesearch/create_config.py
+++ b/bootstrap/codesearch/create_config.py
@@ -8,8 +8,8 @@ import json
import urllib.request
import sys
-DEFAULT_GERRIT = "https://gerrit.onap.org/r"
-DEFAULT_GIT = "https://git.onap.org"
+DEFAULT_GERRIT = "gerrit.onap.org"
+API_PREFIX = "/r"
API_PROJECTS = "/projects/"
MAGIC_PREFIX = ")]}'"
@@ -20,7 +20,7 @@ GIT_ANCHOR = "#n{line}"
def get_projects_list(gerrit):
"""Request list of all available projects from ONAP Gerrit."""
- resp = urllib.request.urlopen(gerrit + API_PROJECTS)
+ resp = urllib.request.urlopen("https://{}{}{}".format(gerrit, API_PREFIX, API_PROJECTS))
resp_body = resp.read()
no_magic = resp_body[len(MAGIC_PREFIX):]
@@ -30,23 +30,23 @@ def get_projects_list(gerrit):
return projects.keys()
-def create_repos_list(projects, gitweb, git, gerrit):
+def create_repos_list(projects, gerrit, ssh, git):
"""Create a map of all projects to their repositories' URLs."""
- gerrit_project_url = "{}/{}.git"
- gitweb_code_url = "{}/gitweb?p={}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}"
-
- git_project_url = "{}/{}"
- git_code_url = "{url}/tree/{path}{anchor}"
+ gerrit_url = "https://{}{}".format(gerrit, API_PREFIX)
+ gerrit_project_url = "{}/{{}}.git".format(gerrit_url)
+ gitweb_code_url = "{}/gitweb?p={{}}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}".format(gerrit_url)
repos_list = {}
for project in projects:
- if gitweb:
- project_url = gerrit_project_url.format(gerrit, project)
- code_url = gitweb_code_url.format(gerrit, project)
- anchor = GITWEB_ANCHOR
- else:
- project_url = git_project_url.format(git, project)
- code_url = git_code_url
+ project_url = gerrit_project_url.format(project)
+ code_url = gitweb_code_url.format(project)
+ anchor = GITWEB_ANCHOR
+
+ if ssh and len(ssh) == 2:
+ user, port = ssh[0], ssh[1]
+ project_url = "ssh://{}@{}:{}/{}.git".format(user, gerrit, port, project)
+ if git:
+ code_url = "https://{}/{}/tree/{{path}}{{anchor}}".format(git, project)
anchor = GIT_ANCHOR
repos_list[project] = {
@@ -64,9 +64,8 @@ def parse_arguments():
"""Return parsed command-line arguments."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--gerrit', help='Gerrit address', default=DEFAULT_GERRIT)
- access = parser.add_mutually_exclusive_group()
- access.add_argument('--gitweb', help='use Gerrit\'s gitweb (bool)', action='store_true')
- access.add_argument('--git', help='use external git (address)', default=DEFAULT_GIT)
+ parser.add_argument('--ssh', help='SSH information: user, port', nargs=2)
+ parser.add_argument('--git', help='external git address')
return parser.parse_args()
@@ -76,7 +75,7 @@ def main():
arguments = parse_arguments()
projects = get_projects_list(arguments.gerrit)
- repos = create_repos_list(projects, arguments.gitweb, arguments.git, arguments.gerrit)
+ repos = create_repos_list(projects, arguments.gerrit, arguments.ssh, arguments.git)
config = {
"max-concurrent-indexers": 2,
"dbpath": "data",
--
cgit 1.2.3-korg