aboutsummaryrefslogtreecommitdiffstats
path: root/bootstrap/codesearch
diff options
context:
space:
mode:
authorPawel Wieczorek <p.wieczorek2@samsung.com>2020-01-07 15:28:55 +0100
committerPawel Wieczorek <p.wieczorek2@samsung.com>2020-01-07 15:28:55 +0100
commit8c7373d8c5c432bc0aca3c508edc2948659f3136 (patch)
treee3e0d751974d3f6a2ab842b074a2cb56894971a9 /bootstrap/codesearch
parent049202bf4fa1b9d3ccd6a83fa00c8e87ce4ebe04 (diff)
Add support for using SSH
This patch changes "create_config.py" script interface and its behaviour: * by default anonymous Gerrit is used for repository cloning and code URLS, * external git mirror for code URLs might be provided but there is no default one, * SSH credentials might be provided for repository cloning. Cloning repository using SSH is required due to current ONAP infrastructure HTTPS bandwidth constraints. User wishing to use this setup should provide two environmental variables: 1) API_USER - Gerrit username for cloning repositories via SSH 2) API_KEY - Gerrit API key (User menu -> Settings -> HTTP Credentials) Be aware that a new RSA key will be generated and added to the user's Gerrit account for purposes of Hound code search operation inside a VM. Do remember to delete it (User menu -> Settings -> SSH keys) after evaluating this environment. Issue-ID: ONAPARC-540 Change-Id: I3e98dfa8582322d5b3bbbbf377d748faea0da57b Signed-off-by: Pawel Wieczorek <p.wieczorek2@samsung.com>
Diffstat (limited to 'bootstrap/codesearch')
-rw-r--r--bootstrap/codesearch/Vagrantfile38
-rwxr-xr-xbootstrap/codesearch/create_config.py39
2 files changed, 54 insertions, 23 deletions
diff --git a/bootstrap/codesearch/Vagrantfile b/bootstrap/codesearch/Vagrantfile
index 0407a57b5..774a02134 100644
--- a/bootstrap/codesearch/Vagrantfile
+++ b/bootstrap/codesearch/Vagrantfile
@@ -5,6 +5,12 @@ host_ip = "192.168.121.1"
synced_folder = "/vagrant"
houndd_bin = "${HOME}/go/bin/houndd"
houndd_config = "${HOME}/config.json"
+key_file = "${HOME}/.ssh/id_rsa"
+api_base = "https://gerrit.onap.org/r"
+api_user = ENV.fetch('API_USER') { |user| abort("missing env: #{user}") }
+api_key = ENV.fetch('API_KEY') { |key| abort("missing env: #{key}") }
+onap_git = "git.onap.org"
+gerrit_port = "29418"
$replace_dns = <<-SCRIPT
HOST_IP="$1"
@@ -12,11 +18,35 @@ $replace_dns = <<-SCRIPT
echo nameserver "$HOST_IP" | tee /etc/resolv.conf
SCRIPT
+$generate_key = <<-SCRIPT
+ KEY_FILE="$1"
+ echo "Generating SSH key (${KEY_FILE})"
+ ssh-keygen -q -b 4096 -t rsa -f "$KEY_FILE" -N ""
+SCRIPT
+
+$upload_key = <<-SCRIPT
+ KEY_FILE="$1"
+ API_BASE="$2"
+ echo "Uploading SSH pubkey (${KEY_FILE}.pub) for user: ${API_USER}"
+ curl -sS \
+ -u "${API_USER}:${API_KEY}" \
+ -d "@${KEY_FILE}.pub" \
+ -H "Content-Type: text/plain" \
+ -X POST "${API_BASE}/a/accounts/${API_USER}/sshkeys"
+SCRIPT
+
Vagrant.configure("2") do |config|
config.vm.box = "generic/ubuntu1804"
config.vm.synced_folder ".", synced_folder, type: "rsync", rsync__exclude: "Vagrantfile"
config.vm.network "forwarded_port", guest: 6080, host: 6080
config.vm.provision "replace_dns", type: :shell, run: "always", inline: $replace_dns, args: host_ip
+ config.vm.provision "generate_key", type: :shell, privileged: false, inline: $generate_key, args: key_file
+ config.vm.provision "upload_key", type: :shell do |s|
+ s.privileged = false
+ s.inline = $upload_key
+ s.args = [key_file, api_base]
+ s.env = {'API_USER': api_user, 'API_KEY': api_key}
+ end
config.vm.provision "dependencies", type: :shell, inline: <<-SHELL
export DEBIAN_FRONTEND=noninteractive
apt-get update
@@ -26,9 +56,11 @@ Vagrant.configure("2") do |config|
export GOPATH="${HOME}/go"
go get -u github.com/hound-search/hound/cmds/...
SHELL
- config.vm.provision "generate_config", type: :shell, privileged: false, inline: <<-SHELL
- python3 #{synced_folder}/create_config.py > #{houndd_config}
- SHELL
+ config.vm.provision "generate_config", type: :shell do |s|
+ s.privileged = false
+ s.inline = "python3 #{synced_folder}/create_config.py --ssh ${1} ${2} --git ${3} > #{houndd_config}"
+ s.args = [api_user, gerrit_port, onap_git]
+ end
config.vm.provision "run_codesearch", type: :shell, privileged: false, inline: <<-SHELL
tmux new -d -s codesearch #{houndd_bin} -conf #{houndd_config}
SHELL
diff --git a/bootstrap/codesearch/create_config.py b/bootstrap/codesearch/create_config.py
index aef99e778..115b4672b 100755
--- a/bootstrap/codesearch/create_config.py
+++ b/bootstrap/codesearch/create_config.py
@@ -8,8 +8,8 @@ import json
import urllib.request
import sys
-DEFAULT_GERRIT = "https://gerrit.onap.org/r"
-DEFAULT_GIT = "https://git.onap.org"
+DEFAULT_GERRIT = "gerrit.onap.org"
+API_PREFIX = "/r"
API_PROJECTS = "/projects/"
MAGIC_PREFIX = ")]}'"
@@ -20,7 +20,7 @@ GIT_ANCHOR = "#n{line}"
def get_projects_list(gerrit):
"""Request list of all available projects from ONAP Gerrit."""
- resp = urllib.request.urlopen(gerrit + API_PROJECTS)
+ resp = urllib.request.urlopen("https://{}{}{}".format(gerrit, API_PREFIX, API_PROJECTS))
resp_body = resp.read()
no_magic = resp_body[len(MAGIC_PREFIX):]
@@ -30,23 +30,23 @@ def get_projects_list(gerrit):
return projects.keys()
-def create_repos_list(projects, gitweb, git, gerrit):
+def create_repos_list(projects, gerrit, ssh, git):
"""Create a map of all projects to their repositories' URLs."""
- gerrit_project_url = "{}/{}.git"
- gitweb_code_url = "{}/gitweb?p={}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}"
-
- git_project_url = "{}/{}"
- git_code_url = "{url}/tree/{path}{anchor}"
+ gerrit_url = "https://{}{}".format(gerrit, API_PREFIX)
+ gerrit_project_url = "{}/{{}}.git".format(gerrit_url)
+ gitweb_code_url = "{}/gitweb?p={{}}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}".format(gerrit_url)
repos_list = {}
for project in projects:
- if gitweb:
- project_url = gerrit_project_url.format(gerrit, project)
- code_url = gitweb_code_url.format(gerrit, project)
- anchor = GITWEB_ANCHOR
- else:
- project_url = git_project_url.format(git, project)
- code_url = git_code_url
+ project_url = gerrit_project_url.format(project)
+ code_url = gitweb_code_url.format(project)
+ anchor = GITWEB_ANCHOR
+
+ if ssh and len(ssh) == 2:
+ user, port = ssh[0], ssh[1]
+ project_url = "ssh://{}@{}:{}/{}.git".format(user, gerrit, port, project)
+ if git:
+ code_url = "https://{}/{}/tree/{{path}}{{anchor}}".format(git, project)
anchor = GIT_ANCHOR
repos_list[project] = {
@@ -64,9 +64,8 @@ def parse_arguments():
"""Return parsed command-line arguments."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--gerrit', help='Gerrit address', default=DEFAULT_GERRIT)
- access = parser.add_mutually_exclusive_group()
- access.add_argument('--gitweb', help='use Gerrit\'s gitweb (bool)', action='store_true')
- access.add_argument('--git', help='use external git (address)', default=DEFAULT_GIT)
+ parser.add_argument('--ssh', help='SSH information: user, port', nargs=2)
+ parser.add_argument('--git', help='external git address')
return parser.parse_args()
@@ -76,7 +75,7 @@ def main():
arguments = parse_arguments()
projects = get_projects_list(arguments.gerrit)
- repos = create_repos_list(projects, arguments.gitweb, arguments.git, arguments.gerrit)
+ repos = create_repos_list(projects, arguments.gerrit, arguments.ssh, arguments.git)
config = {
"max-concurrent-indexers": 2,
"dbpath": "data",