From 950a3237302a9cc5040354083099bce42e8f2436 Mon Sep 17 00:00:00 2001 From: Gary Wu Date: Tue, 26 Mar 2019 13:08:29 -0700 Subject: First draft of k8s deployment using RKE Change-Id: Ifa0eb52b64438df64692aaf58b9ef8e5dd7fd32c Issue-ID: INT-993 Signed-off-by: Gary Wu --- deployment/heat/onap-rke/scripts/Crypto.java | 82 +++++ deployment/heat/onap-rke/scripts/cleanup.sh | 23 ++ deployment/heat/onap-rke/scripts/deploy.sh | 405 +++++++++++++++++++++ .../heat/onap-rke/scripts/gen-onap-oom-yaml.sh | 137 +++++++ deployment/heat/onap-rke/scripts/prepull-docker.sh | 26 ++ .../heat/onap-rke/scripts/redeploy-module.sh | 25 ++ deployment/heat/onap-rke/scripts/redeploy.sh | 106 ++++++ 7 files changed, 804 insertions(+) create mode 100644 deployment/heat/onap-rke/scripts/Crypto.java create mode 100755 deployment/heat/onap-rke/scripts/cleanup.sh create mode 100755 deployment/heat/onap-rke/scripts/deploy.sh create mode 100755 deployment/heat/onap-rke/scripts/gen-onap-oom-yaml.sh create mode 100755 deployment/heat/onap-rke/scripts/prepull-docker.sh create mode 100644 deployment/heat/onap-rke/scripts/redeploy-module.sh create mode 100755 deployment/heat/onap-rke/scripts/redeploy.sh (limited to 'deployment/heat/onap-rke/scripts') diff --git a/deployment/heat/onap-rke/scripts/Crypto.java b/deployment/heat/onap-rke/scripts/Crypto.java new file mode 100644 index 000000000..a9bad509a --- /dev/null +++ b/deployment/heat/onap-rke/scripts/Crypto.java @@ -0,0 +1,82 @@ +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.util.Arrays; + +public class Crypto { + + private static final String AES = "AES"; + private static final int GCM_TAG_LENGTH = 16; + private static final int GCM_IV_LENGTH = 12; + private static final String AES_GCM_NO_PADDING = "AES/GCM/NoPadding"; + + public static void main(String[] args) { + if(args.length != 2) { + System.out.println("Usage: java Crypto value_to_encrypt key"); + System.out.println("exit(1)"); + System.exit(1); + } + + String value = args[0]; + String key = args[1]; + String encrypted = encryptCloudConfigPassword(value, key); + System.out.println(encrypted); + } + + /** + * encrypt a value and generate a keyfile + * if the keyfile is not found then a new one is created + * + * @throws GeneralSecurityException + */ + public static String encrypt (String value, String keyString) throws GeneralSecurityException { + SecretKeySpec sks = getSecretKeySpec (keyString); + Cipher cipher = Cipher.getInstance(AES_GCM_NO_PADDING); + byte[] initVector = new byte[GCM_IV_LENGTH]; + (new SecureRandom()).nextBytes(initVector); + GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH * java.lang.Byte.SIZE, initVector); + cipher.init(Cipher.ENCRYPT_MODE, sks, spec); + byte[] encoded = value.getBytes(java.nio.charset.StandardCharsets.UTF_8); + byte[] cipherText = new byte[initVector.length + cipher.getOutputSize(encoded.length)]; + System.arraycopy(initVector, 0, cipherText, 0, initVector.length); + cipher.doFinal(encoded, 0, encoded.length, cipherText, initVector.length); + return byteArrayToHexString(cipherText); + } + + public static String encryptCloudConfigPassword(String message, String key) { + try { + return Crypto.encrypt(message, key); + } catch (GeneralSecurityException e) { + return null; + } + } + + private static SecretKeySpec getSecretKeySpec (String keyString) { + byte[] key = hexStringToByteArray (keyString); + return new SecretKeySpec (key, AES); + } + + public static String byteArrayToHexString (byte[] b) { + StringBuilder sb = new StringBuilder(b.length * 2); + for (byte aB : b) { + int v = aB & 0xff; + if (v < 16) { + sb.append('0'); + } + sb.append(Integer.toHexString(v)); + } + return sb.toString ().toUpperCase (); + } + + private static byte[] hexStringToByteArray (String s) { + byte[] b = new byte[s.length () / 2]; + for (int i = 0; i < b.length; i++) { + int index = i * 2; + int v = Integer.parseInt (s.substring (index, index + 2), 16); + b[i] = (byte) v; + } + return b; + } +} \ No newline at end of file diff --git a/deployment/heat/onap-rke/scripts/cleanup.sh b/deployment/heat/onap-rke/scripts/cleanup.sh new file mode 100755 index 000000000..7c2a1e29f --- /dev/null +++ b/deployment/heat/onap-rke/scripts/cleanup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +IFS=' +' + +if [ -z $1 ]; then + echo "ONAP component name missing" + echo "Usage: ./cleanup.sh onap_component_name" + exit 1 +fi + +COMPONENT=$1 + +if [ $COMPONENT == "dcae" ] || [ $COMPONENT == "DCAE" ]; then + kubectl delete service consul -n onap +fi + +for op in secrets configmaps pvc pv services deployments statefulsets clusterrolebinding; do + ARRAY=(`kubectl get $op -n onap | grep dev-$COMPONENT | awk '{print $1}'`) + for i in ${ARRAY[*]}; do + kubectl delete $op -n onap $i + done +done diff --git a/deployment/heat/onap-rke/scripts/deploy.sh b/deployment/heat/onap-rke/scripts/deploy.sh new file mode 100755 index 000000000..c4475b5d5 --- /dev/null +++ b/deployment/heat/onap-rke/scripts/deploy.sh @@ -0,0 +1,405 @@ +#!/bin/bash +# +# Copyright 2018 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# + +stack_name="oom" +portal_hostname="portal.api.simpledemo.onap.org" +full_deletion=false + +if [ -z "$WORKSPACE" ]; then + export WORKSPACE=`git rev-parse --show-toplevel` +fi + +usage() { + echo "Usage: $0 [ -n ][ -s ][ -m ][ -d ][ -r ][ -q ] " 1>&2; + + echo "n: Set the number of VM's that will be installed. This number must be between 2 and 15" 1>&2; + echo "s: Set the name to be used for stack. This name will be used for naming of resources" 1>&2; + echo "d: Set the base domain name to be used in portal UI URLs" 1>&2; + echo "m: The docker manifest to apply; must be either \"docker-manifest-staging.csv\" or \"docker-manifest.csv\"." 1>&2; + echo "r: Delete all resources relating to ONAP within enviroment." 1>&2; + echo "q: Quiet Delete of all ONAP resources." 1>&2; + + exit 1; +} + + +while getopts ":n:s:d:m:rq" o; do + case "${o}" in + n) + if [[ ${OPTARG} =~ ^[0-9]+$ ]];then + if [ ${OPTARG} -ge 2 -a ${OPTARG} -le 15 ]; then + vm_num=${OPTARG} + else + usage + fi + else + usage + fi + ;; + s) + if [[ ! ${OPTARG} =~ ^[0-9]+$ ]];then + stack_name=${OPTARG} + else + usage + fi + ;; + d) + if [[ ! ${OPTARG} =~ ^[0-9]+$ ]];then + portal_hostname=${OPTARG} + else + usage + fi + ;; + m) + if [ -f $WORKSPACE/version-manifest/src/main/resources/${OPTARG} ]; then + docker_manifest=${OPTARG} + else + usage + fi + ;; + r) + echo "The following command will delete all information relating to onap within your enviroment" + read -p "Are you certain this is what you want? (type y to confirm):" answer + + if [ $answer = "y" ] || [ $answer = "Y" ] || [ $answer = "yes" ] || [ $answer = "Yes"]; then + echo "This may delete the work of other colleages within the same enviroment" + read -p "Are you certain this is what you want? (type y to confirm):" answer2 + + if [ $answer2 = "y" ] || [ $answer2 = "Y" ] || [ $answer2 = "yes" ] || [ $answer2 = "Yes"]; then + full_deletion=true + else + echo "Ending program" + exit 1 + fi + else + echo "Ending program" + exit 1 + fi + ;; + q) + full_deletion=true + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +if [ "$#" -ne 1 ]; then + usage +fi + +ENV_FILE=$1 + +if [ ! -f $ENV_FILE ];then + echo ENV file does not exist or was not given + exit 1 +fi + +set -x + +SSH_KEY=~/.ssh/onap_key + +source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh + +#SO_ENCRYPTION_KEY=aa3871669d893c7fb8abbcda31b88b4f +#export OS_PASSWORD_ENCRYPTED=$(echo -n "$OS_PASSWORD" | openssl aes-128-ecb -e -K "$SO_ENCRYPTION_KEY" -nosalt | xxd -c 256 -p) + +#Use new encryption method +pushd $WORKSPACE/deployment/heat/onap-rke/scripts +javac Crypto.java +SO_ENCRYPTION_KEY=aa3871669d893c7fb8abbcda31b88b4f +export OS_PASSWORD_ENCRYPTED=$(java Crypto "$OS_PASSWORD" "$SO_ENCRYPTION_KEY") +popd + +for n in $(seq 1 5); do + if [ $full_deletion = true ] ; then + $WORKSPACE/test/ete/scripts/teardown-onap.sh -n $stack_name -q + else + $WORKSPACE/test/ete/scripts/teardown-onap.sh -n $stack_name + fi + + cd $WORKSPACE/deployment/heat/onap-rke + envsubst < $ENV_FILE > $ENV_FILE~ + if [ -z "$vm_num" ]; then + cp onap-oom.yaml onap-oom.yaml~ + else + ./scripts/gen-onap-oom-yaml.sh $vm_num > onap-oom.yaml~ + fi + + if ! openstack stack create -t ./onap-oom.yaml~ -e $ENV_FILE~ $stack_name --parameter docker_manifest=$docker_manifest --parameter portal_hostname=$portal_hostname; then + break + fi + + while [ "CREATE_IN_PROGRESS" == "$(openstack stack show -c stack_status -f value $stack_name)" ]; do + sleep 20 + done + + STATUS=$(openstack stack show -c stack_status -f value $stack_name) + echo $STATUS + if [ "CREATE_COMPLETE" != "$STATUS" ]; then + break + fi + + for i in $(seq 1 30); do + sleep 30 + RANCHER_IP=$(openstack stack output show $stack_name rancher_vm_ip -c output_value -f value) + K8S_IP=$(openstack stack output show $stack_name k8s_01_vm_ip -c output_value -f value) + timeout 1 ping -c 1 "$RANCHER_IP" && break + done + + timeout 1 ping -c 1 "$RANCHER_IP" && break + + echo Error: OpenStack infrastructure issue: unable to reach rancher "$RANCHER_IP" + sleep 10 +done + +if ! timeout 1 ping -c 1 "$RANCHER_IP"; then + exit 2 +fi + +# wait until all k8s VMs have fully initialized +for VM_NAME in $(grep _vm: ./onap-oom.yaml~ | cut -d: -f1); do + echo $VM_NAME + VM_IP=$(openstack stack output show $stack_name ${VM_NAME}_ip -c output_value -f value) + ssh-keygen -R $VM_IP + until ssh -o StrictHostKeychecking=no -i $SSH_KEY ubuntu@$VM_IP ls -ad /dockerdata-nfs/.git; do + sleep 1m + done +done + +cat > ./cluster.yml~ <> ./cluster.yml~ <> ./cluster.yml~ <> ./cluster.yml~ <" + exit 1 +fi +NUM_K8S_VMS=$1 + +if [ -z "$WORKSPACE" ]; then + export WORKSPACE=`git rev-parse --show-toplevel` +fi +PARTS_DIR=$WORKSPACE/deployment/heat/onap-rke/parts + +cat <" + exit 1 +fi +DOCKER_PROXY=$1 + +for MANIFEST in docker-manifest.csv docker-manifest-staging.csv; do + for DOCKER_IMAGE in $(tail -n +2 $WORKSPACE/version-manifest/src/main/resources/$MANIFEST | tr ',' ':'); do + docker pull $DOCKER_PROXY/$DOCKER_IMAGE + done +done diff --git a/deployment/heat/onap-rke/scripts/redeploy-module.sh b/deployment/heat/onap-rke/scripts/redeploy-module.sh new file mode 100644 index 000000000..ab528314a --- /dev/null +++ b/deployment/heat/onap-rke/scripts/redeploy-module.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2019 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# + +if [ "$#" -ne 1 ]; then + echo "Please specify module name, i.e. $0 robot" + exit 1 +fi + +module=$1 +deploy=dev-$1 +cd /root/oom/kubernetes +helm delete $deploy --purge +/root/integration/deployment/heat/onap-oom/scripts/cleanup.sh $module +rm -rf /dockerdata-nfs/$deploy +make $module +make onap +helm deploy $deploy local/onap -f /root/oom/kubernetes/onap/resources/environments/public-cloud.yaml -f /root/integration-override.yaml --namespace onap diff --git a/deployment/heat/onap-rke/scripts/redeploy.sh b/deployment/heat/onap-rke/scripts/redeploy.sh new file mode 100755 index 000000000..1d46f025d --- /dev/null +++ b/deployment/heat/onap-rke/scripts/redeploy.sh @@ -0,0 +1,106 @@ +#!/bin/bash -x +# +# Copyright 2018 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# + +# This is meant to be run from within the Rancher VM to completely +# redeploy ONAP while reusing the existing k8s stack. +# +# This assumes that /root/integration-override.yaml is up-to-date. +# +# This script can also be used after a VM reboot, and will restart +# helm server accordingly. + +export DEBIAN_FRONTEND=noninteractive + +usage() { + echo "Usage: $0 " 1>&2; + echo "This will completely re-deploy ONAP, and delete and re-clone oom/ and integration/ directories." + exit 1; +} + +if [ "$#" -ne 1 ]; then + usage +fi + + +NS=$1 +OOM_GERRIT_BRANCH=master +OOM_GERRIT_REFSPEC=refs/heads/master +INTEGRATION_GERRIT_BRANCH=master +INTEGRATION_GERRIT_REFSPEC=refs/heads/master +DOCKER_MANIFEST="" + +# Verify that k8s works +if [ $(kubectl get pods --namespace kube-system | tail -n +2 | grep -c Running) -lt 6 ]; then + echo "[ERROR] Kubernetes is not healthy; aborting" + exit 1 +fi + +if [ ! -f /dockerdata-nfs/rancher_agent_cmd.sh ]; then + cp /root/rancher_agent_cmd.sh /dockerdata-nfs +fi + + +kubectl delete namespace $NS +for op in secrets configmaps pvc pv services deployments statefulsets clusterrolebinding; do + kubectl delete $op -n $NS --all +done +helm undeploy dev --purge +rm -rf /dockerdata-nfs/dev-*/ + + +# Clone OOM: +cd ~ +rm -rf oom/ +git clone -b $OOM_GERRIT_BRANCH https://gerrit.onap.org/r/oom +cd oom +git fetch https://gerrit.onap.org/r/oom $OOM_GERRIT_REFSPEC +git checkout FETCH_HEAD +git checkout -b workarounds +git log -1 + +# Clone integration +cd ~ +rm -rf integration/ +git clone -b $INTEGRATION_GERRIT_BRANCH https://gerrit.onap.org/r/integration +cd integration +git fetch https://gerrit.onap.org/r/integration $INTEGRATION_GERRIT_REFSPEC +git checkout FETCH_HEAD +git checkout -b workarounds +git log -1 + +if [ ! -z "$DOCKER_MANIFEST" ]; then + cd version-manifest/src/main/scripts + ./update-oom-image-versions.sh ../resources/$DOCKER_MANIFEST ~/oom/ +fi + +cd ~/oom +git diff +git commit -a -m "apply manifest versions" +git tag -a "deploy0" -m "initial deployment" + + +# Run ONAP: +cd ~/oom/kubernetes/ + +if [ $(curl -s -o /dev/null -w "%{http_code}" 127.0.0.1:8879) -ne 200 ]; then + helm init --client-only + helm init --upgrade + helm serve & + sleep 10 + helm repo add local http://127.0.0.1:8879 + helm repo list +fi +make all +rsync -avt ~/oom/kubernetes/helm/plugins ~/.helm/ +helm search -l | grep local +helm deploy dev local/onap -f ~/oom/kubernetes/onap/resources/environments/public-cloud.yaml -f ~/integration-override.yaml --namespace onap | ts | tee -a ~/helm-deploy.log +helm list + -- cgit 1.2.3-korg