Use OAM IPs for k8s connectivity

Use OAM IPs for k8s connectivity to reduce network
congestion.

Change-Id: Ia6114f1e4bf43578ee77c1d39ba36880f4c550d2
Issue-ID: INT-586
Signed-off-by: Gary Wu <gary.i.wu@huawei.com>

6 files changed, 46 insertions, 5 deletions

diff --git a/deployment/heat/onap-oom/env/gwu/onap-oom.env b/deployment/heat/onap-oom/env/gwu/onap-oom.env
index cd78a94d2..3cb9e1834 100644
--- a/deployment/heat/onap-oom/env/gwu/onap-oom.env
+++ b/deployment/heat/onap-oom/env/gwu/onap-oom.env
@@ -5,7 +5,7 @@ parameters:
   apt_proxy: 192.168.1.51:3142
   docker_proxy: 192.168.2.18:5000
 
-  rancher_vm_flavor: m1.xlarge
+  rancher_vm_flavor: c1.xlarge
   k8s_vm_flavor: c1.xlarge
 
   public_net_id: c3352d4c-8452-4172-b09c-15f017673708
diff --git a/deployment/heat/onap-oom/k8s_vm_entrypoint.sh b/deployment/heat/onap-oom/k8s_vm_entrypoint.sh
index a6392354c..11d805799 100644
--- a/deployment/heat/onap-oom/k8s_vm_entrypoint.sh
+++ b/deployment/heat/onap-oom/k8s_vm_entrypoint.sh
@@ -13,6 +13,7 @@ printenv
 
 mkdir -p /opt/config
 echo "__rancher_ip_addr__" > /opt/config/rancher_ip_addr.txt
+echo "__rancher_private_ip_addr__" > /opt/config/rancher_private_ip_addr.txt
 HOST_IP=$(hostname -I)
 echo $HOST_IP `hostname` >> /etc/hosts
 
@@ -33,7 +34,7 @@ fi
 apt-get -y update
 
 mkdir -p /dockerdata-nfs
-echo "__rancher_ip_addr__:/dockerdata-nfs /dockerdata-nfs nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0" | tee -a /etc/fstab
+echo "__rancher_private_ip_addr__:/dockerdata-nfs /dockerdata-nfs nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0" | tee -a /etc/fstab
 
 apt-get -y install linux-image-extra-$(uname -r) jq nfs-common
 
diff --git a/deployment/heat/onap-oom/onap-oom.yaml b/deployment/heat/onap-oom/onap-oom.yaml
index 34e71fd79..c378d0a50 100644
--- a/deployment/heat/onap-oom/onap-oom.yaml
+++ b/deployment/heat/onap-oom/onap-oom.yaml
@@ -151,6 +151,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
             __integration_override_yaml__: { get_param: integration_override_yaml }
             __gerrit_branch__: { get_param: gerrit_branch }
             __gerrit_refspec__: { get_param: gerrit_refspec }
@@ -173,6 +174,19 @@ resources:
               get_attr: [k8s_10_floating_ip, floating_ip_address],
               get_attr: [k8s_11_floating_ip, floating_ip_address],
             ]
+            __k8s_private_ips__: [
+              get_attr: [k8s_1_floating_ip, fixed_ip_address],
+              get_attr: [k8s_2_floating_ip, fixed_ip_address],
+              get_attr: [k8s_3_floating_ip, fixed_ip_address],
+              get_attr: [k8s_4_floating_ip, fixed_ip_address],
+              get_attr: [k8s_5_floating_ip, fixed_ip_address],
+              get_attr: [k8s_6_floating_ip, fixed_ip_address],
+              get_attr: [k8s_7_floating_ip, fixed_ip_address],
+              get_attr: [k8s_8_floating_ip, fixed_ip_address],
+              get_attr: [k8s_9_floating_ip, fixed_ip_address],
+              get_attr: [k8s_10_floating_ip, fixed_ip_address],
+              get_attr: [k8s_11_floating_ip, fixed_ip_address],
+            ]
   k8s_1_private_port:
     type: OS::Neutron::Port
     properties:
@@ -203,6 +217,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -236,6 +251,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -269,6 +285,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -302,6 +319,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -335,6 +353,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -368,6 +387,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -401,6 +421,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -434,6 +455,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -467,6 +489,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -500,6 +523,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
@@ -533,6 +557,7 @@ resources:
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
diff --git a/deployment/heat/onap-oom/parts/onap-oom-2.yaml b/deployment/heat/onap-oom/parts/onap-oom-2.yaml
index a5b1b3332..f0a0a8bd0 100644
--- a/deployment/heat/onap-oom/parts/onap-oom-2.yaml
+++ b/deployment/heat/onap-oom/parts/onap-oom-2.yaml
@@ -28,6 +28,7 @@
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
           template:
             get_file: k8s_vm_entrypoint.sh
 
diff --git a/deployment/heat/onap-oom/rancher_vm_entrypoint.sh b/deployment/heat/onap-oom/rancher_vm_entrypoint.sh
index 26e8378c6..098abbd39 100644
--- a/deployment/heat/onap-oom/rancher_vm_entrypoint.sh
+++ b/deployment/heat/onap-oom/rancher_vm_entrypoint.sh
@@ -14,6 +14,7 @@ printenv
 mkdir -p /opt/config
 echo "__rancher_ip_addr__" > /opt/config/rancher_ip_addr.txt
 echo "__k8s_vm_ips__" > /opt/config/k8s_vm_ips.txt
+echo "__k8s_private_ips__" > /opt/config/k8s_private_ips.txt
 echo "__public_net_id__" > /opt/config/public_net_id.txt
 echo "__oam_network_cidr__" > /opt/config/oam_network_cidr.txt
 echo "__oam_network_id__" > /opt/config/oam_network_id.txt
@@ -72,7 +73,7 @@ git commit -m "initial commit"
 
 # export NFS mount
 NFS_EXP=""
-for K8S_VM_IP in $(tr -d ',[]' < /opt/config/k8s_vm_ips.txt); do
+for K8S_VM_IP in $(tr -d ',[]' < /opt/config/k8s_private_ips.txt); do
     NFS_EXP+="$K8S_VM_IP(rw,fsid=1,sync,no_root_squash,no_subtree_check) "
 done
 echo "/dockerdata-nfs $NFS_EXP" | tee /etc/exports
@@ -100,7 +101,7 @@ wget -q http://storage.googleapis.com/kubernetes-helm/helm-v2.8.2-linux-amd64.ta
 tar -zxvf helm-v2.8.2-linux-amd64.tar.gz
 sudo mv linux-amd64/helm /usr/local/bin/helm
 
-echo export RANCHER_IP=__rancher_ip_addr__ > api-keys-rc
+echo export RANCHER_IP=__rancher_private_ip_addr__ > api-keys-rc
 source api-keys-rc
 
 sleep 50
@@ -169,7 +170,7 @@ clusters:
 - cluster:
     api-version: v1
     insecure-skip-tls-verify: true
-    server: "https://$RANCHER_IP:8080/r/projects/$PID/kubernetes:6443"
+    server: "https://__rancher_ip_addr__:8080/r/projects/$PID/kubernetes:6443"
   name: "oom"
 contexts:
 - context:
diff --git a/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh b/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
index e4968e048..33d2c3db8 100755
--- a/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
+++ b/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
@@ -49,6 +49,7 @@ cat <<EOF
             __docker_proxy__: { get_param: docker_proxy }
             __apt_proxy__: { get_param: apt_proxy }
             __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
             __integration_override_yaml__: { get_param: integration_override_yaml }
             __gerrit_branch__: { get_param: gerrit_branch }
             __gerrit_refspec__: { get_param: gerrit_refspec }
@@ -70,6 +71,18 @@ done
 
 cat <<EOF
             ]
+            __k8s_private_ips__: [
+EOF
+
+for VM_NUM in $(seq $NUM_K8S_VMS); do
+    K8S_VM_NAME=k8s_$VM_NUM
+    cat <<EOF
+              get_attr: [${K8S_VM_NAME}_floating_ip, fixed_ip_address],
+EOF
+done
+
+cat <<EOF
+            ]
 EOF
 
 for VM_NUM in $(seq $NUM_K8S_VMS); do