Merge "First draft of k8s deployment using RKE"

1 files changed, 959 insertions, 0 deletions

diff --git a/deployment/heat/onap-rke/onap-oom.yaml b/deployment/heat/onap-rke/onap-oom.yaml
new file mode 100644
index 000000000..c999327b4
--- /dev/null
+++ b/deployment/heat/onap-rke/onap-oom.yaml
@@ -0,0 +1,959 @@
+#
+# Generated by scripts/gen-onap-oom-yaml.sh; MANUAL CHANGES WILL BE LOST
+#
+heat_template_version: 2015-10-15
+description: ONAP on RKE Kubernetes using OOM
+
+parameters:
+  docker_proxy:
+    type: string
+
+  apt_proxy:
+    type: string
+
+  public_net_id:
+    type: string
+    description: The ID of the Public network for floating IP address allocation
+
+  oam_network_cidr:
+    type: string
+    description: CIDR of the OAM ONAP network
+
+  ubuntu_1804_image:
+    type: string
+    description: Name of the Ubuntu 18.04 image
+
+  rancher_vm_flavor:
+    type: string
+    description: VM flavor for Rancher
+
+  k8s_vm_flavor:
+    type: string
+    description: VM flavor for k8s hosts
+
+  etcd_vm_flavor:
+    type: string
+    description: VM flavor for etcd hosts
+
+  orch_vm_flavor:
+    type: string
+    description: VM flavor for orch hosts
+
+  integration_override_yaml:
+    type: string
+    description: Content for integration_override.yaml
+
+  integration_gerrit_branch:
+    type: string
+    default: "master"
+
+  integration_gerrit_refspec:
+    type: string
+    default: ""
+
+  oom_gerrit_branch:
+    type: string
+    default: "master"
+
+  oom_gerrit_refspec:
+    type: string
+    default: ""
+
+  docker_manifest:
+    type: string
+    default: ""
+
+  key_name:
+    type: string
+    default: "onap_key"
+
+  docker_version:
+    type: string
+    default: "17.03.2"
+
+  rancher_version:
+    type: string
+    default: "1.6.26"
+
+  rancher_agent_version:
+    type: string
+    default: "1.2.11"
+
+  kubectl_version:
+    type: string
+    default: "1.13.4"
+
+  helm_version:
+    type: string
+    default: "2.9.1"
+
+  helm_deploy_delay:
+    type: string
+    default: "3m"
+
+  use_ramdisk:
+    type: string
+    description: Set to "true" if you want to use a RAM disk for /dockerdata-nfs/.
+    default: "false"
+
+  mtu:
+    type: number
+    default: 1500
+
+  portal_hostname:
+    type: string
+    description: The FQDN of the k8s host that will be used for the Portal UI component URLs; this needs to be resolveable at the client
+    default: "portal.api.simpledemo.onap.org"
+
+resources:
+  random-str:
+    type: OS::Heat::RandomString
+    properties:
+      length: 4
+
+  # ONAP security group
+  onap_sg:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      name:
+        str_replace:
+          template: base_rand
+          params:
+            base: onap_sg
+            rand: { get_resource: random-str }
+      description: security group used by ONAP
+      rules:
+        # All egress traffic
+        - direction: egress
+          ethertype: IPv4
+        - direction: egress
+          ethertype: IPv6
+        # ingress traffic
+        # ICMP
+        - protocol: icmp
+        - protocol: udp
+          port_range_min: 1
+          port_range_max: 65535
+        - protocol: tcp
+          port_range_min: 1
+          port_range_max: 65535
+        # Protocols used for vLB/vDNS use case
+        - protocol: 47
+        - protocol: 53
+        - protocol: 132
+
+
+  # ONAP management private network
+  oam_network:
+    type: OS::Neutron::Net
+    properties:
+      name:
+        str_replace:
+          template: oam_network_rand
+          params:
+            rand: { get_resource: random-str }
+
+  oam_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      name:
+        str_replace:
+          template: oam_network_rand
+          params:
+            rand: { get_resource: random-str }
+      network_id: { get_resource: oam_network }
+      cidr: { get_param: oam_network_cidr }
+      dns_nameservers: [ "8.8.8.8" ]
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      name:
+        list_join: ['-', [{ get_param: 'OS::stack_name' }, 'router']]
+      external_gateway_info:
+        network: { get_param: public_net_id }
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: oam_subnet }
+
+  rancher_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  rancher_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: rancher_private_port }
+
+  rancher_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [{ get_param: 'OS::stack_name' }, 'rancher']]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: rancher_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: rancher_private_port }
+      user_data_format: RAW
+      user_data:
+        str_replace:
+          template:
+            get_file: rancher_vm_entrypoint.sh
+          params:
+            __docker_proxy__: { get_param: docker_proxy }
+            __apt_proxy__: { get_param: apt_proxy }
+            __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+            __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+            __integration_override_yaml__: { get_param: integration_override_yaml }
+            __integration_gerrit_branch__: { get_param: integration_gerrit_branch }
+            __integration_gerrit_refspec__: { get_param: integration_gerrit_refspec }
+            __oom_gerrit_branch__: { get_param: oom_gerrit_branch }
+            __oom_gerrit_refspec__: { get_param: oom_gerrit_refspec }
+            __docker_manifest__: { get_param: docker_manifest }
+            __docker_version__: { get_param: docker_version }
+            __rancher_version__: { get_param: rancher_version }
+            __rancher_agent_version__: { get_param: rancher_agent_version }
+            __kubectl_version__: { get_param: kubectl_version }
+            __helm_version__: { get_param: helm_version }
+            __helm_deploy_delay__: { get_param: helm_deploy_delay }
+            __use_ramdisk__: { get_param: use_ramdisk }
+            __mtu__: { get_param: mtu }
+            __portal_hostname__: { get_param: portal_hostname }
+            __public_net_id__: { get_param: public_net_id }
+            __oam_network_cidr__: { get_param: oam_network_cidr }
+            __oam_network_id__: { get_resource: oam_network }
+            __oam_subnet_id__: { get_resource: oam_subnet }
+            __sec_group__: { get_resource: onap_sg }
+            __k8s_01_vm_ip__: { get_attr: [k8s_01_floating_ip, floating_ip_address] }
+            __k8s_vm_ips__: [
+              get_attr: [k8s_01_floating_ip, floating_ip_address],
+              get_attr: [k8s_02_floating_ip, floating_ip_address],
+              get_attr: [k8s_03_floating_ip, floating_ip_address],
+              get_attr: [k8s_04_floating_ip, floating_ip_address],
+              get_attr: [k8s_05_floating_ip, floating_ip_address],
+              get_attr: [k8s_06_floating_ip, floating_ip_address],
+            ]
+            __k8s_private_ips__: [
+              get_attr: [k8s_01_floating_ip, fixed_ip_address],
+              get_attr: [k8s_02_floating_ip, fixed_ip_address],
+              get_attr: [k8s_03_floating_ip, fixed_ip_address],
+              get_attr: [k8s_04_floating_ip, fixed_ip_address],
+              get_attr: [k8s_05_floating_ip, fixed_ip_address],
+              get_attr: [k8s_06_floating_ip, fixed_ip_address],
+            ]
+  k8s_01_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  k8s_01_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: k8s_01_private_port }
+
+  k8s_01_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [k8s_01_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [k8s_01_floating_ip, fixed_ip_address] }
+                __host_label__: 'compute'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  k8s_01_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: k8s_01_vm_scripts }
+
+  k8s_01_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'k8s', '01' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: k8s_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: k8s_01_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: k8s_01_vm_config }
+
+  k8s_02_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  k8s_02_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: k8s_02_private_port }
+
+  k8s_02_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [k8s_02_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [k8s_02_floating_ip, fixed_ip_address] }
+                __host_label__: 'compute'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  k8s_02_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: k8s_02_vm_scripts }
+
+  k8s_02_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'k8s', '02' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: k8s_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: k8s_02_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: k8s_02_vm_config }
+
+  k8s_03_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  k8s_03_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: k8s_03_private_port }
+
+  k8s_03_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [k8s_03_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [k8s_03_floating_ip, fixed_ip_address] }
+                __host_label__: 'compute'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  k8s_03_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: k8s_03_vm_scripts }
+
+  k8s_03_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'k8s', '03' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: k8s_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: k8s_03_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: k8s_03_vm_config }
+
+  k8s_04_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  k8s_04_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: k8s_04_private_port }
+
+  k8s_04_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [k8s_04_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [k8s_04_floating_ip, fixed_ip_address] }
+                __host_label__: 'compute'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  k8s_04_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: k8s_04_vm_scripts }
+
+  k8s_04_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'k8s', '04' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: k8s_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: k8s_04_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: k8s_04_vm_config }
+
+  k8s_05_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  k8s_05_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: k8s_05_private_port }
+
+  k8s_05_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [k8s_05_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [k8s_05_floating_ip, fixed_ip_address] }
+                __host_label__: 'compute'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  k8s_05_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: k8s_05_vm_scripts }
+
+  k8s_05_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'k8s', '05' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: k8s_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: k8s_05_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: k8s_05_vm_config }
+
+  k8s_06_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  k8s_06_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: k8s_06_private_port }
+
+  k8s_06_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [k8s_06_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [k8s_06_floating_ip, fixed_ip_address] }
+                __host_label__: 'compute'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  k8s_06_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: k8s_06_vm_scripts }
+
+  k8s_06_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'k8s', '06' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: k8s_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: k8s_06_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: k8s_06_vm_config }
+
+  orch_1_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  orch_1_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: orch_1_private_port }
+
+  orch_1_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [orch_1_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [orch_1_floating_ip, fixed_ip_address] }
+                __host_label__: 'orchestration'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  orch_1_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: orch_1_vm_scripts }
+
+  orch_1_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'orch', '1' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: orch_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: orch_1_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: orch_1_vm_config }
+
+  orch_2_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  orch_2_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: orch_2_private_port }
+
+  orch_2_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [orch_2_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [orch_2_floating_ip, fixed_ip_address] }
+                __host_label__: 'orchestration'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  orch_2_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: orch_2_vm_scripts }
+
+  orch_2_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'orch', '2' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: orch_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: orch_2_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: orch_2_vm_config }
+
+  orch_3_private_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_resource: oam_network }
+      fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
+      security_groups:
+      - { get_resource: onap_sg }
+
+  orch_3_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network_id: { get_param: public_net_id }
+      port_id: { get_resource: orch_3_private_port }
+
+  orch_3_vm_scripts:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        power_state:
+          mode: reboot
+        runcmd:
+        - [ /opt/k8s_vm_install.sh ]
+        write_files:
+        - path: /opt/k8s_vm_install.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __docker_proxy__: { get_param: docker_proxy }
+                __apt_proxy__: { get_param: apt_proxy }
+                __docker_version__: { get_param: docker_version }
+                __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+                __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+                __host_private_ip_addr__: { get_attr: [orch_3_floating_ip, fixed_ip_address] }
+                __mtu__: { get_param: mtu }
+              template:
+                get_file: k8s_vm_install.sh
+        - path: /opt/k8s_vm_init.sh
+          permissions: '0755'
+          content:
+            str_replace:
+              params:
+                __host_private_ip_addr__: { get_attr: [orch_3_floating_ip, fixed_ip_address] }
+                __host_label__: 'orchestration'
+              template:
+                get_file: k8s_vm_init.sh
+        - path: /etc/init.d/k8s_vm_init_serv
+          permissions: '0755'
+          content:
+            get_file: k8s_vm_init_serv.sh
+
+  orch_3_vm_config:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: { get_resource: orch_3_vm_scripts }
+
+  orch_3_vm:
+    type: OS::Nova::Server
+    properties:
+      name:
+        list_join: ['-', [ { get_param: 'OS::stack_name' }, 'orch', '3' ] ]
+      image: { get_param: ubuntu_1804_image }
+      flavor: { get_param: orch_vm_flavor }
+      key_name: { get_param: key_name }
+      networks:
+      - port: { get_resource: orch_3_private_port }
+      user_data_format: SOFTWARE_CONFIG
+      user_data: { get_resource: orch_3_vm_config }
+
+outputs:
+  rancher_vm_ip:
+    description: The IP address of the rancher instance
+    value: { get_attr: [rancher_floating_ip, floating_ip_address] }
+
+  k8s_01_vm_ip:
+    description: The IP address of the k8s_01 instance
+    value: { get_attr: [k8s_01_floating_ip, floating_ip_address] }
+
+  k8s_01_vm_private_ip:
+    description: The private IP address of the k8s_01 instance
+    value: { get_attr: [k8s_01_floating_ip, fixed_ip_address] }
+
+  k8s_02_vm_ip:
+    description: The IP address of the k8s_02 instance
+    value: { get_attr: [k8s_02_floating_ip, floating_ip_address] }
+
+  k8s_02_vm_private_ip:
+    description: The private IP address of the k8s_02 instance
+    value: { get_attr: [k8s_02_floating_ip, fixed_ip_address] }
+
+  k8s_03_vm_ip:
+    description: The IP address of the k8s_03 instance
+    value: { get_attr: [k8s_03_floating_ip, floating_ip_address] }
+
+  k8s_03_vm_private_ip:
+    description: The private IP address of the k8s_03 instance
+    value: { get_attr: [k8s_03_floating_ip, fixed_ip_address] }
+
+  k8s_04_vm_ip:
+    description: The IP address of the k8s_04 instance
+    value: { get_attr: [k8s_04_floating_ip, floating_ip_address] }
+
+  k8s_04_vm_private_ip:
+    description: The private IP address of the k8s_04 instance
+    value: { get_attr: [k8s_04_floating_ip, fixed_ip_address] }
+
+  k8s_05_vm_ip:
+    description: The IP address of the k8s_05 instance
+    value: { get_attr: [k8s_05_floating_ip, floating_ip_address] }
+
+  k8s_05_vm_private_ip:
+    description: The private IP address of the k8s_05 instance
+    value: { get_attr: [k8s_05_floating_ip, fixed_ip_address] }
+
+  k8s_06_vm_ip:
+    description: The IP address of the k8s_06 instance
+    value: { get_attr: [k8s_06_floating_ip, floating_ip_address] }
+
+  k8s_06_vm_private_ip:
+    description: The private IP address of the k8s_06 instance
+    value: { get_attr: [k8s_06_floating_ip, fixed_ip_address] }
+
+  orch_1_vm_ip:
+    description: The IP address of the orch_1 instance
+    value: { get_attr: [orch_1_floating_ip, floating_ip_address] }
+
+  orch_1_vm_private_ip:
+    description: The private IP address of the orch_1 instance
+    value: { get_attr: [orch_1_floating_ip, fixed_ip_address] }
+
+  orch_2_vm_ip:
+    description: The IP address of the orch_2 instance
+    value: { get_attr: [orch_2_floating_ip, floating_ip_address] }
+
+  orch_2_vm_private_ip:
+    description: The private IP address of the orch_2 instance
+    value: { get_attr: [orch_2_floating_ip, fixed_ip_address] }
+
+  orch_3_vm_ip:
+    description: The IP address of the orch_3 instance
+    value: { get_attr: [orch_3_floating_ip, floating_ip_address] }
+
+  orch_3_vm_private_ip:
+    description: The private IP address of the orch_3 instance
+    value: { get_attr: [orch_3_floating_ip, fixed_ip_address] }
+