diff options
author | 2018-09-17 09:17:27 +0100 | |
---|---|---|
committer | 2018-09-25 14:37:26 +0100 | |
commit | b4b9a05017c7c6a337a67e6500edb3c2aad656bc (patch) | |
tree | bb61cfe71111c192f12d48a7b4766431f1b8bd80 /kubernetes/pnda/charts/dcae-pnda-bootstrap/resources | |
parent | 3d8330ad449c1801ed793f98fb8c510d5a450c6d (diff) |
Add PNDA bootstrap job to dcaegen2 chart
Change-Id: I39974a948d261e3e6d80faf0219f67d4ffc8c90c
Issue-ID: DCAEGEN2-779
Signed-off-by: Donald Hunter <donaldh@cisco.com>
Diffstat (limited to 'kubernetes/pnda/charts/dcae-pnda-bootstrap/resources')
-rw-r--r-- | kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/inputs/pnda_env.yaml | 230 | ||||
-rw-r--r-- | kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/scripts/bootstrap.sh | 67 |
2 files changed, 297 insertions, 0 deletions
diff --git a/kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/inputs/pnda_env.yaml b/kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/inputs/pnda_env.yaml new file mode 100644 index 0000000000..555b9e4de4 --- /dev/null +++ b/kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/inputs/pnda_env.yaml @@ -0,0 +1,230 @@ +# ================================================================================ +# Copyright (c) 2018 Cisco Systems. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +infrastructure : + # infrastructure used for pnda deployment + # Valid Values are: + # - aws + # - openstack + # - existing-machines + # - terraform + INFRASTRUCTURE_TYPE: openstack + + # The user name to use when logging into the instances + # For aws target user-name allowed : + # Target AWS Openstack + # + # Distro Redhat: ec2-user cloud-user + # CentOS: centos cloud-user + OS_USER: {{ .Values.pnda.osUser }} + + # CIDR specifying the address range for the network containing all PNDA instances + networkCidr: {{ .Values.pnda.networkCidr }} + +openstack_parameters: + # KEYSTONE_USER: Username for the openstack clients to use + KEYSTONE_USER: {{ .Values.openstack.keystoneUser }} + + # KEYSTONE_PASSWORD: Password for the openstack clients to use + KEYSTONE_PASSWORD: {{ .Values.openstack.keystonePassword }} + + # KEYSTONE_TENANT: Name of the tenant / project in the openstack environment. The + # PNDA stack will be created in this project. + KEYSTONE_TENANT: {{ .Values.openstack.keystoneTenant }} + + # KEYSTONE_AUTH_URL: Keystone authentication URL. The Openstack console provides this + # under the Access & Security section. + KEYSTONE_AUTH_URL: {{ .Values.openstack.keystoneAuthUrl }} + + # KEYSTONE_AUTH_VERSION: Keystone authentication version. The Openstack console provides this + # under the Access & Security section. + KEYSTONE_AUTH_VERSION: '2' + + # KEYSTONE_REGION_NAME: Keystone region. The Openstack console provides this + # under the Access & Security section. + KEYSTONE_REGION_NAME: {{ .Values.openstack.keystoneRegion }} + + # imageId: Base image to use for the created instances. It should be created by + # following the guide in https://github.com/pndaproject/pnda-dib-elements + # + imageId: {{ .Values.openstack.imageId }} + + # CIDR specifying the address range that may access the created PNDA instances + whitelistSshAccess: {{ .Values.openstack.whitelistSshAccess }} + + # UUID of the public network in openstack to use + externalPublicNetworkId: {{ .Values.openstack.publicNetworkId }} + + useExistingNetwork: {{ .Values.openstack.useExistingNetwork }} + + existingNetworkId: {{ .Values.openstack.existingNetworkId }} + + existingSubnetId: {{ .Values.openstack.existingSubnetId }} + + # CIDR specifying the address range for the public subnet (bastion access) + publicSubnetCidr: {{ .Values.openstack.publicSubnetCidr }} + +platform_salt: + # Use either PLATFORM_GIT_REPO_URI + PLATFORM_GIT_BRANCH or PLATFORM_SALT_LOCAL + PLATFORM_SALT_LOCAL: /platform-salt + +pnda_application_repo: + # Type of storage to use for PNDA application packages + # s3 - AWS S3. Also set PNDA_APPS_CONTAINER, PNDA_APPS_FOLDER, PNDA_APPS_REGION, PNDA_APPS_ACCESS_KEY_ID, PNDA_APPS_SECRET_ACCESS_KEY + # sshfs - standard file system. Also set PR_FS_LOCATION_PATH, PR_SSHFS_USER, PR_SSHFS_HOST, PR_SSHFS_PATH and PR_SSHFS_KEY + # local - local filesystem on the package repository service server. Also set PR_FS_LOCATION_PATH. + # swift - Openstack swift. Also set PNDA_APPS_CONTAINER and PNDA_APPS_FOLDER + PR_FS_TYPE: {{ .Values.pnda.apps.fsType }} + + # S3 container to use for PNDA application packages + PNDA_APPS_CONTAINER: {{ .Values.pnda.apps.s3container }} + + # Name of folder within PNDA_APPS_CONTAINER that contains the PNDA application packages + PNDA_APPS_FOLDER: {{ .Values.pnda.apps.s3folder }} + + # AWS region that contains the PNDA_APPS_CONTAINER bucket + PNDA_APPS_REGION: {{ .Values.pnda.apps.s3region }} + + # API key for s3 access to PNDA_APPS_CONTAINER. These keys are stored on the cloud instances so should be restricted + # only allow access to the PNDA_APPS_CONTAINER bucket + PNDA_APPS_ACCESS_KEY_ID: {{ .Values.pnda.apps.s3keyid }} + PNDA_APPS_SECRET_ACCESS_KEY: {{ .Values.pnda.apps.s3secret }} + + # Path on file system if PR_FS_TYPE is 'local' or 'sshfs' + PR_FS_LOCATION_PATH: {{ .Values.pnda.apps.fsLocation | print "/opt/pnda/packages" }} + + # SSH accessed file system to use for PNDA application packages + PR_SSHFS_USER: centos + PR_SSHFS_HOST: 127.0.0.1 + PR_SSHFS_PATH: /mnt/packages + PR_SSHFS_KEY: key.pem + +pnda_data_archive: + # S3 container to use for archiving PNDA datasets + PNDA_ARCHIVE_CONTAINER: pnda-archive + + # AWS region that contains the PNDA_ARCHIVE_CONTAINER bucket + PNDA_ARCHIVE_REGION: eu-west-1 + + # API key for s3 access to PNDA_ARCHIVE_CONTAINER. These keys are stored on the cloud instances so should be restricted + # only allow access to the PNDA_ARCHIVE_CONTAINER bucket + PNDA_ARCHIVE_ACCESS_KEY_ID: xxxx + PNDA_ARCHIVE_SECRET_ACCESS_KEY: xxxx + +ntp: + # Optional ntp servers. Use this if the standard NTP servers on the Internet cannot be reached + # and a local NTP server has been configured. PNDA will not work without NTP. + # example format: 'xxx.ntp.org' + #For REJECT_OUTBOUND="YES" then NTP server/s must. + NTP_SERVERS: + - {{ .Values.pnda.ntp }} + +dns: + # External DNS servers list + nameServers: + - {{ .Values.pnda.nameserver }} + +mirrors: + # Mirror of resources required for provisioning PNDA, see PNDA guide for instructions on how to set this up + PNDA_MIRROR: + +hadoop: + # Hadoop distribution to install + # Valid values are: + # - HDP + # - CDH + HADOOP_DISTRO: HDP + # Spark version to enable for oozie (HDP only) + # Valid values are: + # - 1 + # - 2 + OOZIE_SPARK_VERSION: 1 + +connectivity: + # The IP address of the client that created PNDA + CLIENT_IP: {{ .Values.pnda.outboundCidr }} + # Add online repositories for yum, apt-get, pip, etc alongside PNDA mirror + ADD_ONLINE_REPOS: "YES" + # RPM Extras repository to enable when ADD_ONLINE_REPOS=YES + RPM_EXTRAS_REPO_NAME: rhui-REGION-rhel-server-optional + # RPM Optional repository to enable when ADD_ONLINE_REPOS=YES + RPM_OPTIONAL_REPO_NAME: rhui-REGION-rhel-server-extras + +network_interfaces: + PNDA_INTERNAL_NETWORK: eth0 + PNDA_INGEST_NETWORK: eth0 + +cli: + # Maximum number of outbound connections that the CLI will attempt to open at once + # Consider increasing this when creating clusters with more than 100 nodes to speed + # up PNDA creation time. + MAX_SIMULTANEOUS_OUTBOUND_CONNECTIONS: 100 + +security: + # The path were to find the security material (certificate/key). + # The directory should be structured as defined in this' repo's directory structure with the same name. + # The security material should conform to the guidelines defined in the README.md file in + # the containing sub directory. + SECURITY_MATERIAL_PATH: ./platform-certificates/ + + # Address of LDAP server + # All instances will have PAM configured to authenticate with this LDAP server if set + # Leave blank to disable LDAP-PAM integration + LDAP_SERVER: '' + + # Base DN for LDAP server to use when enabling client PAM integration with LDAP + LDAP_BASE_DN: dc=nodomain + +features: + # Include experimental features. + # Set to "NO", omit setting or omit features section entirely to turn off experimental features + EXPERIMENTAL_FEATURES: "NO" + +domain: + # Top-level domain + TOP_LEVEL_DOMAIN: pnda.local + + # Second-level domain + SECOND_LEVEL_DOMAIN: dc1 + +dataset_compaction: + # Enable/Disable compaction on datasets. + # "YES" to enable. + # "NO" to disable. + COMPACTION: "NO" + # If compaction is enabled, PATTERN sets the frequency of compaction. + # H - hourly compaction. + # d - daily compaction. + # M - monthly compaction. + # Y - yearly compaction. + PATTERN: d + +datanode: + # DATANODE_VOLUME_COUNT sets the number of data volumes on each hadoop datanode + DATA_VOLUME_COUNT: 1 + # DEVICE_ROOT sets the disk device root name + DEVICE_ROOT: xvdb + +kafka: + # DATA_DIRS sets the data dirs on kafka node + KAFKA_DATA_DIRS: + - /var/kafka-logs + # DEVICE_ROOT sets the disk device root name + KAFKA_DEVICE_ROOT: xvdb + +generic: + #GENERIC_DEVICE_ROOT sets the disk device root name for generic instances. + GENERIC_DEVICE_ROOT: xvdb diff --git a/kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/scripts/bootstrap.sh b/kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/scripts/bootstrap.sh new file mode 100644 index 0000000000..61b584240d --- /dev/null +++ b/kubernetes/pnda/charts/dcae-pnda-bootstrap/resources/scripts/bootstrap.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# ================================================================================ +# Copyright (c) 2018 Cisco Systems. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +# Install PNDA in Openstack with Heat templates +# Expects: +# Input files for components to be installed in /inputs + +if [ "z{{ .Values.enabled }}" != "ztrue" ] +then + echo + echo "PNDA bootstrap is disabled - skipping pnda-cli launch" + echo + exit 0 +fi + +set -ex + +CLUSTER_PREFIX="{{ include "common.namespace" . }}-pnda" +DATANODES="{{ .Values.pnda.dataNodes }}" +KAFKANODES="{{ .Values.pnda.kafkaNodes }}" +VERSION="{{ .Values.pnda.version }}" +KEYPAIR_NAME="{{ .Values.pnda_keypair_name }}" +KEYFILE="$KEYPAIR_NAME.pem" + +cd /pnda-cli + +cp /inputs/pnda_env.yaml . +cp /secrets/pnda.pem $KEYFILE +chmod 600 $KEYFILE + +(cd tools && ./gen-certs.py) + +KUBE_API="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT_HTTPS/api/v1" +KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + +for i in 1 2 3 4 5 6 7 8 9 +do + MIRROR_IP=$(curl -s $KUBE_API/namespaces/{{ include "common.namespace" . }}/services/dcae-pnda-mirror \ + --header "Authorization: Bearer $KUBE_TOKEN" \ + --insecure | jq -r '.status.loadBalancer.ingress[0].ip') + + if [ "$MIRROR_IP" != "null" ]; then + break + fi + sleep 5 +done + +PNDA_MIRROR="http://$MIRROR_IP:80" + +sed -i -e 's?CLIENT_IP/32?CLIENT_IP?' bootstrap-scripts/package-install.sh + +./cli/pnda-cli.py create -e $CLUSTER_PREFIX -f pico -n $DATANODES -k $KAFKANODES \ + -b $VERSION -s $KEYPAIR_NAME --set "mirrors.PNDA_MIRROR=$PNDA_MIRROR" |