Merge "[COMMON][MARIADB] Upgrade Mariadb DB galera version"

1 files changed, 542 insertions, 88 deletions

diff --git a/kubernetes/common/mariadb-galera/values.yaml b/kubernetes/common/mariadb-galera/values.yaml
index 6b1676fba7..12d2d75e38 100644
--- a/kubernetes/common/mariadb-galera/values.yaml
+++ b/kubernetes/common/mariadb-galera/values.yaml
@@ -1,5 +1,6 @@
 # Copyright © 2018 Amdocs, Bell Canada
 # Copyright © 2019 Samsung Electronics
+# Copyright © 2020 Bitnami, Orange
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,20 +14,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 #################################################################
 # Secrets metaconfig
 #################################################################
 secrets:
   - uid: '{{ include "common.mariadb.secret.rootPassUID" . }}'
     type: password
-    externalSecret: '{{ tpl (default "" .Values.config.mariadbRootPasswordExternalSecret) . }}'
-    password: '{{ .Values.config.mariadbRootPassword }}'
+    externalSecret: '{{ tpl (default "" .Values.rootUser.externalSecret) . }}'
+    password: '{{ .Values.rootUser.password }}'
   - uid: '{{ include "common.mariadb.secret.userCredentialsUID" . }}'
     type: basicAuth
-    externalSecret: '{{ tpl (default "" .Values.config.userCredentialsExternalSecret) . }}'
-    login: '{{ .Values.config.userName }}'
-    password: '{{ .Values.config.userPassword }}'
+    externalSecret: '{{ tpl (default "" .Values.db.externalSecret) . }}'
+    login: '{{ .Values.db.user }}'
+    password: '{{ .Values.db.password }}'
+  - uid: '{{ include "common.mariadb.secret.backupCredentialsUID" . }}'
+    type: basicAuth
+    externalSecret: '{{ tpl (default "" .Values.galera.mariabackup.externalSecret) . }}'
+    login: '{{ .Values.galera.mariabackup.user }}'
+    password: '{{ .Values.galera.mariabackup.password }}'
 
+# bitnami image doesn't support well single quote in password
+passwordStrengthOverride: basic
 
 #################################################################
 # Global configuration defaults.
@@ -37,56 +46,399 @@ global:
     mountPath: /dockerdata-nfs
     backup:
       mountPath: /dockerdata-nfs/backup
+  clusterDomain: cluster.local
+  metrics: {}
 
-#################################################################
-# Application configuration defaults.
-#################################################################
+image: bitnami/mariadb-galera:10.5.8
+## Specify a imagePullPolicy
+## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+##
+pullPolicy: Always
+
+## Set to true if you would like to see extra information on logs
+## It turns BASH debugging in minideb-extras-base
+##
+debug: true
+
+## Sometimes, especially when a lot of pods are created at the same time,
+## actions performed on the databases are tried to be done before actual start.
+init_sleep_time: 5
+
+## String to partially override common.names.fullname template (will maintain the release name)
+##
+nameOverride: mariadb-galera
+
+## Use an alternate scheduler, e.g. "stork".
+## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
+##
+# schedulerName:
+
+## StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel
+## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy
+##
+podManagementPolicy: OrderedReady
+
+## MariaDB Gallera K8s svc properties
+##
+service:
+  ## Kubernetes service type and port number
+  ##
+  type: ClusterIP
+  headless: {}
+  ports:
+    - name: mysql
+      port: 3306
+  headlessPorts:
+    - name: galera
+      port: 4567
+    - name: ist
+      port: 4568
+    - name: sst
+      port: 4444
+
+
+## Pods Service Account
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+##
+serviceAccount:
+  nameOverride: mariadb-galera
+  roles:
+    - read
+
+## Pod Security Context
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+##
+securityContext:
+  enabled: true
+  user_id: 10001
+  group_id: 10001
+
+## Database credentials for root (admin) user
+##
+rootUser:
+  ## MariaDB admin user
+  user: root
+  ## MariaDB admin password
+  ## Password is ignored if externalSecret is specified.
+  ## If not set, password will be "randomly" generated
+  ## ref: https://github.com/bitnami/bitnami-docker-mariadb-galera#setting-the-root-password-on-first-run
+  ##
+  # password:
+  # externalSecret:
+
+## Custom db configuration
+##
+db:
+  ## MariaDB username and password
+  ## Password is ignored if externalSecret is specified.
+  ## If not set, password will be "randomly" generated
+  ## ref: https://github.com/bitnami/bitnami-docker-mariadb-galera#creating-a-database-user-on-first-run
+  ##
+  user: my-user
+  # password:
+  # externalSecret:
+  ## Database to create
+  ## ref: https://github.com/bitnami/bitnami-docker-mariadb-galera#creating-a-database-on-first-run
+  ##
+  # name: my_database
+
+## Galera configuration
+##
+galera:
+  ## Galera cluster name
+  ##
+  name: galera
+
+  ## Bootstraping options
+  ## ref: https://github.com/bitnami/bitnami-docker-mariadb-galera#bootstraping
+  bootstrap:
+    ## Node to bootstrap from, you will need to change this parameter incase you want to bootstrap from other node
+    ##
+    bootstrapFromNode:
+    ## Force safe_to_bootstrap in grastate.date file.
+    ## This will set safe_to_bootstrap=1 in the node indicated by bootstrapFromNode.
+    forceSafeToBootstrap: false
+
+  ## Credentials to perform backups
+  ##
+  mariabackup:
+    ## MariaBackup username and password
+    ## Password is ignored if externalSecret is specified.
+    ## If not set, password will be "randomly" generated
+    ## ref: https://github.com/bitnami/bitnami-docker-mariadb-galera#setting-up-a-multi-master-cluster
+    ##
+    user: mariabackup
+    # password:
+    # externalSecret:
+
+backup:
+  enabled: false
+  cron: "00 00 * * *"
+  retentionPeriod: 3
+  persistence:
+    ## If true, use a Persistent Volume Claim, If false, use emptyDir
+    ##
+    enabled: true
+    # Enable persistence using an existing PVC
+    # existingClaim:
+    ## selector can be used to match an existing PersistentVolume
+    ## selector:
+    ##   matchLabels:
+    ##     app: my-app
+    selector: {}
+    ## Persistent Volume Storage Class
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: "-"
+    ## Persistent Volume Claim annotations
+    ##
+    annotations:
+    ## Persistent Volume Access Mode
+    ##
+    accessMode: ReadWriteOnce
+    ## Persistent Volume size
+    ##
+    size: 2Gi
+
+
+readinessCheck:
+  wait_for:
+    - '{{ include "common.name" . }}'
+
+## TLS configuration
+##
+tls:
+  ## Enable TLS
+  ##
+  enabled: false
+  ## Name of the secret that contains the certificates
+  ##
+  # certificatesSecret:
+  ## Certificate filename
+  ##
+  # certFilename:
+  ## Certificate Key filename
+  ##
+  # certKeyFilename:
+  ## CA Certificate filename
+  ##
+  # certCAFilename:
+
+## Configure MariaDB with a custom my.cnf file
+## ref: https://mysql.com/kb/en/mysql/configuring-mysql-with-mycnf/#example-of-configuration-file
+## Alternatively, you can put your my.cnf under the files/ directory
+##
+mariadbConfiguration: |-
+  [client]
+  port=3306
+  socket=/opt/bitnami/mariadb/tmp/mysql.sock
+  plugin_dir=/opt/bitnami/mariadb/plugin
+
+  [mysqld]
+  lower_case_table_names = 1
+  default_storage_engine=InnoDB
+  basedir=/opt/bitnami/mariadb
+  datadir=/bitnami/mariadb/data
+  plugin_dir=/opt/bitnami/mariadb/plugin
+  tmpdir=/opt/bitnami/mariadb/tmp
+  socket=/opt/bitnami/mariadb/tmp/mysql.sock
+  pid_file=/opt/bitnami/mariadb/tmp/mysqld.pid
+  bind_address=0.0.0.0
+
+  ## Character set
+  collation_server=utf8_unicode_ci
+  init_connect='SET NAMES utf8'
+  character_set_server=utf8
+
+  ## MyISAM
+  key_buffer_size=32M
+  myisam_recover_options=FORCE,BACKUP
+
+  ## Safety
+  skip_host_cache
+  skip_name_resolve
+  max_allowed_packet=16M
+  max_connect_errors=1000000
+  sql_mode=STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY
+  sysdate_is_now=1
+
+  ## Binary Logging
+  log_bin=mysql-bin
+  expire_logs_days=14
+  # Disabling for performance per http://severalnines.com/blog/9-tips-going-production-galera-cluster-mysql
+  sync_binlog=0
+  # Required for Galera
+  binlog_format=row
+
+  ## Caches and Limits
+  tmp_table_size=32M
+  max_heap_table_size=32M
+  # Re-enabling as now works with Maria 10.1.2
+  query_cache_type=1
+  query_cache_limit=4M
+  query_cache_size=256M
+  max_connections=500
+  thread_cache_size=50
+  open_files_limit=65535
+  table_definition_cache=4096
+  table_open_cache=4096
+
+  ## InnoDB
+  innodb=FORCE
+  innodb_strict_mode=1
+  # Mandatory per https://github.com/codership/documentation/issues/25
+  innodb_autoinc_lock_mode=2
+  # Per https://www.percona.com/blog/2006/08/04/innodb-double-write/
+  innodb_doublewrite=1
+  innodb_flush_method=O_DIRECT
+  innodb_log_files_in_group=2
+  innodb_log_file_size=128M
+  innodb_flush_log_at_trx_commit=1
+  innodb_file_per_table=1
+  # 80% Memory is default reco.
+  # Need to re-evaluate when DB size grows
+  innodb_buffer_pool_size=2G
+  innodb_file_format=Barracuda
+
+  ## Logging
+  log_error=/opt/bitnami/mariadb/logs/mysqld.log
+  slow_query_log_file=/opt/bitnami/mariadb/logs/mysqld.log
+  log_queries_not_using_indexes=1
+  slow_query_log=1
+
+  ## SSL
+  ## Use extraVolumes and extraVolumeMounts to mount /certs filesystem
+  # ssl_ca=/certs/ca.pem
+  # ssl_cert=/certs/server-cert.pem
+  # ssl_key=/certs/server-key.pem
+
+  [galera]
+  wsrep_on=ON
+  wsrep_provider=/opt/bitnami/mariadb/lib/libgalera_smm.so
+  wsrep_sst_method=mariabackup
+  wsrep_slave_threads=4
+  wsrep_cluster_address=gcomm://
+  wsrep_cluster_name=galera
+  wsrep_sst_auth="root:"
+  # Enabled for performance per https://mariadb.com/kb/en/innodb-system-variables/#innodb_flush_log_at_trx_commit
+  innodb_flush_log_at_trx_commit=2
+  # MYISAM REPLICATION SUPPORT #
+  wsrep_replicate_myisam=ON
+
+  [mariadb]
+  plugin_load_add=auth_pam
+
+  ## Data-at-Rest Encryption
+  ## Use extraVolumes and extraVolumeMounts to mount /encryption filesystem
+  # plugin_load_add=file_key_management
+  # file_key_management_filename=/encryption/keyfile.enc
+  # file_key_management_filekey=FILE:/encryption/keyfile.key
+  # file_key_management_encryption_algorithm=AES_CTR
+  # encrypt_binlog=ON
+  # encrypt_tmp_files=ON
+
+  ## InnoDB/XtraDB Encryption
+  # innodb_encrypt_tables=ON
+  # innodb_encrypt_temporary_tables=ON
+  # innodb_encrypt_log=ON
+  # innodb_encryption_threads=4
+  # innodb_encryption_rotate_key_age=1
 
-#repository: mysql
-image: adfinissygroup/k8s-mariadb-galera-centos:v002
-pullPolicy: IfNotPresent
+  ## Aria Encryption
+  # aria_encrypt_tables=ON
+  # encrypt_tmp_disk_tables=ON
 
-# application configuration
-config:
-  # .mariadbRootPasswordExternalSecret: 'some-external-secret'
-  # mariadbRootPassword: secretpassword
-  # .userCredentialsExternalSecret: 'some-external-secret'
-  userName: my-user
-  # userPassword: my-password
-  # mysqlDatabase: my-database
+## MariaDB additional command line flags
+## Can be used to specify command line flags, for example:
+##
+## extraFlags: "--max-connect-errors=1000 --max_connections=155"
 
-# default number of instances in the StatefulSet
+## Desired number of cluster nodes
+##
 replicaCount: 3
 
-nodeSelector: {}
+## updateStrategy for MariaDB Master StatefulSet
+## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies
+##
+updateStrategy:
+  type: RollingUpdate
+
+## Additional pod annotations for MariaDB Galera pods
+## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+##
+podAnnotations: {}
+
+## Pod affinity preset
+## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity
+## Allowed values: soft, hard
+##
+podAffinityPreset: ""
+
+## Pod anti-affinity preset
+## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity
+## Allowed values: soft, hard
+##
+podAntiAffinityPreset: soft
+
+## Node affinity preset
+## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity
+## Allowed values: soft, hard
+##
+nodeAffinityPreset:
+  ## Node affinity type
+  ## Allowed values: soft, hard
+  type: ""
+  ## Node label key to match
+  ## E.g.
+  ## key: "kubernetes.io/e2e-az-name"
+  ##
+  key: ""
+  ## Node label values to match
+  ## E.g.
+  ## values:
+  ##   - e2e-az1
+  ##   - e2e-az2
+  ##
+  values: []
 
+## Affinity for pod assignment. Evaluated as a template.
+## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set
+##
 affinity: {}
 
-# probe configuration parameters
-liveness:
-  initialDelaySeconds: 30
-  periodSeconds: 10
-  timeoutSeconds: 5
-  # necessary to disable liveness probe when setting breakpoints
-  # in debugger so K8s doesn't restart unresponsive container
-  enabled: true
+## Node labels for pod assignment. Evaluated as a template.
+## ref: https://kubernetes.io/docs/user-guide/node-selection/
+##
+nodeSelector: {}
 
-readiness:
-  initialDelaySeconds: 15
-  periodSeconds: 10
-  timeoutSeconds: 5
+## Tolerations for pod assignment. Evaluated as a template.
+## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+##
+tolerations: []
 
-## Persist data to a persitent volume
+## Enable persistence using Persistent Volume Claims
+## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/
+##
 persistence:
-  enabled:  true
-
-  ## A manually managed Persistent Volume and Claim
-  ## Requires persistence.enabled: true
-  ## If defined, PVC must be created manually before volume will be bound
+  ## If true, use a Persistent Volume Claim, If false, use emptyDir
+  ##
+  enabled: true
+  # Enable persistence using an existing PVC
   # existingClaim:
-  volumeReclaimPolicy: Retain
-
-  ## database data Persistent Volume Storage Class
+  mountPath: /dockerdata-nfs
+  mountSubPath: "mariadb-galera/data"
+  ## selector can be used to match an existing PersistentVolume
+  ## selector:
+  ##   matchLabels:
+  ##     app: my-app
+  selector: {}
+  ## Persistent Volume Storage Class
   ## If defined, storageClassName: <storageClass>
   ## If set to "-", storageClassName: "", which disables dynamic provisioning
   ## If undefined (the default) or set to null, no storageClassName spec is
@@ -94,53 +446,34 @@ persistence:
   ##   GKE, AWS & OpenStack)
   ##
   # storageClass: "-"
+  ## Persistent Volume Claim annotations
+  ##
+  annotations:
+  ## Persistent Volume Access Mode
+  ##
   accessMode: ReadWriteOnce
+  ## Persistent Volume size
+  ##
   size: 2Gi
-  mountPath: /dockerdata-nfs
-  mountSubPath: "mariadb-galera/data"
-  mysqlPath: /var/lib/mysql
-  backup:
-    mountPath: /dockerdata-nfs/backup{{- if or (or .Values.storageClassOverride .Values.persistence.storageClass) .Values.global.persistence.storageClass -}}
 
-service:
-  internalPort: 3306
-  name: mariadb-galera
-  portName: mariadb-galera
-  sstPort: 4444
-  sstPortName: sst
-  replicationPort: 4567
-  replicationName: replication
-  istPort: 4568
-  istPortName: ist
-
-ingress:
-  enabled: false
+## Additional pod labels
+##
+# podLabels:
+#   extraLabel: extraValue
 
+## Priority Class Name
+#
+# priorityClassName: 'priorityClass'
 
-## Configure MariaDB-Galera with a custom my.cnf file
-## ref: https://mariadb.com/kb/en/mariadb/configuring-mariadb-with-mycnf/#example-of-configuration-file
-##
-#externalConfig: ""
-externalConfig: |-
-   [mysqld]
-   lower_case_table_names = 1
-
-#resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  #
-  # Example:
-  # Configure resource requests and limits
-  # ref: http://kubernetes.io/docs/user-guide/compute-resources/
-  # Minimum memory for development is 2 CPU cores and 4GB memory
-  # Minimum memory for production is 4 CPU cores and 8GB memory
+## MariaDB Galera containers' resource requests and limits
+## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+##
+flavor: small
 resources:
   small:
     limits:
       cpu: 500m
-      memory: 1.5Gi
+      memory: 2.5Gi
     requests:
       cpu: 100m
       memory: 750Mi
@@ -153,13 +486,134 @@ resources:
       memory: 2Gi
   unlimited: {}
 
-# Name for mariadb-galera cluster - should be unique accross all projects or other clusters
-nameOverride: mariadb-galera
+## MariaDB Galera containers' liveness and readiness probes
+## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes
+##
+livenessProbe:
+  enabled: true
+  ## Initializing the database could take some time
+  ##
+  initialDelaySeconds: 150
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+readinessProbe:
+  enabled: true
+  initialDelaySeconds: 60
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
 
-# DNS name for mariadb-galera cluster - should be unique accross all projects other clusters
-#dnsnameOverride: mariadb-galera
+## Pod disruption budget configuration
+##
+podDisruptionBudget:
+  ## Specifies whether a Pod disruption budget should be created
+  ##
+  create: true
+  minAvailable: 1
+  # maxUnavailable: 1
 
-backup:
-  enabled: false
-  cron: "00 00 * * *"
-  retentionPeriod: 3
+## Prometheus exporter configuration
+##
+metrics:
+  ## Bitnami MySQL Prometheus exporter image
+  ## ref: https://hub.docker.com/r/bitnami/mysqld-exporter/tags/
+  ##
+  image: bitnami/mysqld-exporter:0.12.1-debian-10-r264
+  pullPolicy: Always
+  ## MySQL exporter additional command line flags
+  ## Can be used to specify command line flags
+  ## E.g.:
+  ## extraFlags:
+  ##   - --collect.binlog_size
+  ##
+  extraFlags: []
+  ## MySQL Prometheus exporter containers' resource requests and limits
+  ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+  ##
+  resources:
+    # We usually recommend not to specify default resources and to leave this as a conscious
+    # choice for the user. This also increases chances charts run on environments with little
+    # resources, such as Minikube. If you do want to specify resources, uncomment the following
+    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    limits:
+      cpu: 0.5
+      memory: 256Mi
+    requests:
+      cpu: 0.5
+      memory: 256Mi
+  ## MySQL Prometheus exporter service parameters
+  ##
+  service:
+    type: ClusterIP
+    port: 9104
+    annotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/port: "9104"
+
+  ## Prometheus Operator ServiceMonitor configuration
+  ##
+  serviceMonitor:
+    enabled: false
+    ## Namespace in which Prometheus is running
+    ##
+    # namespace: monitoring
+
+    ## Interval at which metrics should be scraped.
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    # interval: 10s
+
+    ## Timeout after which the scrape is ended
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    # scrapeTimeout: 10s
+
+    ## ServiceMonitor selector labels
+    ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration
+    ##
+    selector:
+      prometheus: kube-prometheus
+
+    ## RelabelConfigs to apply to samples before scraping
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
+    ## Value is evalued as a template
+    ##
+    relabelings: []
+
+    ## MetricRelabelConfigs to apply to samples before ingestion
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
+    ## Value is evalued as a template
+    ##
+    metricRelabelings: []
+    #  - sourceLabels:
+    #      - "__name__"
+    #    targetLabel: "__name__"
+    #    action: replace
+    #    regex: '(.*)'
+    #    replacement: 'example_prefix_$1'
+
+  ## Prometheus Operator PrometheusRule configuration
+  ##
+  prometheusRules:
+    enabled: false
+
+    ## Additional labels to add to the PrometheusRule so it is picked up by the operator.
+    ## If using the [Helm Chart](https://github.com/helm/charts/tree/master/stable/prometheus-operator) this is the name of the Helm release and 'app: prometheus-operator'
+    selector:
+      app: prometheus-operator
+      release: prometheus
+
+    ## Rules as a map.
+    rules: {}
+    #  - alert: MariaDB-Down
+    #    annotations:
+    #      message: 'MariaDB instance {{ $labels.instance }} is down'
+    #      summary: MariaDB instance is down
+    #    expr: absent(up{job="mariadb-galera"} == 1)
+    #    labels:
+    #      severity: warning
+    #      service: mariadb-galera
+    #    for: 5m