summaryrefslogtreecommitdiffstats
path: root/kud/deployment_infra/emco/examples
diff options
context:
space:
mode:
authorTodd Malsbary <todd.malsbary@intel.com>2021-09-09 11:24:45 -0700
committerTodd Malsbary <todd.malsbary@intel.com>2021-09-15 09:40:39 -0700
commit6027abb96369a30e6ccd3f747f4029e36232c0af (patch)
treef9c5ed58c4737e5728b67280debbba4b2eb034dd /kud/deployment_infra/emco/examples
parent5aa8c4de9fd620ef42ac5bf73b62f76d80e713a0 (diff)
Fix KubeVirt and SR-IOV addon interaction
SR-IOV wants to drain the nodes during reconciliation of SriovNetwork resources, while KubeVirt wants to keep at least one instance running at all times via a PodDisruptionBudget. KubeVirt's behavior is not customizable, so split the addons into different composite apps that allow finer control of the instantiation order. Issue-ID: MULTICLOUD-1324 Signed-off-by: Todd Malsbary <todd.malsbary@intel.com> Change-Id: I15c5cec3ef524b0b1d60dc201e04157272cbe376
Diffstat (limited to 'kud/deployment_infra/emco/examples')
-rw-r--r--kud/deployment_infra/emco/examples/01-cluster.yaml8
-rw-r--r--kud/deployment_infra/emco/examples/02-project.yaml6
-rw-r--r--kud/deployment_infra/emco/examples/03-addons-app.yaml68
-rw-r--r--kud/deployment_infra/emco/examples/04-addon-resources-app.yaml110
-rw-r--r--kud/deployment_infra/emco/examples/README.md34
-rw-r--r--kud/deployment_infra/emco/examples/values.yaml.example46
6 files changed, 89 insertions, 183 deletions
diff --git a/kud/deployment_infra/emco/examples/01-cluster.yaml b/kud/deployment_infra/emco/examples/01-cluster.yaml
index 6f7ce4ba..18d05f73 100644
--- a/kud/deployment_infra/emco/examples/01-cluster.yaml
+++ b/kud/deployment_infra/emco/examples/01-cluster.yaml
@@ -9,14 +9,14 @@ resourceContext:
metadata :
name: {{ .ClusterProvider }}
-{{- range $index, $cluster := .Clusters }}
+{{- range $clusterName, $cluster := .Clusters }}
---
#creating cluster
version: emco/v2
resourceContext:
anchor: cluster-providers/{{ $.ClusterProvider }}/clusters
metadata :
- name: {{ $cluster.Name }}
+ name: {{ $clusterName }}
file:
{{ $cluster.KubeConfig }}
@@ -24,6 +24,6 @@ file:
#Add label cluster
version: emco/v2
resourceContext:
- anchor: cluster-providers/{{ $.ClusterProvider }}/clusters/{{ $cluster.Name }}/labels
+ anchor: cluster-providers/{{ $.ClusterProvider }}/clusters/{{ $clusterName }}/labels
label-name: {{ $.ClustersLabel }}
-{{- end }} \ No newline at end of file
+{{- end }}
diff --git a/kud/deployment_infra/emco/examples/02-project.yaml b/kud/deployment_infra/emco/examples/02-project.yaml
index d62a4f65..224126ce 100644
--- a/kud/deployment_infra/emco/examples/02-project.yaml
+++ b/kud/deployment_infra/emco/examples/02-project.yaml
@@ -65,17 +65,17 @@ spec:
verbs:
- "*"
-{{- range $index, $cluster := .Clusters }}
+{{- range $clusterName, $cluster := .Clusters }}
---
#add cluster reference to logical cloud
version: emco/v2
resourceContext:
anchor: projects/{{ $.ProjectName }}/logical-clouds/{{ $.LogicalCloud }}/cluster-references
metadata:
- name: {{ $cluster.Name }}
+ name: {{ $clusterName }}
spec:
cluster-provider: {{ $.ClusterProvider }}
- cluster-name: {{ $cluster.Name }}
+ cluster-name: {{ $clusterName }}
loadbalancer-ip: "0.0.0.0"
{{- end }}
diff --git a/kud/deployment_infra/emco/examples/03-addons-app.yaml b/kud/deployment_infra/emco/examples/03-addons-app.yaml
index 0fd15e0f..c5fb7ea5 100644
--- a/kud/deployment_infra/emco/examples/03-addons-app.yaml
+++ b/kud/deployment_infra/emco/examples/03-addons-app.yaml
@@ -1,96 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2020 Intel Corporation
+{{- range $compositeAppName, $compositeApp := .CompositeApps }}
---
#creating composite app entry
version: emco/v2
resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps
+ anchor: projects/{{ $.ProjectName }}/composite-apps
metadata :
- name: {{ .AddonsApp }}
- description: "KUD addons"
+ name: {{ $compositeAppName }}
spec:
version: v1
-{{- range $index, $addon := .Addons }}
+{{- range $index, $app := $compositeApp.Apps }}
---
#adding app to the composite app
version: emco/v2
resourceContext:
- anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/apps
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/apps
metadata :
- name: {{ $addon }}
+ name: {{ $app }}
file:
- {{ $.PackagesPath }}/{{ $addon }}.tar.gz
+ {{ $.PackagesPath }}/{{ $app }}.tar.gz
{{- end }}
---
#creating composite profile entry
version: emco/v2
resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/composite-profiles
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/composite-profiles
metadata :
- name: {{ .AddonsProfile }}
+ name: {{ $compositeAppName }}
-{{- range $index, $addon := .Addons }}
+{{- range $index, $app := $compositeApp.Apps }}
---
#adding app profiles to the composite profile
version: emco/v2
resourceContext:
- anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/composite-profiles/{{ $.AddonsProfile }}/profiles
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/composite-profiles/{{ $compositeAppName }}/profiles
metadata :
- name: {{ $addon }}-profile
+ name: {{ $app }}-profile
spec:
- app-name: {{ $addon }}
+ app-name: {{ $app }}
file:
- {{ $.PackagesPath }}/{{ $addon }}_profile.tar.gz
+ {{ $.PackagesPath }}/{{ $app }}_profile.tar.gz
{{- end }}
---
#create deployment intent group
version: emco/v2
resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups
metadata :
- name: {{ .AddonsDeploymentIntentGroup }}
- description: "description"
+ name: deployment
spec:
- profile: {{ .AddonsProfile }}
+ profile: {{ $compositeAppName }}
version: r1
- logical-cloud: {{ .LogicalCloud }}
+ logical-cloud: {{ $.LogicalCloud }}
override-values: []
---
#create intent in deployment intent group
version: emco/v2
resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/intents
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/intents
metadata :
- name: {{ .AddonsDeploymentIntent }}
+ name: deployment-intent
spec:
intent:
- genericPlacementIntent: {{ .AddonsPlacementIntent }}
+ genericPlacementIntent: placement-intent
---
#create the generic placement intent
version: emco/v2
resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/generic-placement-intents
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/generic-placement-intents
metadata :
- name: {{ .AddonsPlacementIntent }}
+ name: placement-intent
spec:
- logical-cloud: {{ .LogicalCloud }}
+ logical-cloud: {{ $.LogicalCloud }}
-{{- range $index, $addon := .Addons }}
+{{- range $index, $app := $compositeApp.Apps }}
---
#add the app placement intent to the generic placement intent
version: emco/v2
resourceContext:
- anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/deployment-intent-groups/{{ $.AddonsDeploymentIntentGroup }}/generic-placement-intents/{{ $.AddonsPlacementIntent }}/app-intents
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/generic-placement-intents/placement-intent/app-intents
metadata:
- name: {{ $addon }}-placement-intent
+ name: {{ $app }}-placement-intent
spec:
- app-name: {{ $addon }}
+ app-name: {{ $app }}
intent:
allOf:
- provider-name: {{ $.ClusterProvider }}
@@ -98,13 +97,8 @@ spec:
{{- end }}
---
-#Approve
+#approve
version: emco/v2
resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/approve
-
----
-#Instantiate
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/instantiate
+ anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/approve
+{{- end }}
diff --git a/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml b/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml
deleted file mode 100644
index 92fd9539..00000000
--- a/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml
+++ /dev/null
@@ -1,110 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# Copyright (c) 2020 Intel Corporation
-
----
-#creating composite app entry
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps
-metadata :
- name: {{ .AddonResourcesApp }}
- description: "KUD addons"
-spec:
- version: v1
-
-{{- range $index, $addon := .AddonResources }}
----
-#adding app to the composite app
-version: emco/v2
-resourceContext:
- anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/apps
-metadata :
- name: {{ $addon }}
-file:
- {{ $.PackagesPath }}/{{ $addon }}.tar.gz
-{{- end }}
-
----
-#creating composite profile entry
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/composite-profiles
-metadata :
- name: {{ .AddonResourcesProfile }}
-
-{{- range $index, $addon := .AddonResources }}
----
-#adding app profiles to the composite profile
-version: emco/v2
-resourceContext:
- anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/composite-profiles/{{ $.AddonResourcesProfile }}/profiles
-metadata :
- name: {{ $addon }}-profile
-spec:
- app-name: {{ $addon }}
-file:
- {{ $.PackagesPath }}/{{ $addon }}_profile.tar.gz
-{{- end }}
-
----
-#create deployment intent group
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups
-metadata :
- name: {{ .AddonResourcesDeploymentIntentGroup }}
- description: "description"
-spec:
- profile: {{ .AddonResourcesProfile }}
- version: r1
- logical-cloud: {{ .LogicalCloud }}
- override-values: []
-
----
-#create intent in deployment intent group
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/intents
-metadata :
- name: {{ .AddonResourcesDeploymentIntent }}
-spec:
- intent:
- genericPlacementIntent: {{ .AddonResourcesPlacementIntent }}
-
----
-#create the generic placement intent
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/generic-placement-intents
-metadata :
- name: {{ .AddonResourcesPlacementIntent }}
-spec:
- logical-cloud: {{ .LogicalCloud }}
-
-{{- range $index, $addon := .AddonResources }}
----
-#add the app placement intent to the generic placement intent
-version: emco/v2
-resourceContext:
- anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/deployment-intent-groups/{{ $.AddonResourcesDeploymentIntentGroup }}/generic-placement-intents/{{ $.AddonResourcesPlacementIntent }}/app-intents
-metadata:
- name: {{ $addon }}-placement-intent
-spec:
- app-name: {{ $addon }}
- intent:
- allOf:
- - provider-name: {{ $.ClusterProvider }}
- cluster-label-name: {{ $.ClustersLabel }}
-{{- end }}
-
----
-#Approve
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/approve
-
----
-#Instantiate
-version: emco/v2
-resourceContext:
- anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/instantiate
diff --git a/kud/deployment_infra/emco/examples/README.md b/kud/deployment_infra/emco/examples/README.md
index 203b83fd..dcf9edde 100644
--- a/kud/deployment_infra/emco/examples/README.md
+++ b/kud/deployment_infra/emco/examples/README.md
@@ -15,6 +15,8 @@ needs to be installed and configured for the edge cluster.
5. SR-IOV Network
6. QuickAssist Technology (QAT) Device Plugin
7. CPU Manager for Kubernetes
+8. KubeVirt and CDI Operators
+9. KubeVirt and CDI Instances
## Setup environment to deploy addons
@@ -38,22 +40,40 @@ required to be done only once.
`$ emcoctl apply -f 01-cluster.yaml -v values.yaml`
`$ emcoctl apply -f 02-project.yaml -v values.yaml`
-## Deploying addons
+## Create addons project
-This deploys the applications listed in the `Addons` and
-`AddonResources` values.
+This creates the project with the addons listed `CompositeApps` value.
`$ emcoctl apply -f 03-addons-app.yaml -v values.yaml`
- `$ emcoctl apply -f 04-addon-resources-app.yaml -v values.yaml`
+
+## Instantiate the addons
+
+This instantiates each composite app listed in the `CompositeApps`
+value.
+
+NOTE: The ordering is important when both the sriov-network and
+kubevirt addons are enabled. The sriov-network addon will trigger a
+drain of the nodes and kubevirt will prevent the drain from
+completing, so kubevirt must be instantiated after sriov-network has
+completed the drain.
+
+ `$ emcoctl apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/instantiate`
+ `$ emcoctl apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/instantiate`
+ `$ emcoctl apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/instantiate`
## Cleanup
-1. Delete addons.
+1. Terminate addons.
+
+ `$ emcoctl apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/terminate`
+ `$ emcoctl apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/terminate`
+ `$ emcoctl apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/terminate`
+
+2. Delete addons.
- `$ emcoctl delete -f 04-addon-resources-app.yaml -v values.yaml`
`$ emcoctl delete -f 03-addons-app.yaml -v values.yaml`
-2. Cleanup prerequisites.
+3. Cleanup prerequisites.
`$ emcoctl delete -f 02-project.yaml -v values.yaml`
`$ emcoctl delete -f 01-cluster.yaml -v values.yaml`
diff --git a/kud/deployment_infra/emco/examples/values.yaml.example b/kud/deployment_infra/emco/examples/values.yaml.example
index 67944eb8..41e3cc82 100644
--- a/kud/deployment_infra/emco/examples/values.yaml.example
+++ b/kud/deployment_infra/emco/examples/values.yaml.example
@@ -7,31 +7,33 @@ DtcPort: 30483
ClusterProvider: kud
ClustersLabel: kud-cluster
Clusters:
-- KubeConfig: $KUBE_PATH
- Name: cluster
+ cluster:
+ KubeConfig: $KUBE_PATH
ProjectName: kud
LogicalCloud: kud
PackagesPath: $PWD/../output/packages
-AddonsApp: addons
-AddonsProfile: addons-profile
-AddonsDeploymentIntentGroup: addons-deployment-intent-group
-AddonsDeploymentIntent: addons-deployment-intent
-AddonsPlacementIntent: addons-placement-intent
-Addons:
-- multus-cni
-- ovn4nfv
-- node-feature-discovery
-- sriov-network-operator
-- qat-device-plugin
-- cpu-manager
-AddonResourcesApp: addon-resources
-AddonResourcesProfile: addon-resources-profile
-AddonResourcesDeploymentIntentGroup: addon-resources-deployment-intent-group
-AddonResourcesDeploymentIntent: addon-resources-deployment-intent
-AddonResourcesPlacementIntent: addon-resources-placement-intent
-AddonResources:
-- ovn4nfv-network
-- sriov-network
+# Each composite app will be contained in its own deployment intent
+# group. This is to enable instantiating the addons in a specified
+# order.
+CompositeApps:
+ addons:
+ Apps:
+ - kubevirt-operator
+ - cdi-operator
+ - multus-cni
+ - ovn4nfv
+ - node-feature-discovery
+ - sriov-network-operator
+ - qat-device-plugin
+ - cpu-manager
+ networks:
+ Apps:
+ - ovn4nfv-network
+ - sriov-network
+ kubevirt:
+ Apps:
+ - kubevirt
+ - cdi