diff options
Diffstat (limited to 'kud')
| -rw-r--r-- | kud/deployment_infra/emco/examples/01-cluster.yaml | 8 | ||||
| -rw-r--r-- | kud/deployment_infra/emco/examples/02-project.yaml | 6 | ||||
| -rw-r--r-- | kud/deployment_infra/emco/examples/03-addons-app.yaml | 68 | ||||
| -rw-r--r-- | kud/deployment_infra/emco/examples/04-addon-resources-app.yaml | 110 | ||||
| -rw-r--r-- | kud/deployment_infra/emco/examples/README.md | 34 | ||||
| -rw-r--r-- | kud/deployment_infra/emco/examples/values.yaml.example | 46 | ||||
| -rw-r--r-- | kud/hosting_providers/containerized/addons/README.md.tmpl | 38 | ||||
| -rw-r--r-- | kud/hosting_providers/containerized/addons/values.yaml.tmpl | 50 | ||||
| -rwxr-xr-x | kud/tests/plugin_fw_v2.sh | 4 |
9 files changed, 142 insertions, 222 deletions
diff --git a/kud/deployment_infra/emco/examples/01-cluster.yaml b/kud/deployment_infra/emco/examples/01-cluster.yaml index 6f7ce4ba..18d05f73 100644 --- a/kud/deployment_infra/emco/examples/01-cluster.yaml +++ b/kud/deployment_infra/emco/examples/01-cluster.yaml @@ -9,14 +9,14 @@ resourceContext: metadata : name: {{ .ClusterProvider }} -{{- range $index, $cluster := .Clusters }} +{{- range $clusterName, $cluster := .Clusters }} --- #creating cluster version: emco/v2 resourceContext: anchor: cluster-providers/{{ $.ClusterProvider }}/clusters metadata : - name: {{ $cluster.Name }} + name: {{ $clusterName }} file: {{ $cluster.KubeConfig }} @@ -24,6 +24,6 @@ file: #Add label cluster version: emco/v2 resourceContext: - anchor: cluster-providers/{{ $.ClusterProvider }}/clusters/{{ $cluster.Name }}/labels + anchor: cluster-providers/{{ $.ClusterProvider }}/clusters/{{ $clusterName }}/labels label-name: {{ $.ClustersLabel }} -{{- end }}
\ No newline at end of file +{{- end }} diff --git a/kud/deployment_infra/emco/examples/02-project.yaml b/kud/deployment_infra/emco/examples/02-project.yaml index d62a4f65..224126ce 100644 --- a/kud/deployment_infra/emco/examples/02-project.yaml +++ b/kud/deployment_infra/emco/examples/02-project.yaml @@ -65,17 +65,17 @@ spec: verbs: - "*" -{{- range $index, $cluster := .Clusters }} +{{- range $clusterName, $cluster := .Clusters }} --- #add cluster reference to logical cloud version: emco/v2 resourceContext: anchor: projects/{{ $.ProjectName }}/logical-clouds/{{ $.LogicalCloud }}/cluster-references metadata: - name: {{ $cluster.Name }} + name: {{ $clusterName }} spec: cluster-provider: {{ $.ClusterProvider }} - cluster-name: {{ $cluster.Name }} + cluster-name: {{ $clusterName }} loadbalancer-ip: "0.0.0.0" {{- end }} diff --git a/kud/deployment_infra/emco/examples/03-addons-app.yaml b/kud/deployment_infra/emco/examples/03-addons-app.yaml index 0fd15e0f..c5fb7ea5 100644 --- a/kud/deployment_infra/emco/examples/03-addons-app.yaml +++ b/kud/deployment_infra/emco/examples/03-addons-app.yaml @@ -1,96 +1,95 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation +{{- range $compositeAppName, $compositeApp := .CompositeApps }} --- #creating composite app entry version: emco/v2 resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps + anchor: projects/{{ $.ProjectName }}/composite-apps metadata : - name: {{ .AddonsApp }} - description: "KUD addons" + name: {{ $compositeAppName }} spec: version: v1 -{{- range $index, $addon := .Addons }} +{{- range $index, $app := $compositeApp.Apps }} --- #adding app to the composite app version: emco/v2 resourceContext: - anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/apps + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/apps metadata : - name: {{ $addon }} + name: {{ $app }} file: - {{ $.PackagesPath }}/{{ $addon }}.tar.gz + {{ $.PackagesPath }}/{{ $app }}.tar.gz {{- end }} --- #creating composite profile entry version: emco/v2 resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/composite-profiles + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/composite-profiles metadata : - name: {{ .AddonsProfile }} + name: {{ $compositeAppName }} -{{- range $index, $addon := .Addons }} +{{- range $index, $app := $compositeApp.Apps }} --- #adding app profiles to the composite profile version: emco/v2 resourceContext: - anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/composite-profiles/{{ $.AddonsProfile }}/profiles + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/composite-profiles/{{ $compositeAppName }}/profiles metadata : - name: {{ $addon }}-profile + name: {{ $app }}-profile spec: - app-name: {{ $addon }} + app-name: {{ $app }} file: - {{ $.PackagesPath }}/{{ $addon }}_profile.tar.gz + {{ $.PackagesPath }}/{{ $app }}_profile.tar.gz {{- end }} --- #create deployment intent group version: emco/v2 resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups metadata : - name: {{ .AddonsDeploymentIntentGroup }} - description: "description" + name: deployment spec: - profile: {{ .AddonsProfile }} + profile: {{ $compositeAppName }} version: r1 - logical-cloud: {{ .LogicalCloud }} + logical-cloud: {{ $.LogicalCloud }} override-values: [] --- #create intent in deployment intent group version: emco/v2 resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/intents + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/intents metadata : - name: {{ .AddonsDeploymentIntent }} + name: deployment-intent spec: intent: - genericPlacementIntent: {{ .AddonsPlacementIntent }} + genericPlacementIntent: placement-intent --- #create the generic placement intent version: emco/v2 resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/generic-placement-intents + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/generic-placement-intents metadata : - name: {{ .AddonsPlacementIntent }} + name: placement-intent spec: - logical-cloud: {{ .LogicalCloud }} + logical-cloud: {{ $.LogicalCloud }} -{{- range $index, $addon := .Addons }} +{{- range $index, $app := $compositeApp.Apps }} --- #add the app placement intent to the generic placement intent version: emco/v2 resourceContext: - anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/deployment-intent-groups/{{ $.AddonsDeploymentIntentGroup }}/generic-placement-intents/{{ $.AddonsPlacementIntent }}/app-intents + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/generic-placement-intents/placement-intent/app-intents metadata: - name: {{ $addon }}-placement-intent + name: {{ $app }}-placement-intent spec: - app-name: {{ $addon }} + app-name: {{ $app }} intent: allOf: - provider-name: {{ $.ClusterProvider }} @@ -98,13 +97,8 @@ spec: {{- end }} --- -#Approve +#approve version: emco/v2 resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/approve - ---- -#Instantiate -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/instantiate + anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/approve +{{- end }} diff --git a/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml b/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml deleted file mode 100644 index 92fd9539..00000000 --- a/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - ---- -#creating composite app entry -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps -metadata : - name: {{ .AddonResourcesApp }} - description: "KUD addons" -spec: - version: v1 - -{{- range $index, $addon := .AddonResources }} ---- -#adding app to the composite app -version: emco/v2 -resourceContext: - anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/apps -metadata : - name: {{ $addon }} -file: - {{ $.PackagesPath }}/{{ $addon }}.tar.gz -{{- end }} - ---- -#creating composite profile entry -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/composite-profiles -metadata : - name: {{ .AddonResourcesProfile }} - -{{- range $index, $addon := .AddonResources }} ---- -#adding app profiles to the composite profile -version: emco/v2 -resourceContext: - anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/composite-profiles/{{ $.AddonResourcesProfile }}/profiles -metadata : - name: {{ $addon }}-profile -spec: - app-name: {{ $addon }} -file: - {{ $.PackagesPath }}/{{ $addon }}_profile.tar.gz -{{- end }} - ---- -#create deployment intent group -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups -metadata : - name: {{ .AddonResourcesDeploymentIntentGroup }} - description: "description" -spec: - profile: {{ .AddonResourcesProfile }} - version: r1 - logical-cloud: {{ .LogicalCloud }} - override-values: [] - ---- -#create intent in deployment intent group -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/intents -metadata : - name: {{ .AddonResourcesDeploymentIntent }} -spec: - intent: - genericPlacementIntent: {{ .AddonResourcesPlacementIntent }} - ---- -#create the generic placement intent -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/generic-placement-intents -metadata : - name: {{ .AddonResourcesPlacementIntent }} -spec: - logical-cloud: {{ .LogicalCloud }} - -{{- range $index, $addon := .AddonResources }} ---- -#add the app placement intent to the generic placement intent -version: emco/v2 -resourceContext: - anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/deployment-intent-groups/{{ $.AddonResourcesDeploymentIntentGroup }}/generic-placement-intents/{{ $.AddonResourcesPlacementIntent }}/app-intents -metadata: - name: {{ $addon }}-placement-intent -spec: - app-name: {{ $addon }} - intent: - allOf: - - provider-name: {{ $.ClusterProvider }} - cluster-label-name: {{ $.ClustersLabel }} -{{- end }} - ---- -#Approve -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/approve - ---- -#Instantiate -version: emco/v2 -resourceContext: - anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/instantiate diff --git a/kud/deployment_infra/emco/examples/README.md b/kud/deployment_infra/emco/examples/README.md index 203b83fd..dcf9edde 100644 --- a/kud/deployment_infra/emco/examples/README.md +++ b/kud/deployment_infra/emco/examples/README.md @@ -15,6 +15,8 @@ needs to be installed and configured for the edge cluster. 5. SR-IOV Network 6. QuickAssist Technology (QAT) Device Plugin 7. CPU Manager for Kubernetes +8. KubeVirt and CDI Operators +9. KubeVirt and CDI Instances ## Setup environment to deploy addons @@ -38,22 +40,40 @@ required to be done only once. `$ emcoctl apply -f 01-cluster.yaml -v values.yaml` `$ emcoctl apply -f 02-project.yaml -v values.yaml` -## Deploying addons +## Create addons project -This deploys the applications listed in the `Addons` and -`AddonResources` values. +This creates the project with the addons listed `CompositeApps` value. `$ emcoctl apply -f 03-addons-app.yaml -v values.yaml` - `$ emcoctl apply -f 04-addon-resources-app.yaml -v values.yaml` + +## Instantiate the addons + +This instantiates each composite app listed in the `CompositeApps` +value. + +NOTE: The ordering is important when both the sriov-network and +kubevirt addons are enabled. The sriov-network addon will trigger a +drain of the nodes and kubevirt will prevent the drain from +completing, so kubevirt must be instantiated after sriov-network has +completed the drain. + + `$ emcoctl apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/instantiate` + `$ emcoctl apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/instantiate` + `$ emcoctl apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/instantiate` ## Cleanup -1. Delete addons. +1. Terminate addons. + + `$ emcoctl apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/terminate` + `$ emcoctl apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/terminate` + `$ emcoctl apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/terminate` + +2. Delete addons. - `$ emcoctl delete -f 04-addon-resources-app.yaml -v values.yaml` `$ emcoctl delete -f 03-addons-app.yaml -v values.yaml` -2. Cleanup prerequisites. +3. Cleanup prerequisites. `$ emcoctl delete -f 02-project.yaml -v values.yaml` `$ emcoctl delete -f 01-cluster.yaml -v values.yaml` diff --git a/kud/deployment_infra/emco/examples/values.yaml.example b/kud/deployment_infra/emco/examples/values.yaml.example index 67944eb8..41e3cc82 100644 --- a/kud/deployment_infra/emco/examples/values.yaml.example +++ b/kud/deployment_infra/emco/examples/values.yaml.example @@ -7,31 +7,33 @@ DtcPort: 30483 ClusterProvider: kud ClustersLabel: kud-cluster Clusters: -- KubeConfig: $KUBE_PATH - Name: cluster + cluster: + KubeConfig: $KUBE_PATH ProjectName: kud LogicalCloud: kud PackagesPath: $PWD/../output/packages -AddonsApp: addons -AddonsProfile: addons-profile -AddonsDeploymentIntentGroup: addons-deployment-intent-group -AddonsDeploymentIntent: addons-deployment-intent -AddonsPlacementIntent: addons-placement-intent -Addons: -- multus-cni -- ovn4nfv -- node-feature-discovery -- sriov-network-operator -- qat-device-plugin -- cpu-manager -AddonResourcesApp: addon-resources -AddonResourcesProfile: addon-resources-profile -AddonResourcesDeploymentIntentGroup: addon-resources-deployment-intent-group -AddonResourcesDeploymentIntent: addon-resources-deployment-intent -AddonResourcesPlacementIntent: addon-resources-placement-intent -AddonResources: -- ovn4nfv-network -- sriov-network +# Each composite app will be contained in its own deployment intent +# group. This is to enable instantiating the addons in a specified +# order. +CompositeApps: + addons: + Apps: + - kubevirt-operator + - cdi-operator + - multus-cni + - ovn4nfv + - node-feature-discovery + - sriov-network-operator + - qat-device-plugin + - cpu-manager + networks: + Apps: + - ovn4nfv-network + - sriov-network + kubevirt: + Apps: + - kubevirt + - cdi diff --git a/kud/hosting_providers/containerized/addons/README.md.tmpl b/kud/hosting_providers/containerized/addons/README.md.tmpl index 4ed4610a..eed30b43 100644 --- a/kud/hosting_providers/containerized/addons/README.md.tmpl +++ b/kud/hosting_providers/containerized/addons/README.md.tmpl @@ -22,28 +22,40 @@ cloud. \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 01-cluster.yaml -v values.yaml\` \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 02-project.yaml -v values.yaml\` -3. Deploy addons +3. Create addons project -This deploys the addons listed in the \`Addons\` and -\`AddonResources\` values in values.yaml. - -NOTE: On a single node cluster, the SRIOV addon resource will trigger -a drain of the worker node. KubeVirt will prevent the drain from -completing due to its PodDisruptionBudget. The workaround is to scale -down the KubeVirt operator before applying 04-addon-resources-app.yaml -then scaling it back up after the node is ready again. +This creates the project with the addons listed `CompositeApps` value. \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 03-addons-app.yaml -v values.yaml\` - \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 04-addon-resources-app.yaml -v values.yaml\` + +4. Instantiate the addons + +This instantiates each composite app listed in the `CompositeApps` +value. + +NOTE: The ordering is important when both the sriov-network and +kubevirt addons are enabled. The sriov-network addon will trigger a +drain of the nodes and kubevirt will prevent the drain from +completing, so kubevirt must be instantiated after sriov-network has +completed the drain. + + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/instantiate\` + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/instantiate\` + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/instantiate\` # Uninstalling KUD addons with emcoctl -1. Delete addons +1. Terminate the addons + + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/terminate\` + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/terminate\` + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/terminate\` + +2. Delete addons - \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 04-addon-resources-app.yaml -v values.yaml\` \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 03-addons-app.yaml -v values.yaml\` -2. Cleanup prerequisites +3. Cleanup prerequisites \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 02-project.yaml -v values.yaml\` \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 01-cluster.yaml -v values.yaml\` diff --git a/kud/hosting_providers/containerized/addons/values.yaml.tmpl b/kud/hosting_providers/containerized/addons/values.yaml.tmpl index b3e5845c..25e4cc48 100644 --- a/kud/hosting_providers/containerized/addons/values.yaml.tmpl +++ b/kud/hosting_providers/containerized/addons/values.yaml.tmpl @@ -7,35 +7,33 @@ DtcPort: 30483 ClusterProvider: kud ClustersLabel: kud-cluster Clusters: -- KubeConfig: ${KUBE_PATH} - Name: cluster + cluster: + KubeConfig: ${KUBE_PATH} ProjectName: kud LogicalCloud: kud PackagesPath: ${PACKAGES_PATH} -AddonsApp: addons -AddonsProfile: addons-profile -AddonsDeploymentIntentGroup: addons-deployment-intent-group -AddonsDeploymentIntent: addons-deployment-intent -AddonsPlacementIntent: addons-placement-intent -Addons: -- kubevirt-operator -- cdi-operator -- multus-cni -- ovn4nfv -- node-feature-discovery -- sriov-network-operator -- qat-device-plugin -- cpu-manager -AddonResourcesApp: addon-resources -AddonResourcesProfile: addon-resources-profile -AddonResourcesDeploymentIntentGroup: addon-resources-deployment-intent-group -AddonResourcesDeploymentIntent: addon-resources-deployment-intent -AddonResourcesPlacementIntent: addon-resources-placement-intent -AddonResources: -- ovn4nfv-network -- sriov-network -- kubevirt -- cdi +# Each composite app will be contained in its own deployment intent +# group. This is to enable instantiating the addons in a specified +# order. +CompositeApps: + addons: + Apps: + - kubevirt-operator + - cdi-operator + - multus-cni + - ovn4nfv + - node-feature-discovery + - sriov-network-operator + - qat-device-plugin + - cpu-manager + networks: + Apps: + - ovn4nfv-network + - sriov-network + kubevirt: + Apps: + - kubevirt + - cdi diff --git a/kud/tests/plugin_fw_v2.sh b/kud/tests/plugin_fw_v2.sh index abab9004..f19e0f90 100755 --- a/kud/tests/plugin_fw_v2.sh +++ b/kud/tests/plugin_fw_v2.sh @@ -223,6 +223,10 @@ else KUBECONFIG=$file kubectl get providernetwork emco-private-net -o name KUBECONFIG=$file kubectl get providernetwork unprotected-private-net -o name done + # Give some time for the Pods to show up on the clusters. kubectl + # wait may return with "error: no matching resources found" if the + # Pods have not started yet. + sleep 30s for name in $(cluster_names); do print_msg "Wait for all pods to start on cluster $name" file=$(cluster_file "$name") |