Fix KubeVirt and SR-IOV addon interaction

SR-IOV wants to drain the nodes during reconciliation of SriovNetwork
resources, while KubeVirt wants to keep at least one instance running
at all times via a PodDisruptionBudget.

KubeVirt's behavior is not customizable, so split the addons into
different composite apps that allow finer control of the instantiation
order.

Issue-ID: MULTICLOUD-1324
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I15c5cec3ef524b0b1d60dc201e04157272cbe376

9 files changed, 142 insertions, 222 deletions

diff --git a/kud/deployment_infra/emco/examples/01-cluster.yaml b/kud/deployment_infra/emco/examples/01-cluster.yaml
index 6f7ce4ba..18d05f73 100644
--- a/kud/deployment_infra/emco/examples/01-cluster.yaml
+++ b/kud/deployment_infra/emco/examples/01-cluster.yaml
@@ -9,14 +9,14 @@ resourceContext:
 metadata :
    name: {{ .ClusterProvider }}
 
-{{- range $index, $cluster := .Clusters }}
+{{- range $clusterName, $cluster := .Clusters }}
 ---
 #creating cluster
 version: emco/v2
 resourceContext:
   anchor: cluster-providers/{{ $.ClusterProvider }}/clusters
 metadata :
-   name: {{ $cluster.Name }}
+   name: {{ $clusterName }}
 file:
   {{ $cluster.KubeConfig }}
 
@@ -24,6 +24,6 @@ file:
 #Add label cluster
 version: emco/v2
 resourceContext:
-  anchor: cluster-providers/{{ $.ClusterProvider }}/clusters/{{ $cluster.Name }}/labels
+  anchor: cluster-providers/{{ $.ClusterProvider }}/clusters/{{ $clusterName }}/labels
 label-name: {{ $.ClustersLabel }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/kud/deployment_infra/emco/examples/02-project.yaml b/kud/deployment_infra/emco/examples/02-project.yaml
index d62a4f65..224126ce 100644
--- a/kud/deployment_infra/emco/examples/02-project.yaml
+++ b/kud/deployment_infra/emco/examples/02-project.yaml
@@ -65,17 +65,17 @@ spec:
   verbs:
   - "*"
 
-{{- range $index, $cluster := .Clusters }}
+{{- range $clusterName, $cluster := .Clusters }}
 ---
 #add cluster reference to logical cloud
 version: emco/v2
 resourceContext:
   anchor: projects/{{ $.ProjectName }}/logical-clouds/{{ $.LogicalCloud }}/cluster-references
 metadata:
-  name: {{ $cluster.Name }}
+  name: {{ $clusterName }}
 spec:
   cluster-provider: {{ $.ClusterProvider }}
-  cluster-name: {{ $cluster.Name }}
+  cluster-name: {{ $clusterName }}
   loadbalancer-ip: "0.0.0.0"
 {{- end }}
 
diff --git a/kud/deployment_infra/emco/examples/03-addons-app.yaml b/kud/deployment_infra/emco/examples/03-addons-app.yaml
index 0fd15e0f..c5fb7ea5 100644
--- a/kud/deployment_infra/emco/examples/03-addons-app.yaml
+++ b/kud/deployment_infra/emco/examples/03-addons-app.yaml
@@ -1,96 +1,95 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright (c) 2020 Intel Corporation
 
+{{- range $compositeAppName, $compositeApp := .CompositeApps }}
 ---
 #creating composite app entry
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps
+  anchor: projects/{{ $.ProjectName }}/composite-apps
 metadata :
-  name: {{ .AddonsApp }}
-  description: "KUD addons"
+  name: {{ $compositeAppName }}
 spec:
   version: v1
 
-{{- range $index, $addon := .Addons }}
+{{- range $index, $app := $compositeApp.Apps }}
 ---
 #adding app to the composite app
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/apps
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/apps
 metadata :
-  name: {{ $addon }}
+  name: {{ $app }}
 file:
-  {{ $.PackagesPath }}/{{ $addon }}.tar.gz
+  {{ $.PackagesPath }}/{{ $app }}.tar.gz
 {{- end }}
 
 ---
 #creating composite profile entry
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/composite-profiles
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/composite-profiles
 metadata :
-  name: {{ .AddonsProfile }}
+  name: {{ $compositeAppName }}
 
-{{- range $index, $addon := .Addons }}
+{{- range $index, $app := $compositeApp.Apps }}
 ---
 #adding app profiles to the composite profile
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/composite-profiles/{{ $.AddonsProfile }}/profiles
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/composite-profiles/{{ $compositeAppName }}/profiles
 metadata :
-  name: {{ $addon }}-profile
+  name: {{ $app }}-profile
 spec:
-  app-name: {{ $addon }}
+  app-name: {{ $app }}
 file:
-  {{ $.PackagesPath }}/{{ $addon }}_profile.tar.gz
+  {{ $.PackagesPath }}/{{ $app }}_profile.tar.gz
 {{- end }}
 
 ---
 #create deployment intent group
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups
 metadata :
-  name: {{ .AddonsDeploymentIntentGroup }}
-  description: "description"
+  name: deployment
 spec:
-  profile: {{ .AddonsProfile }}
+  profile: {{ $compositeAppName }}
   version: r1
-  logical-cloud: {{ .LogicalCloud }}
+  logical-cloud: {{ $.LogicalCloud }}
   override-values: []
 
 ---
 #create intent in deployment intent group
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/intents
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/intents
 metadata :
-  name: {{ .AddonsDeploymentIntent }}
+  name: deployment-intent
 spec:
   intent:
-    genericPlacementIntent: {{ .AddonsPlacementIntent }}
+    genericPlacementIntent: placement-intent
 
 ---
 #create the generic placement intent
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/generic-placement-intents
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/generic-placement-intents
 metadata :
-  name: {{ .AddonsPlacementIntent }}
+  name: placement-intent
 spec:
-  logical-cloud: {{ .LogicalCloud }}
+  logical-cloud: {{ $.LogicalCloud }}
 
-{{- range $index, $addon := .Addons }}
+{{- range $index, $app := $compositeApp.Apps }}
 ---
 #add the app placement intent to the generic placement intent
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonsApp }}/v1/deployment-intent-groups/{{ $.AddonsDeploymentIntentGroup }}/generic-placement-intents/{{ $.AddonsPlacementIntent }}/app-intents
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/generic-placement-intents/placement-intent/app-intents
 metadata:
-  name: {{ $addon }}-placement-intent
+  name: {{ $app }}-placement-intent
 spec:
-  app-name: {{ $addon }}
+  app-name: {{ $app }}
   intent:
     allOf:
     - provider-name: {{ $.ClusterProvider }}
@@ -98,13 +97,8 @@ spec:
 {{- end }}
 
 ---
-#Approve
+#approve
 version: emco/v2
 resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/approve
-
----
-#Instantiate
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonsApp }}/v1/deployment-intent-groups/{{ .AddonsDeploymentIntentGroup }}/instantiate
+  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $compositeAppName }}/v1/deployment-intent-groups/deployment/approve
+{{- end }}
diff --git a/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml b/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml
deleted file mode 100644
index 92fd9539..00000000
--- a/kud/deployment_infra/emco/examples/04-addon-resources-app.yaml
+++ /dev/null
@@ -1,110 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# Copyright (c) 2020 Intel Corporation
-
----
-#creating composite app entry
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps
-metadata :
-  name: {{ .AddonResourcesApp }}
-  description: "KUD addons"
-spec:
-  version: v1
-
-{{- range $index, $addon := .AddonResources }}
----
-#adding app to the composite app
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/apps
-metadata :
-  name: {{ $addon }}
-file:
-  {{ $.PackagesPath }}/{{ $addon }}.tar.gz
-{{- end }}
-
----
-#creating composite profile entry
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/composite-profiles
-metadata :
-  name: {{ .AddonResourcesProfile }}
-
-{{- range $index, $addon := .AddonResources }}
----
-#adding app profiles to the composite profile
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/composite-profiles/{{ $.AddonResourcesProfile }}/profiles
-metadata :
-  name: {{ $addon }}-profile
-spec:
-  app-name: {{ $addon }}
-file:
-  {{ $.PackagesPath }}/{{ $addon }}_profile.tar.gz
-{{- end }}
-
----
-#create deployment intent group
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups
-metadata :
-  name: {{ .AddonResourcesDeploymentIntentGroup }}
-  description: "description"
-spec:
-  profile: {{ .AddonResourcesProfile }}
-  version: r1
-  logical-cloud: {{ .LogicalCloud }}
-  override-values: []
-
----
-#create intent in deployment intent group
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/intents
-metadata :
-  name: {{ .AddonResourcesDeploymentIntent }}
-spec:
-  intent:
-    genericPlacementIntent: {{ .AddonResourcesPlacementIntent }}
-
----
-#create the generic placement intent
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/generic-placement-intents
-metadata :
-  name: {{ .AddonResourcesPlacementIntent }}
-spec:
-  logical-cloud: {{ .LogicalCloud }}
-
-{{- range $index, $addon := .AddonResources }}
----
-#add the app placement intent to the generic placement intent
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ $.ProjectName }}/composite-apps/{{ $.AddonResourcesApp }}/v1/deployment-intent-groups/{{ $.AddonResourcesDeploymentIntentGroup }}/generic-placement-intents/{{ $.AddonResourcesPlacementIntent }}/app-intents
-metadata:
-  name: {{ $addon }}-placement-intent
-spec:
-  app-name: {{ $addon }}
-  intent:
-    allOf:
-    - provider-name: {{ $.ClusterProvider }}
-      cluster-label-name: {{ $.ClustersLabel }}
-{{- end }}
-
----
-#Approve
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/approve
-
----
-#Instantiate
-version: emco/v2
-resourceContext:
-  anchor: projects/{{ .ProjectName }}/composite-apps/{{ .AddonResourcesApp }}/v1/deployment-intent-groups/{{ .AddonResourcesDeploymentIntentGroup }}/instantiate
diff --git a/kud/deployment_infra/emco/examples/README.md b/kud/deployment_infra/emco/examples/README.md
index 203b83fd..dcf9edde 100644
--- a/kud/deployment_infra/emco/examples/README.md
+++ b/kud/deployment_infra/emco/examples/README.md
@@ -15,6 +15,8 @@ needs to be installed and configured for the edge cluster.
 5. SR-IOV Network
 6. QuickAssist Technology (QAT) Device Plugin
 7. CPU Manager for Kubernetes
+8. KubeVirt and CDI Operators
+9. KubeVirt and CDI Instances
 
 ## Setup environment to deploy addons
 
@@ -38,22 +40,40 @@ required to be done only once.
     `$ emcoctl apply -f 01-cluster.yaml -v values.yaml`
     `$ emcoctl apply -f 02-project.yaml -v values.yaml`
 
-## Deploying addons
+## Create addons project
 
-This deploys the applications listed in the `Addons` and
-`AddonResources` values.
+This creates the project with the addons listed `CompositeApps` value.
 
     `$ emcoctl apply -f 03-addons-app.yaml -v values.yaml`
-    `$ emcoctl apply -f 04-addon-resources-app.yaml -v values.yaml`
+
+## Instantiate the addons
+
+This instantiates each composite app listed in the `CompositeApps`
+value.
+
+NOTE: The ordering is important when both the sriov-network and
+kubevirt addons are enabled.  The sriov-network addon will trigger a
+drain of the nodes and kubevirt will prevent the drain from
+completing, so kubevirt must be instantiated after sriov-network has
+completed the drain.
+
+	`$ emcoctl apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/instantiate`
+	`$ emcoctl apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/instantiate`
+	`$ emcoctl apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/instantiate`
 
 ## Cleanup
 
-1. Delete addons.
+1. Terminate addons.
+
+	`$ emcoctl apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/terminate`
+	`$ emcoctl apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/terminate`
+	`$ emcoctl apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/terminate`
+
+2. Delete addons.
 
-    `$ emcoctl delete -f 04-addon-resources-app.yaml -v values.yaml`
     `$ emcoctl delete -f 03-addons-app.yaml -v values.yaml`
 
-2. Cleanup prerequisites.
+3. Cleanup prerequisites.
 
     `$ emcoctl delete -f 02-project.yaml -v values.yaml`
     `$ emcoctl delete -f 01-cluster.yaml -v values.yaml`
diff --git a/kud/deployment_infra/emco/examples/values.yaml.example b/kud/deployment_infra/emco/examples/values.yaml.example
index 67944eb8..41e3cc82 100644
--- a/kud/deployment_infra/emco/examples/values.yaml.example
+++ b/kud/deployment_infra/emco/examples/values.yaml.example
@@ -7,31 +7,33 @@ DtcPort: 30483
 ClusterProvider: kud
 ClustersLabel: kud-cluster
 Clusters:
-- KubeConfig: $KUBE_PATH
-  Name: cluster
+  cluster:
+    KubeConfig: $KUBE_PATH
 
 ProjectName: kud
 LogicalCloud: kud
 
 PackagesPath: $PWD/../output/packages
-AddonsApp: addons
-AddonsProfile: addons-profile
-AddonsDeploymentIntentGroup: addons-deployment-intent-group
-AddonsDeploymentIntent: addons-deployment-intent
-AddonsPlacementIntent: addons-placement-intent
-Addons:
-- multus-cni
-- ovn4nfv
-- node-feature-discovery
-- sriov-network-operator
-- qat-device-plugin
-- cpu-manager
 
-AddonResourcesApp: addon-resources
-AddonResourcesProfile: addon-resources-profile
-AddonResourcesDeploymentIntentGroup: addon-resources-deployment-intent-group
-AddonResourcesDeploymentIntent: addon-resources-deployment-intent
-AddonResourcesPlacementIntent: addon-resources-placement-intent
-AddonResources:
-- ovn4nfv-network
-- sriov-network
+# Each composite app will be contained in its own deployment intent
+# group.  This is to enable instantiating the addons in a specified
+# order.
+CompositeApps:
+  addons:
+    Apps:
+    - kubevirt-operator
+    - cdi-operator
+    - multus-cni
+    - ovn4nfv
+    - node-feature-discovery
+    - sriov-network-operator
+    - qat-device-plugin
+    - cpu-manager
+  networks:
+    Apps:
+    - ovn4nfv-network
+    - sriov-network
+  kubevirt:
+    Apps:
+    - kubevirt
+    - cdi
diff --git a/kud/hosting_providers/containerized/addons/README.md.tmpl b/kud/hosting_providers/containerized/addons/README.md.tmpl
index 4ed4610a..eed30b43 100644
--- a/kud/hosting_providers/containerized/addons/README.md.tmpl
+++ b/kud/hosting_providers/containerized/addons/README.md.tmpl
@@ -22,28 +22,40 @@ cloud.
     \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 01-cluster.yaml -v values.yaml\`
     \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 02-project.yaml -v values.yaml\`
 
-3. Deploy addons
+3. Create addons project
 
-This deploys the addons listed in the \`Addons\` and
-\`AddonResources\` values in values.yaml.
-
-NOTE: On a single node cluster, the SRIOV addon resource will trigger
-a drain of the worker node.  KubeVirt will prevent the drain from
-completing due to its PodDisruptionBudget.  The workaround is to scale
-down the KubeVirt operator before applying 04-addon-resources-app.yaml
-then scaling it back up after the node is ready again.
+This creates the project with the addons listed `CompositeApps` value.
 
     \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 03-addons-app.yaml -v values.yaml\`
-    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 04-addon-resources-app.yaml -v values.yaml\`
+
+4. Instantiate the addons
+
+This instantiates each composite app listed in the `CompositeApps`
+value.
+
+NOTE: The ordering is important when both the sriov-network and
+kubevirt addons are enabled.  The sriov-network addon will trigger a
+drain of the nodes and kubevirt will prevent the drain from
+completing, so kubevirt must be instantiated after sriov-network has
+completed the drain.
+
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/instantiate\`
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/instantiate\`
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/instantiate\`
 
 # Uninstalling KUD addons with emcoctl
 
-1. Delete addons
+1. Terminate the addons
+
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/kubevirt/v1/deployment-intent-groups/deployment/terminate\`
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/networks/v1/deployment-intent-groups/deployment/terminate\`
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply projects/kud/composite-apps/addons/v1/deployment-intent-groups/deployment/terminate\`
+
+2. Delete addons
 
-    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 04-addon-resources-app.yaml -v values.yaml\`
     \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 03-addons-app.yaml -v values.yaml\`
 
-2. Cleanup prerequisites
+3. Cleanup prerequisites
 
     \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 02-project.yaml -v values.yaml\`
     \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f 01-cluster.yaml -v values.yaml\`
diff --git a/kud/hosting_providers/containerized/addons/values.yaml.tmpl b/kud/hosting_providers/containerized/addons/values.yaml.tmpl
index b3e5845c..25e4cc48 100644
--- a/kud/hosting_providers/containerized/addons/values.yaml.tmpl
+++ b/kud/hosting_providers/containerized/addons/values.yaml.tmpl
@@ -7,35 +7,33 @@ DtcPort: 30483
 ClusterProvider: kud
 ClustersLabel: kud-cluster
 Clusters:
-- KubeConfig: ${KUBE_PATH}
-  Name: cluster
+  cluster:
+    KubeConfig: ${KUBE_PATH}
 
 ProjectName: kud
 LogicalCloud: kud
 
 PackagesPath: ${PACKAGES_PATH}
-AddonsApp: addons
-AddonsProfile: addons-profile
-AddonsDeploymentIntentGroup: addons-deployment-intent-group
-AddonsDeploymentIntent: addons-deployment-intent
-AddonsPlacementIntent: addons-placement-intent
-Addons:
-- kubevirt-operator
-- cdi-operator
-- multus-cni
-- ovn4nfv
-- node-feature-discovery
-- sriov-network-operator
-- qat-device-plugin
-- cpu-manager
 
-AddonResourcesApp: addon-resources
-AddonResourcesProfile: addon-resources-profile
-AddonResourcesDeploymentIntentGroup: addon-resources-deployment-intent-group
-AddonResourcesDeploymentIntent: addon-resources-deployment-intent
-AddonResourcesPlacementIntent: addon-resources-placement-intent
-AddonResources:
-- ovn4nfv-network
-- sriov-network
-- kubevirt
-- cdi
+# Each composite app will be contained in its own deployment intent
+# group.  This is to enable instantiating the addons in a specified
+# order.
+CompositeApps:
+  addons:
+    Apps:
+    - kubevirt-operator
+    - cdi-operator
+    - multus-cni
+    - ovn4nfv
+    - node-feature-discovery
+    - sriov-network-operator
+    - qat-device-plugin
+    - cpu-manager
+  networks:
+    Apps:
+    - ovn4nfv-network
+    - sriov-network
+  kubevirt:
+    Apps:
+    - kubevirt
+    - cdi
diff --git a/kud/tests/plugin_fw_v2.sh b/kud/tests/plugin_fw_v2.sh
index abab9004..f19e0f90 100755
--- a/kud/tests/plugin_fw_v2.sh
+++ b/kud/tests/plugin_fw_v2.sh
@@ -223,6 +223,10 @@ else
         KUBECONFIG=$file kubectl get providernetwork emco-private-net -o name
         KUBECONFIG=$file kubectl get providernetwork unprotected-private-net -o name
     done
+    # Give some time for the Pods to show up on the clusters.  kubectl
+    # wait may return with "error: no matching resources found" if the
+    # Pods have not started yet.
+    sleep 30s
     for name in $(cluster_names); do
         print_msg "Wait for all pods to start on cluster $name"
         file=$(cluster_file "$name")