Skip to content

Commit 4d91440

Browse files
chrischdichrischdi
authored
🌱 test: shutdown clustercache when ctx is done (#3344)
* test: shutdown clustercache when ctx is done * use separate testenv for tests using clustercache * fixup naming * Properly turn off testenvs * fixup
1 parent 411ccc3 commit 4d91440

8 files changed

+93
-45
lines changed

‎controllers/controllers_suite_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ func setup() {
9898
if err != nil {
9999
panic(fmt.Sprintf("Unable to setup ClusterCache: %v", err))
100100
}
101+
go func() {
102+
<-ctx.Done()
103+
clusterCache.(interface{ Shutdown() }).Shutdown()
104+
}()
101105

102106
controllerOpts := controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: ptr.To(true)}
103107

‎controllers/vmware/controllers_suite_test.go

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package vmware
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"os"
2223
"path/filepath"
@@ -60,18 +61,18 @@ var (
6061
)
6162

6263
func TestMain(m *testing.M) {
63-
setup()
64+
testEnv, clusterCache = setup(ctx)
6465
code := m.Run()
6566
teardown()
6667
os.Exit(code)
6768
}
6869

69-
func setup() {
70+
func setup(ctx context.Context) (*helpers.TestEnvironment, clustercache.ClusterCache) {
7071
utilruntime.Must(infrav1.AddToScheme(scheme.Scheme))
7172
utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme))
7273
utilruntime.Must(vmwarev1.AddToScheme(scheme.Scheme))
7374

74-
testEnv = helpers.NewTestEnvironment(ctx)
75+
testEnv := helpers.NewTestEnvironment(ctx)
7576

7677
secretCachingClient, err := client.New(testEnv.Manager.GetConfig(), client.Options{
7778
HTTPClient: testEnv.Manager.GetHTTPClient(),
@@ -99,6 +100,10 @@ func setup() {
99100
if err != nil {
100101
panic(fmt.Sprintf("Unable to setup ClusterCache: %v", err))
101102
}
103+
go func() {
104+
<-ctx.Done()
105+
clusterCache.(interface{ Shutdown() }).Shutdown()
106+
}()
102107

103108
controllerOpts := controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: ptr.To(true)}
104109

@@ -127,8 +132,10 @@ func setup() {
127132
},
128133
}
129134
if err := testEnv.Create(ctx, ns); err != nil {
130-
panic("unable to create controller namespace")
135+
panic(fmt.Sprintf("unable to create controller namespace: %v", err))
131136
}
137+
138+
return testEnv, clusterCache
132139
}
133140

134141
func teardown() {

‎controllers/vmware/serviceaccount_controller_intg_test.go

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package vmware
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"reflect"
2223
"time"
@@ -30,23 +31,40 @@ import (
3031
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3132
"k8s.io/apimachinery/pkg/types"
3233
"k8s.io/utils/ptr"
34+
"sigs.k8s.io/cluster-api/controllers/clustercache"
3335
"sigs.k8s.io/cluster-api/util/conditions"
3436
"sigs.k8s.io/cluster-api/util/patch"
3537
"sigs.k8s.io/controller-runtime/pkg/client"
3638

3739
vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
38-
helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
40+
"sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers"
41+
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
3942
)
4043

4144
var _ = Describe("ProviderServiceAccount controller integration tests", func() {
42-
var intCtx *helpers.IntegrationTestContext
45+
// This test suite requires its own management cluster and controllers including clustercache.
46+
// Otherwise the workload cluster's kube-apiserver would not shutdown due to the global
47+
// test's clustercache still being running.
48+
var (
49+
intCtx *vmwarehelpers.IntegrationTestContext
50+
testEnv *helpers.TestEnvironment
51+
clusterCache clustercache.ClusterCache
52+
clusterCacheCancel context.CancelFunc
53+
)
4354

4455
BeforeEach(func() {
45-
intCtx = helpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())
56+
var clusterCacheCtx context.Context
57+
clusterCacheCtx, clusterCacheCancel = context.WithCancel(ctx)
58+
testEnv, clusterCache = setup(clusterCacheCtx)
59+
intCtx = vmwarehelpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())
4660
})
4761

4862
AfterEach(func() {
63+
// Stop clustercache
64+
clusterCacheCancel()
65+
4966
intCtx.AfterEach()
67+
Expect(testEnv.Stop()).To(Succeed())
5068
})
5169

5270
Describe("When the ProviderServiceAccount is created", func() {
@@ -56,10 +74,10 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
5674
)
5775
BeforeEach(func() {
5876
By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() {
59-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
60-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
61-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
62-
helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
77+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
78+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
79+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
80+
vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
6381
})
6482

6583
By("Verifying that the guest cluster client works")
@@ -142,8 +160,8 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
142160
Context("With non-existent Cluster object", func() {
143161
It("cannot reconcile the ProviderServiceAccount object", func() {
144162
By("Creating the vSphereCluster and KubeconfigSecret only", func() {
145-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
146-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
163+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
164+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
147165
})
148166

149167
By("Creating the ProviderServiceAccount", func() {
@@ -164,8 +182,8 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
164182
Context("With non-existent Cluster credentials secret", func() {
165183
It("cannot reconcile the ProviderServiceAccount object", func() {
166184
By("Creating the Cluster and vSphereCluster only", func() {
167-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
168-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
185+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
186+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
169187
})
170188

171189
By("Creating the ProviderServiceAccount", func() {
@@ -189,10 +207,10 @@ var _ = Describe("ProviderServiceAccount controller integration tests", func() {
189207
var roleBinding *rbacv1.RoleBinding
190208
BeforeEach(func() {
191209
By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() {
192-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
193-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
194-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
195-
helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
210+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
211+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
212+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
213+
vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
196214
})
197215
pSvcAccount = getTestProviderServiceAccount(intCtx.Namespace, intCtx.VSphereCluster)
198216
pSvcAccount.Spec.TargetNamespace = "default"

‎controllers/vmware/serviceaccount_controller_unit_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,23 +27,23 @@ import (
2727
"sigs.k8s.io/controller-runtime/pkg/client"
2828

2929
vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
30-
helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
30+
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
3131
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake"
3232
)
3333

3434
var _ = Describe("ServiceAccountReconciler reconcileNormal", unitTestsReconcileNormal)
3535

3636
func unitTestsReconcileNormal() {
3737
var (
38-
controllerCtx *helpers.UnitTestContextForController
38+
controllerCtx *vmwarehelpers.UnitTestContextForController
3939
vsphereCluster *vmwarev1.VSphereCluster
4040
initObjects []client.Object
4141
namespace string
4242
reconciler ServiceAccountReconciler
4343
)
4444

4545
JustBeforeEach(func() {
46-
controllerCtx = helpers.NewUnitTestContextForController(ctx, namespace, vsphereCluster, false, initObjects, nil)
46+
controllerCtx = vmwarehelpers.NewUnitTestContextForController(ctx, namespace, vsphereCluster, false, initObjects, nil)
4747
// Note: The service account provider requires a reference to the vSphereCluster hence the need to create
4848
// a fake vSphereCluster in the test and pass it to during context setup.
4949
reconciler = ServiceAccountReconciler{
@@ -128,7 +128,7 @@ func unitTestsReconcileNormal() {
128128

129129
// Updates the service account secret similar to how a token controller would act upon a service account
130130
// and then re-invokes reconcileNormal.
131-
func updateServiceAccountSecretAndReconcileNormal(ctx context.Context, controllerCtx *helpers.UnitTestContextForController, reconciler ServiceAccountReconciler, object client.Object) {
131+
func updateServiceAccountSecretAndReconcileNormal(ctx context.Context, controllerCtx *vmwarehelpers.UnitTestContextForController, reconciler ServiceAccountReconciler, object client.Object) {
132132
assertServiceAccountAndUpdateSecret(ctx, controllerCtx.ControllerManagerContext.Client, object.GetNamespace(), object.GetName())
133133
_, err := reconciler.reconcileNormal(ctx, controllerCtx.GuestClusterContext)
134134
Expect(err).NotTo(HaveOccurred())

‎controllers/vmware/servicediscovery_controller_intg_test.go

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,30 +17,43 @@ limitations under the License.
1717
package vmware
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"time"
2223

2324
. "github.com/onsi/ginkgo/v2"
2425
. "github.com/onsi/gomega"
2526
corev1 "k8s.io/api/core/v1"
2627
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"sigs.k8s.io/cluster-api/controllers/clustercache"
2729
"sigs.k8s.io/controller-runtime/pkg/client"
2830

29-
helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
31+
"sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers"
32+
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
3033
)
3134

3235
var _ = Describe("Service Discovery controller integration tests", func() {
36+
// This test suite requires its own management cluster and controllers including clustercache.
37+
// Otherwise the workload cluster's kube-apiserver would not shutdown due to the global
38+
// test's clustercache still being running.
3339
var (
34-
intCtx *helpers.IntegrationTestContext
35-
initObjects []client.Object
40+
intCtx *vmwarehelpers.IntegrationTestContext
41+
testEnv *helpers.TestEnvironment
42+
clusterCache clustercache.ClusterCache
43+
clusterCacheCancel context.CancelFunc
44+
initObjects []client.Object
3645
)
3746
BeforeEach(func() {
38-
intCtx = helpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())
47+
var clusterCacheCtx context.Context
48+
clusterCacheCtx, clusterCacheCancel = context.WithCancel(ctx)
49+
testEnv, clusterCache = setup(clusterCacheCtx)
50+
intCtx = vmwarehelpers.NewIntegrationTestContextWithClusters(ctx, testEnv.Manager.GetClient())
51+
3952
By(fmt.Sprintf("Creating the Cluster (%s), vSphereCluster (%s) and KubeconfigSecret", intCtx.Cluster.Name, intCtx.VSphereCluster.Name), func() {
40-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
41-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
42-
helpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
43-
helpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
53+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.Cluster)
54+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.VSphereCluster)
55+
vmwarehelpers.CreateAndWait(ctx, intCtx.Client, intCtx.KubeconfigSecret)
56+
vmwarehelpers.ClusterInfrastructureReady(ctx, intCtx.Client, clusterCache, intCtx.Cluster)
4457
})
4558

4659
By("Verifying that the guest cluster client works")
@@ -54,10 +67,14 @@ var _ = Describe("Service Discovery controller integration tests", func() {
5467
Expect(guestClient.List(ctx, &corev1.ServiceList{}, client.InNamespace(metav1.NamespaceDefault))).To(Succeed())
5568
})
5669
AfterEach(func() {
70+
// Stop clustercache
71+
clusterCacheCancel()
72+
5773
deleteTestResource(ctx, intCtx.Client, intCtx.VSphereCluster)
5874
deleteTestResource(ctx, intCtx.Client, intCtx.Cluster)
5975
deleteTestResource(ctx, intCtx.Client, intCtx.KubeconfigSecret)
6076
intCtx.AfterEach()
77+
Expect(testEnv.Stop()).To(Succeed())
6178
})
6279

6380
Context("When VIP is available", func() {

‎controllers/vmware/servicediscovery_controller_unit_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,23 +26,23 @@ import (
2626
"sigs.k8s.io/controller-runtime/pkg/client"
2727

2828
vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
29-
helpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
29+
vmwarehelpers "sigs.k8s.io/cluster-api-provider-vsphere/internal/test/helpers/vmware"
3030
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake"
3131
)
3232

3333
var _ = Describe("ServiceDiscoveryReconciler reconcileNormal", serviceDiscoveryUnitTestsReconcileNormal)
3434

3535
func serviceDiscoveryUnitTestsReconcileNormal() {
3636
var (
37-
controllerCtx *helpers.UnitTestContextForController
37+
controllerCtx *vmwarehelpers.UnitTestContextForController
3838
vsphereCluster vmwarev1.VSphereCluster
3939
initObjects []client.Object
4040
reconciler serviceDiscoveryReconciler
4141
)
4242
namespace := capiutil.RandomString(6)
4343
JustBeforeEach(func() {
4444
vsphereCluster = fake.NewVSphereCluster(namespace)
45-
controllerCtx = helpers.NewUnitTestContextForController(ctx, namespace, &vsphereCluster, false, initObjects, nil)
45+
controllerCtx = vmwarehelpers.NewUnitTestContextForController(ctx, namespace, &vsphereCluster, false, initObjects, nil)
4646
reconciler = serviceDiscoveryReconciler{
4747
Client: controllerCtx.ControllerManagerContext.Client,
4848
}

‎internal/test/helpers/envtest.go

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,8 @@ func init() {
7878
}
7979

8080
var (
81-
scheme = runtime.NewScheme()
82-
env *envtest.Environment
81+
scheme = runtime.NewScheme()
82+
crdPaths []string
8383
)
8484

8585
func init() {
@@ -97,7 +97,7 @@ func init() {
9797
}
9898
root := path.Join(path.Dir(filename), "..", "..", "..")
9999

100-
crdPaths := []string{
100+
crdPaths = []string{
101101
filepath.Join(root, "config", "default", "crd", "bases"),
102102
filepath.Join(root, "config", "supervisor", "crd", "bases"),
103103
}
@@ -106,12 +106,6 @@ func init() {
106106
if capiPaths := getFilePathToCAPICRDs(); capiPaths != nil {
107107
crdPaths = append(crdPaths, capiPaths...)
108108
}
109-
110-
// Create the test environment.
111-
env = &envtest.Environment{
112-
ErrorIfCRDPathMissing: true,
113-
CRDDirectoryPaths: crdPaths,
114-
}
115109
}
116110

117111
var (
@@ -130,13 +124,20 @@ type (
130124
client.Client
131125
Config *rest.Config
132126
Simulator *vcsim.Simulator
127+
env *envtest.Environment
133128

134129
cancel context.CancelFunc
135130
}
136131
)
137132

138133
// NewTestEnvironment creates a new environment spinning up a local api-server.
139134
func NewTestEnvironment(ctx context.Context) *TestEnvironment {
135+
// Create the test environment.
136+
env := &envtest.Environment{
137+
ErrorIfCRDPathMissing: true,
138+
CRDDirectoryPaths: crdPaths,
139+
}
140+
140141
// Get the root of the current file to use in CRD paths.
141142
_, filename, _, ok := goruntime.Caller(0)
142143
if !ok {
@@ -227,6 +228,7 @@ func NewTestEnvironment(ctx context.Context) *TestEnvironment {
227228
Client: mgr.GetClient(),
228229
Config: mgr.GetConfig(),
229230
Simulator: simr,
231+
env: env,
230232
}
231233
}
232234

@@ -241,7 +243,7 @@ func (t *TestEnvironment) StartManager(ctx context.Context) error {
241243
func (t *TestEnvironment) Stop() error {
242244
t.cancel()
243245
t.Simulator.Destroy()
244-
return env.Stop()
246+
return t.env.Stop()
245247
}
246248

247249
// Cleanup removes objects from the TestEnvironment.

‎internal/test/helpers/webhook.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ func InitializeWebhookInEnvironment(e *envtest.Environment, configPath string) {
101101

102102
// WaitForWebhooks waits for TestEnvironment's webhooks to accept connections.
103103
func (t *TestEnvironment) WaitForWebhooks() {
104-
port := env.WebhookInstallOptions.LocalServingPort
104+
port := t.env.WebhookInstallOptions.LocalServingPort
105105

106106
klog.Infof("Waiting for webhook port %d to be open prior to running tests", port)
107107
timeout := 1 * time.Second

0 commit comments

Comments
 (0)