cust-scorecard-tests - reduce parallelism

Signed-off-by: Tesshu Flower <[email protected]>

Author: tesshuflower

custom-scorecard-tests/config.yaml

@@ -35,16 +35,6 @@ stages:
     storage:
       spec:
         mountPath: {}
-  - entrypoint:
-    - volsync-custom-scorecard-tests
-    - test_namespace_role.yml
-    image: quay.io/backube/volsync-custom-scorecard-tests:latest
-    labels:
-      suite: volsync-e2e
-      test: test_namespace_role.yml
-    storage:
-      spec:
-        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_rclone_normal.yml
@@ -187,111 +177,123 @@ stages:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_roles.yml
+    - test_rsync_tls_normal.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_roles.yml
+      test: test_rsync_tls_normal.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_roles_privileged.yml
+    - test_rsync_tls_normal_diskrsync.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_roles_privileged.yml
+      test: test_rsync_tls_normal_diskrsync.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_rsync_tls_normal.yml
+    - test_rsync_tls_priv.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_rsync_tls_normal.yml
+      test: test_rsync_tls_priv.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_rsync_tls_normal_diskrsync.yml
+    - test_rsync_tls_priv_diskrsync.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_rsync_tls_normal_diskrsync.yml
+      test: test_rsync_tls_priv_diskrsync.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_rsync_tls_priv.yml
+    - test_simple_rsync.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_rsync_tls_priv.yml
+      test: test_simple_rsync.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_rsync_tls_priv_diskrsync.yml
+    - test_simple_rsync_diskrsync.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_rsync_tls_priv_diskrsync.yml
+      test: test_simple_rsync_diskrsync.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_simple_rsync.yml
+    - test_volumepopulator.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_simple_rsync.yml
+      test: test_volumepopulator.yml
     storage:
       spec:
         mountPath: {}
+- parallel: true
+  tests:
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_simple_rsync_diskrsync.yml
+    - test_namespace_role.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_simple_rsync_diskrsync.yml
+      test: test_namespace_role.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_syncthing_cluster_sync_normal.yml
+    - test_roles.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_syncthing_cluster_sync_normal.yml
+      test: test_roles.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_syncthing_cluster_sync_priv.yml
+    - test_roles_privileged.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_syncthing_cluster_sync_priv.yml
+      test: test_roles_privileged.yml
     storage:
       spec:
         mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
-    - test_volumepopulator.yml
+    - test_syncthing_cluster_sync_normal.yml
     image: quay.io/backube/volsync-custom-scorecard-tests:latest
     labels:
       suite: volsync-e2e
-      test: test_volumepopulator.yml
+      test: test_syncthing_cluster_sync_normal.yml
+    storage:
+      spec:
+        mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_syncthing_cluster_sync_priv.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_syncthing_cluster_sync_priv.yml
     storage:
       spec:
         mountPath: {}

custom-scorecard-tests/generateE2ETestsConfig.sh

@@ -14,16 +14,28 @@ fi
 
 TESTS=$(echo "${TESTS_UNSORTED}" | LC_ALL=C sort)
 
+# Group tests into 2 stages (each stage gets run sequentially but
+# all tests in a stage can run in parallel)
+E2E_TESTS_GROUP1=$(echo "${TESTS}" | grep -v -e role -e syncthing)
+E2E_TESTS_GROUP2=$(echo "${TESTS}" | grep -e role -e syncthing)
+
+echo "####################"
 echo "# E2E test list is: "
-echo "${TESTS}"
+echo "## Stage 1: "
+echo "${E2E_TESTS_GROUP1}"
+echo "## Stage 2: "
+echo "${E2E_TESTS_GROUP2}"
+echo "####################"
 
-prereqs_patchfile="scorecard/patches/deploy-prereqs.yaml"
-e2e_tests_patchfile="scorecard/patches/e2e-tests.yaml"
+prereqs_patchfile="scorecard/patches/deploy-prereqs-stage0.yaml"
+e2e_tests_patchfile1="scorecard/patches/e2e-tests-stage1.yaml"
+e2e_tests_patchfile2="scorecard/patches/e2e-tests-stage2.yaml"
 
 rm -rf "${prereqs_patchfile}"
-rm -rf "${e2e_tests_patchfile}"
+rm -rf "${e2e_tests_patchfile1}"
+rm -rf "${e2e_tests_patchfile2}"
 
-# Prereqs
+# Prereqs (stage 0)
 cat <<EOF > "${prereqs_patchfile}"
 - op: add
   path: /stages/0/tests/-
@@ -37,15 +49,37 @@ cat <<EOF > "${prereqs_patchfile}"
       test: deploy-prereqs
 EOF
 
-# E2E tests
-cat <<EOF > "${e2e_tests_patchfile}"
+# E2E tests - Group of e2e to run in parallel in stage 1
+cat <<EOF > "${e2e_tests_patchfile1}"
 - op: add
   path: /stages/1/tests
   value:
 EOF
 
-for file in ${TESTS}; do
-  cat <<EOF >> "${e2e_tests_patchfile}"
+for file in ${E2E_TESTS_GROUP1}; do
+  cat <<EOF >> "${e2e_tests_patchfile1}"
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - ${file}
+    image: ${CUSTOM_SCORECARD_IMG}
+    labels:
+      suite: volsync-e2e
+      test: ${file}
+    storage:
+      spec:
+        mountPath: {}
+EOF
+done
+
+# E2E tests - Group of e2e to run in parallel in stage 2
+cat <<EOF > "${e2e_tests_patchfile2}"
+- op: add
+  path: /stages/2/tests
+  value:
+EOF
+
+for file in ${E2E_TESTS_GROUP2}; do
+  cat <<EOF >> "${e2e_tests_patchfile2}"
   - entrypoint:
     - volsync-custom-scorecard-tests
     - ${file}

custom-scorecard-tests/scorecard/bases/config.yaml

@@ -8,6 +8,8 @@ stages:
   tests: []
 - parallel: true
   tests: []
+- parallel: true
+  tests: []
 storage:
   spec:
     mountPath: {}

custom-scorecard-tests/scorecard/kustomization.yaml

@@ -3,13 +3,19 @@ resources:
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 patches:
-- path: patches/deploy-prereqs.yaml
+- path: patches/deploy-prereqs-stage0.yaml
   target:
     group: scorecard.operatorframework.io
     kind: Configuration
     name: config
     version: v1alpha3
-- path: patches/e2e-tests.yaml
+- path: patches/e2e-tests-stage1.yaml
+  target:
+    group: scorecard.operatorframework.io
+    kind: Configuration
+    name: config
+    version: v1alpha3
+- path: patches/e2e-tests-stage2.yaml
   target:
     group: scorecard.operatorframework.io
     kind: Configuration

custom-scorecard-tests/scorecard/patches/deploy-prereqs.yaml renamed to custom-scorecard-tests/scorecard/patches/deploy-prereqs-stage0.yaml

@@ -21,16 +21,6 @@
     storage:
       spec:
         mountPath: {}
-  - entrypoint:
-    - volsync-custom-scorecard-tests
-    - test_namespace_role.yml
-    image: quay.io/backube/volsync-custom-scorecard-tests:latest
-    labels:
-      suite: volsync-e2e
-      test: test_namespace_role.yml
-    storage:
-      spec:
-        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_rclone_normal.yml
@@ -171,26 +161,6 @@
     storage:
       spec:
         mountPath: {}
-  - entrypoint:
-    - volsync-custom-scorecard-tests
-    - test_roles.yml
-    image: quay.io/backube/volsync-custom-scorecard-tests:latest
-    labels:
-      suite: volsync-e2e
-      test: test_roles.yml
-    storage:
-      spec:
-        mountPath: {}
-  - entrypoint:
-    - volsync-custom-scorecard-tests
-    - test_roles_privileged.yml
-    image: quay.io/backube/volsync-custom-scorecard-tests:latest
-    labels:
-      suite: volsync-e2e
-      test: test_roles_privileged.yml
-    storage:
-      spec:
-        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_rsync_tls_normal.yml
@@ -251,26 +221,6 @@
     storage:
       spec:
         mountPath: {}
-  - entrypoint:
-    - volsync-custom-scorecard-tests
-    - test_syncthing_cluster_sync_normal.yml
-    image: quay.io/backube/volsync-custom-scorecard-tests:latest
-    labels:
-      suite: volsync-e2e
-      test: test_syncthing_cluster_sync_normal.yml
-    storage:
-      spec:
-        mountPath: {}
-  - entrypoint:
-    - volsync-custom-scorecard-tests
-    - test_syncthing_cluster_sync_priv.yml
-    image: quay.io/backube/volsync-custom-scorecard-tests:latest
-    labels:
-      suite: volsync-e2e
-      test: test_syncthing_cluster_sync_priv.yml
-    storage:
-      spec:
-        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_volumepopulator.yml

custom-scorecard-tests/scorecard/patches/e2e-tests.yaml renamed to custom-scorecard-tests/scorecard/patches/e2e-tests-stage1.yaml

@@ -0,0 +1,53 @@
+- op: add
+  path: /stages/2/tests
+  value:
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_namespace_role.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_namespace_role.yml
+    storage:
+      spec:
+        mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_roles.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_roles.yml
+    storage:
+      spec:
+        mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_roles_privileged.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_roles_privileged.yml
+    storage:
+      spec:
+        mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_syncthing_cluster_sync_normal.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_syncthing_cluster_sync_normal.yml
+    storage:
+      spec:
+        mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_syncthing_cluster_sync_priv.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_syncthing_cluster_sync_priv.yml
+    storage:
+      spec:
+        mountPath: {}