CI: Simplify running playbooks as collection + various CI Fixes

roles/kubespray_defaults/defaults/main/main.yml

@@ -94,7 +94,7 @@ nginx_config_dir: "/etc/nginx"
 haproxy_config_dir: "/etc/haproxy"
 
 # Directory where the binaries will be installed
-bin_dir: /usr/local/bin
+bin_dir: "{{ '/opt/bin/' if ('Flatcar' in ansible_distribution) else '/usr/local/bin' }}"
 docker_bin_dir: /usr/bin
 containerd_bin_dir: "{{ bin_dir }}"
 etcd_data_dir: /var/lib/etcd

tests/scripts/testcases_run.sh

@@ -29,38 +29,16 @@ export ANSIBLE_BECOME_USER=root
 if [[ "${TESTCASE}" =~ "collection" ]]; then
   # Build and install collection
   ansible-galaxy collection build
-  ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
-
-  # Simply remove all of our files and directories except for our tests directory
-  # to be absolutely certain that none of our playbooks or roles
-  # are interfering with our collection
-  find -mindepth 1 -maxdepth 1 ! -regex './\(tests\|inventory\)' -exec rm -rfv {} +
-
-cat > cluster.yml <<EOF
-- name: Install Kubernetes
-  ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster
-EOF
-
-cat > upgrade-cluster.yml <<EOF
-- name: Install Kubernetes
-  ansible.builtin.import_playbook: kubernetes_sigs.kubespray.upgrade-cluster
-EOF
-
-cat > reset.yml <<EOF
-- name: Remove Kubernetes
-  ansible.builtin.import_playbook: kubernetes_sigs.kubespray.reset
-EOF
-
-cat > remove-node.yml <<EOF
-- name: Remove node from Kubernetes
-  ansible.builtin.import_playbook: kubernetes_sigs.kubespray.remove_node
-EOF
-
+  ansible-galaxy collection install kubernetes_sigs-kubespray-*.tar.gz
 fi
-
 run_playbook () {
-playbook=$1
+if [[ "${TESTCASE}" =~ "collection" ]]; then
+    playbook=kubernetes_sigs.kubespray.$1
+else
+    playbook=$1.yml
+fi
 shift
+
 ansible-playbook \
     -e @tests/common_vars.yml \
     -e @tests/${TESTCASE_FILE} \
@@ -70,22 +48,21 @@ ansible-playbook \
 }
 
 
-
 ## START KUBESPRAY
 
 # Create cluster
-run_playbook cluster.yml
+run_playbook cluster
 
 # Repeat deployment if testing upgrade
 if [ "${UPGRADE_TEST}" != "false" ]; then
   git checkout "${CI_COMMIT_SHA}"
 
   case "${UPGRADE_TEST}" in
     "basic")
-        run_playbook cluster.yml
+        run_playbook cluster
         ;;
     "graceful")
-        run_playbook upgrade-cluster.yml
+        run_playbook upgrade-cluster
         ;;
     *)
         ;;
@@ -94,36 +71,23 @@ fi
 
 # Test control plane recovery
 if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
-    run_playbook reset.yml --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}" -e reset_confirmation=yes
-    run_playbook recover-control-plane.yml -e etcd_retries=10 --limit "etcd:kube_control_plane"
-fi
-
-# Tests Cases
-## Test Control Plane API
-run_playbook tests/testcases/010_check-apiserver.yml
-run_playbook tests/testcases/015_check-nodes-ready.yml
-
-## Test that all nodes are Ready
-
-if [[ ! ( "$TESTCASE" =~ "macvlan" ) ]]; then
-    run_playbook tests/testcases/020_check-pods-running.yml
-    run_playbook tests/testcases/030_check-network.yml
-    if [[ ! ( "$TESTCASE" =~ "hardening" ) ]]; then
-      # TODO: We need to remove this condition by finding alternative container
-      # image instead of netchecker which doesn't work at hardening environments.
-      run_playbook tests/testcases/040_check-network-adv.yml
-    fi
+    run_playbook reset --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}" -e reset_confirmation=yes
+    run_playbook recover-control-plane -e etcd_retries=10 --limit "etcd:kube_control_plane"
 fi
 
-## Kubernetes conformance tests
-run_playbook tests/testcases/100_check-k8s-conformance.yml
+# Run tests
+ansible-playbook \
+    -e @tests/common_vars.yml \
+    -e @tests/${TESTCASE_FILE} \
+    -e local_release_dir=${PWD}/downloads \
+    tests/testcases/tests.yml
 
 # Test node removal procedure
 if [ "${REMOVE_NODE_CHECK}" = "true" ]; then
-  run_playbook remove-node.yml -e skip_confirmation=yes -e node=${REMOVE_NODE_NAME}
+  run_playbook remove-node -e skip_confirmation=yes -e node=${REMOVE_NODE_NAME}
 fi
 
 # Clean up at the end, this is to allow stage1 tests to include cleanup test
 if [ "${RESET_CHECK}" = "true" ]; then
-  run_playbook reset.yml -e reset_confirmation=yes
+  run_playbook reset -e reset_confirmation=yes
 fi

tests/testcases/010_check-apiserver.yml

@@ -1,21 +1,16 @@
 ---
-- name: Testcases for apiserver
-  hosts: kube_control_plane
+- name: Check the API servers are responding
+  uri:
+    url: "https://{{ (access_ip if (ipv4_stack | default(true)) else access_ip6) | default(ansible_default_ipv4.address if (ipv4_stack | default(true)) else ansible_default_ipv6.address) | ansible.utils.ipwrap }}:{{ kube_apiserver_port | default(6443) }}/version"
+    validate_certs: false
+    status_code: 200
+  register: apiserver_response
+  retries: 12
+  delay: 5
+  until: apiserver_response is success
 
-  tasks:
-  - name: Check the API servers are responding
-    uri:
-      url: "https://{{ (access_ip if (ipv4_stack | default(true)) else access_ip6) | default(ansible_default_ipv4.address if (ipv4_stack | default(true)) else ansible_default_ipv6.address) | ansible.utils.ipwrap }}:{{ kube_apiserver_port | default(6443) }}/version"
-      validate_certs: false
-      status_code: 200
-    register: apiserver_response
-    retries: 12
-    delay: 5
-    until: apiserver_response is success
-
-  - name: Check API servers version
-    assert:
-      that:
-      - apiserver_response.json.gitVersion == kube_version
-      fail_msg: "apiserver version different than expected {{ kube_version }}"
-    when: kube_version is defined
+- name: Check API servers version
+  assert:
+    that:
+    - apiserver_response.json.gitVersion == ('v' + kube_version)
+    fail_msg: "apiserver is {{ apiserver_response.json.gitVersion }}, expected {{ kube_version }}"

tests/testcases/015_check-nodes-ready.yml

@@ -1,22 +1,18 @@
 ---
-- name: Testcases checking nodes
-  hosts: kube_control_plane[0]
-  tasks:
+- import_role:  # noqa name[missing]
+    name: cluster-dump
 
-  - import_role:  # noqa name[missing]
-      name: cluster-dump
+- name: Check kubectl output
+  command: "{{ bin_dir }}/kubectl get nodes"
+  changed_when: false
+  register: get_nodes
 
-  - name: Check kubectl output
-    command: "{{ bin_dir }}/kubectl get nodes"
-    changed_when: false
-    register: get_nodes
-
-  - name: Check that all nodes are running and ready
-    command: "{{ bin_dir }}/kubectl get nodes --no-headers -o yaml"
-    changed_when: false
-    register: get_nodes_yaml
-    until:
-    # Check that all nodes are Status=Ready
-    - '(get_nodes_yaml.stdout | from_yaml)["items"] | map(attribute = "status.conditions") | map("items2dict", key_name="type", value_name="status") | map(attribute="Ready") | list | min'
-    retries: 30
-    delay: 10
+- name: Check that all nodes are running and ready
+  command: "{{ bin_dir }}/kubectl get nodes --no-headers -o yaml"
+  changed_when: false
+  register: get_nodes_yaml
+  until:
+  # Check that all nodes are Status=Ready
+  - '(get_nodes_yaml.stdout | from_yaml)["items"] | map(attribute = "status.conditions") | map("items2dict", key_name="type", value_name="status") | map(attribute="Ready") | list | min'
+  retries: 30
+  delay: 10

tests/testcases/020_check-pods-running.yml

@@ -1,27 +1,39 @@
 ---
-- name: Testcases checking pods
-  hosts: kube_control_plane[0]
-  tasks:
+- import_role:  # noqa name[missing]
+    name: cluster-dump
 
-  - import_role:  # noqa name[missing]
-      name: cluster-dump
+- name: Check kubectl output
+  command: "{{ bin_dir }}/kubectl get pods --all-namespaces -owide"
+  changed_when: false
 
-  - name: Check kubectl output
-    command: "{{ bin_dir }}/kubectl get pods --all-namespaces -owide"
-    changed_when: false
-
-  - name: Check that all pods are running and ready
-    command: "{{ bin_dir }}/kubectl get pods --all-namespaces --no-headers -o yaml"
-    changed_when: false
+- name: Check pods
+  vars:
+    query_pods_not_running: "items[?status.phase != 'Running']"
+    query_pods_not_ready: "items[?(status.conditions[?type == 'Ready'])[0].status != 'True']"
+    pods_not_running: "{{ run_pods_log.stdout | from_json | json_query(query_pods_not_running + '.metadata') }}"
+    pods_not_ready: "{{ run_pods_log.stdout | from_json | json_query(query_pods_not_ready + '.metadata') }}"
+  block:
+  - name: Check that all pods are running
+    command: "{{ bin_dir }}/kubectl get pods --all-namespaces -o json"
     register: run_pods_log
+    changed_when: false
     until:
     # Check that all pods are running
-    - '(run_pods_log.stdout | from_yaml)["items"] | map(attribute = "status.phase") | unique | list == ["Running"]'
+    - run_pods_log.stdout | from_json | json_query(query_pods_not_running) == []
     # Check that all pods are ready
-    - '(run_pods_log.stdout | from_yaml)["items"] | map(attribute = "status.containerStatuses") | map("map", attribute = "ready") | map("min") | min'
+    - run_pods_log.stdout | from_json | json_query(query_pods_not_ready) == []
     retries: 30
     delay: 10
-
-  - name: Check kubectl output
-    command: "{{ bin_dir }}/kubectl get pods --all-namespaces -owide"
-    changed_when: false
+  rescue:
+  - name: Describe broken pods
+    command: "{{ bin_dir }}/kubectl describe pod -n {{ item.namespace }} {{ item.name }}"
+    loop: "{{ pods_not_running + pods_not_ready }}"
+    loop_control:
+      label: "{{ item.namespace }}/{{ item.name }}"
+  - name: Get logs from broken pods
+    command: "{{ bin_dir }}/kubectl logs -n {{ item.namespace }} {{ item.pod }}"
+    loop: "{{ pods_not_running + pods_not_ready }}"
+    loop_control:
+      label: "{{ item.namespace }}/{{ item.name }}"
+  - name: Fail CI
+    fail: {}