🌱 Add pre-provision command to check baremetal machines. (#1543)

Author: guettli

api/v1beta1/conditions_const.go

@@ -189,14 +189,6 @@ const (
 	CheckDiskFailedReason = "CheckDiskFailed"
 )
 
-const (
-	// SSHAfterInstallImageSucceededCondition indicates that the host is reachable via ssh after installImage.
-	SSHAfterInstallImageSucceededCondition clusterv1.ConditionType = "SSHAfterInstallImageSucceeded"
-
-	// SSHAfterInstallImageFailedReason indicates that the host was not reachable via ssh.
-	SSHAfterInstallImageFailedReason = "SSHAfterInstallImageFailed"
-)
-
 const (
 	// HostAssociateSucceededCondition indicates that a host has been associated.
 	HostAssociateSucceededCondition clusterv1.ConditionType = "HostAssociateSucceeded"

api/v1beta1/hetznerbaremetalhost_types.go

@@ -161,6 +161,9 @@ const (
 	// StateRegistering means we are getting hardware details.
 	StateRegistering ProvisioningState = "registering"
 
+	// StatePreProvisioning means we run the pre-provisioning-command (if given).
+	StatePreProvisioning ProvisioningState = "pre-provisioning"
+
 	// StateImageInstalling means we install a new image.
 	StateImageInstalling ProvisioningState = "image-installing"
 

controllers/controllers_suite_test.go

@@ -88,10 +88,11 @@ var _ = BeforeSuite(func() {
 	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
 
 	Expect((&HetznerBareMetalHostReconciler{
-		Client:             testEnv.Manager.GetClient(),
-		APIReader:          testEnv.Manager.GetAPIReader(),
-		RobotClientFactory: testEnv.RobotClientFactory,
-		SSHClientFactory:   testEnv.SSHClientFactory,
+		Client:              testEnv.Manager.GetClient(),
+		APIReader:           testEnv.Manager.GetAPIReader(),
+		RobotClientFactory:  testEnv.RobotClientFactory,
+		SSHClientFactory:    testEnv.SSHClientFactory,
+		PreProvisionCommand: "dummy-pre-provision-command",
 	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
 
 	Expect((&HetznerBareMetalMachineReconciler{

controllers/hetznerbaremetalhost_controller.go

@@ -52,11 +52,12 @@ import (
 // HetznerBareMetalHostReconciler reconciles a HetznerBareMetalHost object.
 type HetznerBareMetalHostReconciler struct {
 	client.Client
-	RateLimitWaitTime  time.Duration
-	APIReader          client.Reader
-	RobotClientFactory robotclient.Factory
-	SSHClientFactory   sshclient.Factory
-	WatchFilterValue   string
+	RateLimitWaitTime   time.Duration
+	APIReader           client.Reader
+	RobotClientFactory  robotclient.Factory
+	SSHClientFactory    sshclient.Factory
+	WatchFilterValue    string
+	PreProvisionCommand string
 }
 
 //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=hetznerbaremetalhosts,verbs=get;list;watch;create;update;patch;delete
@@ -200,6 +201,7 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 		OSSSHSecret:             osSSHSecret,
 		RescueSSHSecret:         rescueSSHSecret,
 		SecretManager:           secretManager,
+		PreProvisionCommand:     r.PreProvisionCommand,
 	})
 	if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to create scope: %w", err)

controllers/hetznerbaremetalhost_controller_test.go

@@ -348,6 +348,9 @@ var _ = Describe("HetznerBareMetalHostReconciler", func() {
 						"should-state", infrav1.StateImageInstalling)
 					return false
 				}, 10*time.Second).Should(BeTrue())
+
+				By("Checking if pre-provision-command was executed")
+				rescueSSHClient.AssertCalled(GinkgoT(), "ExecutePreProvisionCommand", mock.Anything, mock.Anything)
 			})
 		})
 
@@ -898,6 +901,7 @@ name="eth0" model="Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express
 	sshClient.On("Reboot").Return(sshclient.Output{})
 	sshClient.On("GetCloudInitOutput").Return(sshclient.Output{StdOut: "dummy content of /var/log/cloud-init-output.log"})
 	sshClient.On("DetectLinuxOnAnotherDisk", mock.Anything).Return(sshclient.Output{})
+	sshClient.On("ExecutePreProvisionCommand", mock.Anything, mock.Anything).Return(0, "", nil)
 	sshClient.On("GetInstallImageState").Return(sshclient.InstallImageStateFinished, nil)
 	sshClient.On("GetResultOfInstallImage").Return(hostpkg.PostInstallScriptFinished, nil)
 }

docs/README.md

@@ -34,3 +34,4 @@ This is the official documentation of Cluster API Provider Hetzner. Before start
 - [Tilt](/docs/caph/04-developers/02-tilt.md)
 - [Releasing](/docs/caph/04-developers/03-releasing.md)
 - [Updating Kubernetes version](/docs/caph/04-developers/04-updating-kubernetes-version.md)
+- [pre-provision-command](/docs/caph/04-developers/05-pre-provision-command.md)

docs/caph/04-developers/05-pre-provision-command.md

@@ -0,0 +1,75 @@
+---
+title: pre-provision-command
+metatitle: Cluster API Provider Hetzner Check Bare Metal Server before Provisioning
+sidebar: pre-provision-command
+description: Documentation on the CAPH pre-provision-command.
+---
+
+The `--pre-provision-command` for the caph controller can be used to execute a custom command
+before install-image starts.
+
+This provides you a flexible way to check if the bare metal server is healthy.
+
+Example:
+
+```sh
+--pre-provision-command=/shared/my-pre-provision-command.sh
+```
+
+The script/binary will be copied into the Hetzner Rescue System and executed.
+
+If the exit code is zero, then all is fine.
+
+If the exit code is non-zero, then provisioning of that machine will be stopped.
+
+The CAPH controller runs in a Kubernetes Pod. The container of that pod needs access to the file.
+
+There are several ways to make this command available:
+
+* You could mount a configMap/secret.
+* You create a container image, and use that as init-container.
+* You build a custom image of CAPH. We do not recommend that.
+
+In this example we use an init-container to provide the script.
+
+In the directory `images/pre-provision-command/` you see these files:
+
+* my-pre-provision-command.sh: A simple Bash script which creates a message and exists with 0.
+* Dockerfile: Needed to create a container image.
+* build-and-push.sh: A script to build and upload the script to a container registry.
+
+When the container image was uploaded, you need to adapt the CAPH deployment:
+
+```yaml
+
+      # Init container, which makes the command available to caph.
+      initContainers:
+      - command:
+        - /bin/sh
+        - -c
+        - cp /my-pre-provision-command.sh /shared/
+        image: ghcr.io/syself/caph-staging:pre-provision-command
+        imagePullPolicy: Always
+        name: init-container
+        resources: {}
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /shared
+          name: shared
+
+
+      # Add this to the args of the caph container:
+        args:
+        - --pre-provision-command=/shared/my-pre-provision-command.sh
+
+        # Add this to the caph container
+        volumeMounts:
+        - mountPath: /shared
+          name: shared
+
+      # Add this after "container"
+      volumes:
+      - emptyDir: {}
+        name: shared
+```

go.mod

@@ -4,6 +4,7 @@ go 1.22.6
 
 require (
 	github.com/blang/semver/v4 v4.0.0
+	github.com/bramvdbogaerde/go-scp v1.5.0
 	github.com/go-logr/logr v1.4.2
 	github.com/go-logr/zapr v1.3.0
 	github.com/hetznercloud/hcloud-go/v2 v2.13.1

go.sum

@@ -26,6 +26,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
+github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=

hack/update-operator-dev-deployment.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This scripts updates the caph deployment in a mgt-cluster. Be sure to be connected to
+# a development cluster.
+# It takes the current code, creates a new image and updates the deployment.
+# By default the image will be uploaded to:
+# ghcr.io/syself/caph-staging:dev-$USER-YOUR_GIT_BRANCH
+# You can change the image path with the --image-path flag, if you want to use a different registry.
+
+# Usually it is better to write a test, than to use this script.
+
+trap 'echo "Warning: A command has failed. Exiting the script. Line was ($0:$LINENO): $(sed -n "${LINENO}p" "$0")"; exit 3' ERR
+set -Eeuo pipefail
+
+image_path="ghcr.io/syself"
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+    --image-path)
+        image_path="$2"
+        shift
+        ;;
+    *)
+        echo "Unknown parameter passed: $1"
+        exit 1
+        ;;
+    esac
+    shift
+done
+
+# remove trailing slash
+image_path="${image_path%/}"
+
+if ! kubectl cluster-info >/dev/null; then
+    echo
+    echo "No kubernetes cluster found."
+    echo "You can use alm to create a mgt-cluster"
+    echo "docs: https://github.com/syself/autopilot-lifecycle-manager"
+    exit 1
+fi
+
+branch=$(git branch --show-current)
+if [ "$branch" == "" ]; then
+    echo "failed to get branch name"
+    exit 1
+fi
+
+tag="dev-$USER-$branch"
+tag="$(echo -n "$tag" | tr -c 'a-zA-Z0-9_.-' '-')"
+
+image="$image_path/caph-staging:$tag"
+
+echo "Building image: $image"
+
+docker build -f images/caph/Dockerfile -t "$image" .
+
+docker push "$image"
+
+# Note: Up to now changes in the CRD are not supported by this script.
+
+kubectl scale --replicas=1 -n mgt-system deployment/caph-controller-manager
+
+kubectl set image -n mgt-system deployment/caph-controller-manager manager="$image"
+
+kubectl patch deployment -n mgt-system -p '[{"op": "replace", "path": "/spec/template/spec/containers/0/imagePullPolicy", "value": "Always"}]' --type='json' caph-controller-manager
+
+kubectl rollout restart -n mgt-system deployment caph-controller-manager
+
+trap "echo 'Interrupted! Exiting...'; exit 1" SIGINT
+
+while ! kubectl rollout status deployment --timeout=3s -n mgt-system caph-controller-manager; do
+    echo "Rollout failed"
+    kubectl events -n mgt-system | grep caph-controller-manager | tail -n 5
+    echo
+    echo
+done

images/pre-provision-command/Dockerfile

@@ -0,0 +1,17 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM bash
+COPY ./my-pre-provision-command.sh /my-pre-provision-command.sh
+RUN chmod +x /my-pre-provision-command.sh

images/pre-provision-command/build-and-push.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script builds and pushed the container image for the init-container.
+# See README.md for more information.
+
+# Bash Strict Mode: https://github.com/guettli/bash-strict-mode
+trap 'echo "Warning: A command has failed. Exiting the script. Line was ($0:$LINENO): $(sed -n "${LINENO}p" "$0")"; exit 3' ERR
+set -Eeuo pipefail
+
+DIR="$(dirname "$0")"
+
+docker build -t ghcr.io/syself/caph-staging:pre-provision-command "$DIR"
+
+docker push ghcr.io/syself/caph-staging:pre-provision-command

images/pre-provision-command/my-pre-provision-command.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "dummy script for caph --pre-provision-command. All is fine."
+exit 0