Dump thread periodically

configured by flag `--experimental_thread_dump_interval`.

The thread dumps can be found under `<output_base>/server/thread_dumps/`.

Dumps from last invocation are deleted at the start of the current invocation to avoid accumulating too much dumps in the output base.

RELNOTES: Added flag `--experimental_thread_dump_interval` to allow Bazel dump threads periodically.
PiperOrigin-RevId: 792131384
Change-Id: Ib9fe7a659f51bdd404a28229055441a6d3a87f3d

Author: coeuvre

src/main/java/com/google/devtools/build/lib/BUILD

@@ -526,6 +526,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/util:shallow_object_size_computer",
         "//src/main/java/com/google/devtools/build/lib/util:string",
         "//src/main/java/com/google/devtools/build/lib/util:string_encoding",
+        "//src/main/java/com/google/devtools/build/lib/util:thread_dumper",
         "//src/main/java/com/google/devtools/build/lib/util/io",
         "//src/main/java/com/google/devtools/build/lib/util/io:io-proto",
         "//src/main/java/com/google/devtools/build/lib/util/io:out-err",

src/main/java/com/google/devtools/build/lib/bazel/Bazel.java

@@ -49,6 +49,7 @@ public final class Bazel {
           CredentialModule.class,
           com.google.devtools.build.lib.runtime.CommandLogModule.class,
           com.google.devtools.build.lib.runtime.MemoryPressureModule.class,
+          com.google.devtools.build.lib.runtime.ThreadDumpModule.class,
           com.google.devtools.build.lib.platform.SleepPreventionModule.class,
           com.google.devtools.build.lib.platform.SystemSuspensionModule.class,
           BazelFileSystemModule.class,

src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java

@@ -533,7 +533,7 @@ public ActionKeyContext getActionKeyContext() {
   }
 
   /** The directory in which blaze stores the server state - that is, the socket file and a log. */
-  private Path getServerDirectory() {
+  public Path getServerDirectory() {
     return workspace.getDirectories().getOutputBase().getChild("server");
   }
 

src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java

@@ -155,6 +155,18 @@ public class CommonCommandOptions extends OptionsBase {
               + " --experimental_action_cache_gc_threshold flags.")
   public Duration actionCacheGcMaxAge;
 
+  @Option(
+      name = "experimental_thread_dump_interval",
+      defaultValue = "0",
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.BAZEL_MONITORING},
+      converter = DurationConverter.class,
+      help =
+          "How often to dump the state of all threads (including virtual threads) to a file. The"
+              + " dumps will be written to the <output_base>/server/thread_dumps/ directory. If"
+              + " zero, no thread dumps are written.")
+  public Duration threadDumpInterval;
+
   /** Converter for UUID. Accepts values as specified by {@link UUID#fromString(String)}. */
   public static class UUIDConverter extends Converter.Contextless<UUID> {
 

src/main/java/com/google/devtools/build/lib/runtime/ThreadDumpModule.java

@@ -0,0 +1,131 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.flogger.GoogleLogger;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.clock.Clock;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.DetailedExitCode;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.JavaSleeper;
+import com.google.devtools.build.lib.util.Sleeper;
+import com.google.devtools.build.lib.util.ThreadDumper;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import javax.annotation.Nullable;
+
+/** A {@link BlazeModule} that dumps the state of all threads periodically. */
+public final class ThreadDumpModule extends BlazeModule {
+  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+  private static final DateTimeFormatter TIME_FORMAT =
+      DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
+
+  @Nullable private Thread dumpThread;
+
+  @Override
+  public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
+    var commandOptions = env.getOptions().getOptions(CommonCommandOptions.class);
+    if (commandOptions == null || commandOptions.threadDumpInterval.isZero()) {
+      return;
+    }
+
+    var runtime = env.getRuntime();
+    var clock = runtime.getClock();
+    var threadDumpInterval = commandOptions.threadDumpInterval;
+
+    var serverDirectory = runtime.getServerDirectory();
+    var dumpDirectory = serverDirectory.getChild("thread_dumps");
+    try {
+      dumpDirectory.deleteTree();
+      dumpDirectory.createDirectoryAndParents();
+    } catch (IOException e) {
+      throw new AbruptExitException(
+          DetailedExitCode.of(
+              ExitCode.LOCAL_ENVIRONMENTAL_ERROR,
+              FailureDetail.newBuilder()
+                  .setMessage("Failed to setup thread dump directory")
+                  .build()),
+          e);
+    }
+
+    var pid = ProcessHandle.current().pid();
+    checkState(dumpThread == null);
+    dumpThread =
+        new Thread(
+            new ThreadDumpTask(pid, clock, new JavaSleeper(), threadDumpInterval, dumpDirectory),
+            "thread-dumper");
+    dumpThread.start();
+  }
+
+  @Override
+  public void afterCommand() {
+    if (dumpThread != null) {
+      dumpThread.interrupt();
+      try (var sc = Profiler.instance().profile("Joining dump thread")) {
+        Uninterruptibles.joinUninterruptibly(dumpThread);
+      }
+      dumpThread = null;
+    }
+  }
+
+  private static final class ThreadDumpTask implements Runnable {
+    private final long pid;
+    private final Clock clock;
+    private final Sleeper sleeper;
+    private final Duration threadDumpInterval;
+    private final Path dumpDirectory;
+
+    private ThreadDumpTask(
+        long pid, Clock clock, Sleeper sleeper, Duration threadDumpInterval, Path dumpDirectory) {
+      this.pid = pid;
+      this.clock = clock;
+      this.sleeper = sleeper;
+      this.threadDumpInterval = threadDumpInterval;
+      this.dumpDirectory = dumpDirectory;
+    }
+
+    @Override
+    public void run() {
+      while (true) {
+        try {
+          sleeper.sleepMillis(threadDumpInterval.toMillis());
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          return;
+        }
+
+        String formattedTime =
+            Instant.ofEpochMilli(clock.currentTimeMillis())
+                .atZone(ZoneOffset.UTC)
+                .format(TIME_FORMAT);
+        Path dumpFile = dumpDirectory.getChild(String.format("dump.%d.%s.txt", pid, formattedTime));
+        try (var sc = Profiler.instance().profile("Dumping threads");
+            var out = dumpFile.getOutputStream()) {
+          ThreadDumper.dumpThreads(out);
+        } catch (IOException e) {
+          logger.atWarning().withCause(e).log("Failed to dump threads.");
+        }
+      }
+    }
+  }
+}