[SPARK-51011][CORE] Add logging for whether a task is going to be int…

…errupted when killed We now log the value of `interruptThread` when a `TaskRunner`'s `kill` method is killed. This should help with debugging when potential zombie Spark tasks do not seem to be exiting. Today, it's tricky to debug why a task is not exiting (and thus, why executors might be getting lost) without knowing for sure if it was issued a Java interrupt. No. Ran `org.apache.spark.executor.ExecutorSuite` and verified the log looked as expected. No. Closes #49699 from neilramaswamy/spark-51011. Lead-authored-by: Neil Ramaswamy <[email protected]> Co-authored-by: Hyukjin Kwon <[email protected]> Signed-off-by: Hyukjin Kwon <[email protected]> (cherry picked from commit f03f45e) Signed-off-by: Hyukjin Kwon <[email protected]>
apache · Feb 3, 2025 · 72fc7f6 · 72fc7f6
1 parent f78ae52
commit 72fc7f6
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala b/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala
@@ -331,6 +331,7 @@ private[spark] object LogKeys {
   case object INPUT extends LogKey
   case object INPUT_SPLIT extends LogKey
   case object INTEGRAL extends LogKey
+  case object INTERRUPT_THREAD extends LogKey
   case object INTERVAL extends LogKey
   case object INVALID_PARAMS extends LogKey
   case object ISOLATION_LEVEL extends LogKey

diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -504,8 +504,9 @@ private[spark] class Executor(
     @volatile var task: Task[Any] = _
 
     def kill(interruptThread: Boolean, reason: String): Unit = {
-      logInfo(log"Executor is trying to kill ${LogMDC(TASK_NAME, taskName)}," +
-        log" reason: ${LogMDC(REASON, reason)}")
+      logInfo(log"Executor is trying to kill ${LogMDC(TASK_NAME, taskName)}, " +
+        log"interruptThread: ${LogMDC(INTERRUPT_THREAD, interruptThread)}, " +
+        log"reason: ${LogMDC(REASON, reason)}")
       reasonIfKilled = Some(reason)
       if (task != null) {
         synchronized {