[GR-54697] Implement debuginfo generation at image-runtime.

substratevm/CHANGELOG.md

@@ -7,6 +7,7 @@ This changelog summarizes major changes to GraalVM Native Image.
 * (GR-59313) Deprecated class-level metadata extraction using `native-image-inspect` and removed option `DumpMethodsData`. Use class-level SBOMs instead by passing `--enable-sbom=class-level,export` to the `native-image` builder. The default value of option `IncludeMethodData` was changed to `false`.  
 * (GR-52400) The build process now uses 85% of system memory in containers and CI environments. Otherwise, it tries to only use available memory. If less than 8GB of memory are available, it falls back to 85% of system memory. The reason for the selected memory limit is now also shown in the build resources section of the build output.
 * (GR-59864) Added JVM version check to the Native Image agent. The agent will abort execution if the JVM major version does not match the version it was built with, and warn if the full JVM version is different.
+* (GR-54697) Parallelize debug info generator and add support run-time debug info generation. `-H:+RuntimeDebugInfo` adds a run-time debug info generator into a native image. The run-time debug info generator notifies GDB via the [JIT Compilation Interface](https://sourceware.org/gdb/current/onlinedocs/gdb.html/JIT-Interface.html) about new object files for run-time compilations.
 
 ## GraalVM for JDK 24 (Internal Version 24.2.0)
 * (GR-59717) Added `DuringSetupAccess.registerObjectReachabilityHandler` to allow registering a callback that is executed when an object of a specified type is marked as reachable during heap scanning.

substratevm/debug/include/gdb_jit_compilation_interface.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SVM_NATIVE_GDBJITCOMPILATIONINTERFACE_H
+#define SVM_NATIVE_GDBJITCOMPILATIONINTERFACE_H
+
+#include <stdint.h>
+
+typedef enum
+{
+  JIT_NOACTION = 0,
+  JIT_REGISTER,
+  JIT_UNREGISTER
+} jit_actions_t;
+
+struct jit_code_entry
+{
+  struct jit_code_entry *next_entry;
+  struct jit_code_entry *prev_entry;
+  const char *symfile_addr;
+  uint64_t symfile_size;
+};
+
+struct jit_descriptor
+{
+  uint32_t version;
+  /* This type should be jit_actions_t, but we use uint32_t
+     to be explicit about the bitwidth.  */
+  uint32_t action_flag;
+  struct jit_code_entry *relevant_entry;
+  struct jit_code_entry *first_entry;
+};
+
+#endif

substratevm/mx.substratevm/gdb_utils.py

@@ -110,6 +110,8 @@ def check(self, text, skip_fails=True):
         for i in range(0, num_rexps):
             rexp = rexps[i]
             match = None
+            if skip_fails:
+                line_idx = 0
             while line_idx < num_lines and match is None:
                 line = lines[line_idx]
                 match = rexp.match(line)

substratevm/mx.substratevm/mx_substratevm.py

@@ -1118,7 +1118,7 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
             else:
                 build_args += ['-o', join(build_dir, image_name)]
 
-            mx.log(f"native_image {' '.join(build_args)}")
+            mx.log(f"native-image {' '.join(build_args)}")
             native_image(build_args)
 
             if build_cinterfacetutorial:
@@ -1135,14 +1135,14 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
                 mx.run(c_command, cwd=build_dir)
         if mx.get_os() == 'linux':
             logfile = join(path, pathlib.Path(testfile).stem + ('' if with_isolates else '_no_isolates') + '.log')
-            os.environ.update({'gdbdebughelperstest_logfile': logfile})
+            os.environ.update({'gdb_logfile': logfile})
             gdb_command = gdb_args + [
                 '-iex', f"set logging file {logfile}",
                 '-iex', 'set logging enabled on',
                 '-iex', f"set auto-load safe-path {join(build_dir, 'gdb-debughelpers.py')}",
                 '-x', testfile, join(build_dir, image_name)
             ]
-            mx.log(' '.join(gdb_command))
+            mx.log(' '.join([(f"'{c}'" if ' ' in c else c) for c in gdb_command]))
             # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
             return mx.run(gdb_command, cwd=build_dir, nonZeroIsFatal=False)
         return 0
@@ -1169,6 +1169,88 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
         mx.abort(status)
 
 
+def _runtimedebuginfotest(native_image, output_path, with_isolates_only, args=None):
+    """Build and run the runtimedebuginfotest"""
+
+    args = [] if args is None else args
+
+    test_proj = mx.dependency('com.oracle.svm.test')
+    test_source_path = test_proj.source_dirs()[0]
+
+    test_python_source_dir = join(test_source_path, 'com', 'oracle', 'svm', 'test', 'debug', 'helper')
+    test_runtime_compilation_py = join(test_python_source_dir, 'test_runtime_compilation.py')
+    test_runtime_deopt_py = join(test_python_source_dir, 'test_runtime_deopt.py')
+
+    gdb_args = [
+        os.environ.get('GDB_BIN', 'gdb'),
+        '--nx',
+        '-q',  # do not print the introductory and copyright messages
+        '-iex', "set pagination off",  # messages from enabling logging could already cause pagination, so this must be done first
+        '-iex', "set logging redirect on",
+        '-iex', "set logging overwrite off",
+    ]
+
+    # clean / create output directory
+    if exists(output_path):
+        mx.rmtree(output_path)
+    mx_util.ensure_dir_exists(output_path)
+
+    # Build the native image from Java code
+    build_args = [
+        '-g', '-O0',
+        '-o', join(output_path, 'runtimedebuginfotest'),
+        '-cp', classpath('com.oracle.svm.test'),
+        # We do not want to step into class initializer, so initialize everything at build time.
+        '--initialize-at-build-time=com.oracle.svm.test.debug.helper',
+        '--features=com.oracle.svm.test.debug.helper.RuntimeCompileDebugInfoTest$RegisterMethodsFeature',
+        'com.oracle.svm.test.debug.helper.RuntimeCompileDebugInfoTest',
+    ] + svm_experimental_options([
+        '-H:DebugInfoSourceSearchPath=' + test_source_path,
+        '-H:+SourceLevelDebug',
+        '-H:+RuntimeDebugInfo',
+    ]) + args
+
+    mx.log(f"native-image {' '.join(build_args)}")
+    runtime_compile_binary = native_image(build_args)
+
+    logfile = join(output_path, 'test_runtime_compilation.log')
+    os.environ.update({'gdb_logfile': logfile})
+    gdb_command = gdb_args + [
+        '-iex', f"set logging file {logfile}",
+        '-iex', "set logging enabled on",
+        '-iex', f"set auto-load safe-path {join(output_path, 'gdb-debughelpers.py')}",
+        '-x', test_runtime_compilation_py, runtime_compile_binary
+    ]
+    mx.log(' '.join([(f"'{c}'" if ' ' in c else c) for c in gdb_command]))
+    # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
+    status = mx.run(gdb_command, cwd=output_path, nonZeroIsFatal=False)
+
+    jslib = mx.add_lib_suffix(native_image(
+        args +
+        svm_experimental_options([
+            '-H:+SourceLevelDebug',
+            '-H:+RuntimeDebugInfo',
+        ]) +
+        ['-g', '-O0', '--macro:jsvm-library']
+    ))
+    js_launcher = get_js_launcher(jslib)
+    testdeopt_js = join(suite.dir, 'mx.substratevm', 'testdeopt.js')
+    logfile = join(output_path, 'test_runtime_deopt.log')
+    os.environ.update({'gdb_logfile': logfile})
+    gdb_command = gdb_args + [
+        '-iex', f"set logging file {logfile}",
+        '-iex', "set logging enabled on",
+        '-iex', f"set auto-load safe-path {join(output_path, 'gdb-debughelpers.py')}",
+        '-x', test_runtime_deopt_py, '--args', js_launcher, testdeopt_js
+    ]
+    mx.log(' '.join([(f"'{c}'" if ' ' in c else c) for c in gdb_command]))
+    # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
+    status |= mx.run(gdb_command, cwd=output_path, nonZeroIsFatal=False)
+
+    if status != 0:
+        mx.abort(status)
+
+
 def _javac_image(native_image, path, args=None):
     args = [] if args is None else args
     mx_util.ensure_dir_exists(path)
@@ -1767,6 +1849,28 @@ def gdbdebughelperstest(args, config=None):
         config=config
     )
 
+
+@mx.command(suite.name, 'runtimedebuginfotest', 'Runs the runtime debuginfo generation test')
+def runtimedebuginfotest(args, config=None):
+    """
+    runs a native image that compiles code and creates debug info at runtime.
+    """
+    parser = ArgumentParser(prog='mx runtimedebuginfotest')
+    all_args = ['--output-path', '--with-isolates-only']
+    masked_args = [_mask(arg, all_args) for arg in args]
+    parser.add_argument(all_args[0], metavar='<output-path>', nargs=1, help='Path of the generated image', default=[join(svmbuild_dir(), "runtimedebuginfotest")])
+    parser.add_argument(all_args[1], action='store_true', help='Only build and test the native image with isolates')
+    parser.add_argument('image_args', nargs='*', default=[])
+    parsed = parser.parse_args(masked_args)
+    output_path = unmask(parsed.output_path)[0]
+    with_isolates_only = parsed.with_isolates_only
+    native_image_context_run(
+        lambda native_image, a:
+        _runtimedebuginfotest(native_image, output_path, with_isolates_only, a), unmask(parsed.image_args),
+        config=config
+    )
+
+
 @mx.command(suite_name=suite.name, command_name='helloworld', usage_msg='[options]')
 def helloworld(args, config=None):
     """
@@ -1859,6 +1963,7 @@ def build_and_test_java_agent_image(native_image, args):
 
     native_image_context_run(build_and_test_java_agent_image, args)
 
+
 @mx.command(suite.name, 'clinittest', 'Runs the ')
 def clinittest(args):
     def build_and_test_clinittest_image(native_image, args):

substratevm/mx.substratevm/suite.py

@@ -308,6 +308,7 @@
             "dependencies": [
                 "com.oracle.svm.common",
                 "com.oracle.svm.shaded.org.objectweb.asm",
+                "com.oracle.objectfile",
             ],
             "requires" : [
                 "java.compiler",
@@ -679,7 +680,6 @@
             "subDir": "src",
             "sourceDirs": ["src"],
             "dependencies": [
-                "com.oracle.objectfile",
                 "com.oracle.graal.reachability",
                 "com.oracle.svm.core.graal.amd64",
             ],
@@ -1077,6 +1077,10 @@
                     "jdk.internal.misc",
                     "sun.security.jca",
                 ],
+                "jdk.internal.vm.ci" : [
+                    "jdk.vm.ci.code",
+                    "jdk.vm.ci.meta",
+                ],
             },
             "checkstyle": "com.oracle.svm.test",
             "checkstyleVersion" : "10.21.0",
@@ -1921,6 +1925,8 @@
                 "com.oracle.objectfile",
                 "com.oracle.objectfile.io",
                 "com.oracle.objectfile.debuginfo",
+                "com.oracle.objectfile.debugentry",
+                "com.oracle.objectfile.debugentry.range",
                 "com.oracle.objectfile.macho",
               ],
 
@@ -2066,6 +2072,7 @@
                             "dependency:com.oracle.svm.native.libchelper/*",
                             "dependency:com.oracle.svm.native.jvm.posix/*",
                             "dependency:com.oracle.svm.native.libcontainer/*",
+                            "file:debug/include",
                         ],
                     },
                 },
@@ -2074,6 +2081,7 @@
                         # on all other os's we don't want libc specific subdirectories
                         "include/": [
                             "dependency:com.oracle.svm.native.libchelper/include/*",
+                            "file:debug/include/*",
                         ],
                         "<os>-<arch>/": [
                             "dependency:com.oracle.svm.native.libchelper/<os>-<arch>/default/*",

substratevm/mx.substratevm/testdeopt.js

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+function add(a, b, test) {
+    if (test) {
+        a += b;
+    }
+    return a + b;
+}
+
+// trigger compilation add for ints and test = true
+for (let i = 0; i < 1000 * 1000; i++) {
+    add(i, i, true);
+}
+
+// deoptimize with failed assumption in compiled method
+// then trigger compilation again
+console.log("deopt1")
+for (let i = 0; i < 1000 * 1000; i++) {
+    add(i, i, false);
+}
+
+// deoptimize with different parameter types
+console.log("deopt2");
+add({f1: "test1", f2: 2}, {x: "x", y: {test: 42}}, false);

substratevm/mx.substratevm/testhello.py

@@ -147,25 +147,25 @@ def test():
 
     # check incoming parameters are bound to sensible values
     exec_string = execute("info args")
-    rexp = [fr"__0 = {digits_pattern}",
-            fr"__1 = 0x{hex_digits_pattern}"]
+    rexp = [fr"__int0 = {digits_pattern}",
+            fr"__long1 = 0x{hex_digits_pattern}"]
     checker = Checker(f"info args : {method_name}", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __0")
+    exec_string = execute("p __int0")
     rexp = [fr"\${digits_pattern} = 1"]
-    checker = Checker("p __0", rexp)
+    checker = Checker("p __int0", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __1")
+    exec_string = execute("p __long1")
     rexp = [fr"\${digits_pattern} = \(org\.graalvm\.nativeimage\.c\.type\.CCharPointerPointer\) 0x{hex_digits_pattern}"]
-    checker = Checker("p __1", rexp)
+    checker = Checker("p __long1", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __1[0]")
+    exec_string = execute("p __long1[0]")
     rexp = [
         fr'\${digits_pattern} = \(org\.graalvm\.nativeimage\.c\.type\.CCharPointer\) 0x{hex_digits_pattern} "{wildcard_pattern}/hello_image"']
-    checker = Checker("p __1[0]", rexp)
+    checker = Checker("p __long1[0]", rexp)
     checker.check(exec_string)
 
     # set a break point at hello.Hello::main

substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java

@@ -219,16 +219,12 @@ public static Format getNativeFormat() {
     }
 
     private static ObjectFile getNativeObjectFile(int pageSize, boolean runtimeDebugInfoGeneration) {
-        switch (ObjectFile.getNativeFormat()) {
-            case ELF:
-                return new ELFObjectFile(pageSize, runtimeDebugInfoGeneration);
-            case MACH_O:
-                return new MachOObjectFile(pageSize);
-            case PECOFF:
-                return new PECoffObjectFile(pageSize);
-            default:
-                throw new AssertionError("Unreachable");
-        }
+        return switch (ObjectFile.getNativeFormat()) {
+            case ELF -> new ELFObjectFile(pageSize, runtimeDebugInfoGeneration);
+            case MACH_O -> new MachOObjectFile(pageSize);
+            case PECOFF -> new PECoffObjectFile(pageSize);
+            default -> throw new AssertionError("Unreachable");
+        };
     }
 
     public static ObjectFile getNativeObjectFile(int pageSize) {
@@ -1828,7 +1824,7 @@ public final SymbolTable getOrCreateSymbolTable() {
      * Temporary storage for a debug context installed in a nested scope under a call. to
      * {@link #withDebugContext}
      */
-    private DebugContext debugContext = null;
+    protected DebugContext debugContext = DebugContext.disabled(null);
 
     /**
      * Allows a task to be executed with a debug context in a named subscope bound to the object