Always miss the first bash readline event in uretprobe

[iyear]

For the bash readline, I can get it up and running fine with the following code, but the first event is never received(trace_pipe and stdout), and all subsequent events are received. I don't know why.

The same situation occurs in go install github.com/cilium/ebpf/examples/uretprobe@latest

And it's fine when use uprobe on readline (received all events)

Clang:

Ubuntu clang version 14.0.0-1ubuntu1.1
Target: x86_64-pc-linux-gnu
Thread model: posix
InstalledDir: /usr/bin

uname: Linux ubuntu2204 5.15.0-88-generic #98-Ubuntu SMP Mon Oct 2 15:18:56 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
vmlinux.h: 5.8

Go code

package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/cilium/ebpf/link"
	"github.com/cilium/ebpf/perf"
	"github.com/cilium/ebpf/rlimit"
	"go.uber.org/multierr"
	"golang.org/x/sys/unix"
	"log"
	"os"
	"os/signal"
	"syscall"
)

//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target amd64 -type event bpf bpf.c -- -I../headers

const (
	// The path to the ELF binary containing the function to trace.
	// On some distributions, the 'readline' function is provided by a
	// dynamically-linked library, so the path of the library will need
	// to be specified instead, e.g. /usr/lib/libreadline.so.8.
	// Use `ldd /bin/bash` to find these paths.
	binPath = "/bin/bash"
	symbol  = "readline"
)

func main() {
	if err := do(); err != nil {
		log.Fatalf("error: %s", err)
	}
}

func do() (rerr error) {
	stopper := make(chan os.Signal, 1)
	signal.Notify(stopper, os.Interrupt, syscall.SIGTERM)

	// Allow the current process to lock memory for eBPF resources.
	if err := rlimit.RemoveMemlock(); err != nil {
		log.Fatal(err)
	}

	// Load pre-compiled programs and maps into the kernel.
	objs := bpfObjects{}
	if err := loadBpfObjects(&objs, nil); err != nil {
		log.Fatalf("loading objects: %s", err)
	}
	defer multierr.AppendInvoke(&rerr, multierr.Close(&objs))

	// Open an ELF binary and read its symbols.
	ex, err := link.OpenExecutable(binPath)
	if err != nil {
		log.Fatalf("opening executable: %s", err)
	}

	// Open a Uretprobe at the exit point of the symbol and attach
	// the pre-compiled eBPF program to it.
	up, err := ex.Uretprobe(symbol, objs.Readlineret, nil)
	if err != nil {
		log.Fatalf("creating uretprobe: %s", err)
	}
	defer multierr.AppendInvoke(&rerr, multierr.Close(up))

	// Open a perf event reader from userspace on the PERF_EVENT_ARRAY map
	// described in the eBPF C program.
	rd, err := perf.NewReader(objs.Events, os.Getpagesize())
	if err != nil {
		log.Fatalf("creating perf event reader: %s", err)
	}
	defer multierr.AppendInvoke(&rerr, multierr.Close(rd))

	go func() {
		// Wait for a signal and close the perf reader,
		// which will interrupt rd.Read() and make the program exit.
		<-stopper
		log.Println("Received signal, exiting program..")

		if err := rd.Close(); err != nil {
			log.Fatalf("closing perf event reader: %s", err)
		}
	}()

	log.Printf("Listening for events..")

	// bpfEvent is generated by bpf2go.
	var event bpfEvent
	for {
		fmt.Printf("Waiting for event..\n")
		record, err := rd.Read()
		if err != nil {
			if errors.Is(err, perf.ErrClosed) {
				return
			}
			log.Printf("reading from perf event reader: %s", err)
			continue
		}

		if record.LostSamples != 0 {
			log.Printf("perf event ring buffer full, dropped %d samples", record.LostSamples)
			continue
		}

		// Parse the perf event entry into a bpfEvent structure.
		if err := binary.Read(bytes.NewBuffer(record.RawSample), binary.LittleEndian, &event); err != nil {
			log.Printf("parsing perf event: %s", err)
			continue
		}

		log.Printf("%s:%s return value: %s, %d, %s", binPath, symbol, unix.ByteSliceToString(event.Line[:]), event.Pid, unix.ByteSliceToString(event.Comm[:]))
	}
}

ebpf code

// go:build ignore

#include "vmlinux.h"
#include "bpf_tracing.h"
#include "bpf_helpers.h"

char __license[] SEC("license") = "Dual MIT/GPL";

struct event
{
	u32 pid;
	u8 line[80];
	u8 comm[80];
};

struct
{
	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
} events SEC(".maps");

// Force emitting struct event into the ELF.
const struct event *unused __attribute__((unused));

SEC("uretprobe/readline")
int BPF_KRETPROBE(readlineret, const void *ret)
{
	struct event event;

	if (!ret)
		return 0;

	bpf_get_current_comm(&event.comm, sizeof(event.comm));
	event.pid = bpf_get_current_pid_tgid() >> 32;
	bpf_probe_read_user_str(&event.line, sizeof(event.line), ret);

	bpf_printk("bash_readline: %s\n", event.line);
	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &event, sizeof(event));

	return 0;
};

[iyear]

I may have figured it out. bash was "readline"ing when prompt showed up in my terminal, so the readline wasn't hijacked this time. Is that right?

[LinhanXu3928]

Same issue, has anyone know how to slove it? If I want to catch the first bash readline event using ebpf?