Go
by smallnest

使用eBPF编写系统调用跟踪器

先决条件

系统调用、eBPF、C语言、底层编程基础。

简介

eBPF(扩展的伯克利数据包过滤器)是一项允许用户在内核中运行自定义程序的技术。BPF或cBPF(经典BPF)是eBPF的前身,它提供了一种简单高效的方法来基于预定义规则过滤数据包。与内核模块相比,eBPF程序提供了更高的安全性、可移植性和可维护性。现有多种高级方法可用于处理eBPF程序,如Cilium的Go语言库、bpftrace、libbpf等。

  • 注意: 本文要求读者对eBPF有基本了解。如果你不熟悉它,ebpf.io上的这篇文章是很好的参考资料。

目标

你应该已经熟悉著名的工具 strace。我们将使用eBPF开发类似的工具。例如:

1
./beetrace /bin/ls

以下是该文本的地道中文翻译：

概念

在开始编写我们的工具之前，我们需要熟悉一些关键概念。

  1. 跟踪点（Tracepoints）：这些是放置在 Linux 内核代码各个部分的检测点。它们提供了一种方法，可以在不修改内核源代码的情况下，钩入内核中的特定事件或代码路径。可用于跟踪的事件可以在 /sys/kernel/debug/tracing/events 中找到。
  2. SEC 宏：它在目标 ELF 文件中创建一个新的段，段名与跟踪点的名称相同。例如，SEC(tracepoint/raw_syscalls/sys_enter) 创建了一个具有这个名称的新段。可以使用 readelf 命令查看这些段。
1
readelf -s --wide somefile.o
  1. 映射（Maps）：这些是可以从 eBPF 程序和用户空间运行的应用程序中访问的共享数据结构。

编写 eBPF 程序

由于 Linux 内核中存在大量的系统调用，我们不会编写一个全面的工具来跟踪所有系统调用。相反，我们将专注于跟踪几个常见的系统调用。为了实现这一目标，我们将编写两类程序：eBPF 程序和加载器（用于将 BPF 对象加载到内核并将其附加进来）。

让我们首先创建一些数据结构来进行初始设置:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
// controller.h
// SYS_ENTER : for retrieving system call arguments
// SYS_EXIT : for retrieving the return values of syscalls
typedef enum
{
    SYS_ENTER,
    SYS_EXIT
} event_mode;
struct inner_syscall_info
{
    union
    {
        struct
        {
            // For SYS_ENTER mode
            char name[32];
            int num_args;
            long syscall_nr;
            void *args[MAX_ARGS];
        };
        long retval; // For SYS_EXIT mode
    };
    event_mode mode;
};
struct default_syscall_info{
    char name[32];
    int num_args;
};
// Array for storing the name and argument count of system calls
const struct default_syscall_info syscalls[MAX_SYSCALL_NR] = {
    [SYS_fork] = {"fork", 0},
    [SYS_alarm] = {"alarm", 1},
    [SYS_brk] = {"brk", 1},
    [SYS_close] = {"close", 1},
    [SYS_exit] = {"exit", 1},
    [SYS_exit_group] = {"exit_group", 1},
    [SYS_set_tid_address] = {"set_tid_address", 1},
    [SYS_set_robust_list] = {"set_robust_list", 1},
    [SYS_access] = {"access", 2},
    [SYS_arch_prctl] = {"arch_prctl", 2},
    [SYS_kill] = {"kill", 2},
    [SYS_listen] = {"listen", 2},
    [SYS_munmap] = {"sys_munmap", 2},
    [SYS_open] = {"open", 2},
    [SYS_stat] = {"stat", 2},
    [SYS_fstat] = {"fstat", 2},
    [SYS_lstat] = {"lstat", 2},
    [SYS_accept] = {"accept", 3},
    [SYS_connect] = {"connect", 3},
    [SYS_execve] = {"execve", 3},
    [SYS_ioctl] = {"ioctl", 3},
    [SYS_getrandom] = {"getrandom", 3},
    [SYS_lseek] = {"lseek", 3},
    [SYS_poll] = {"poll", 3},
    [SYS_read] = {"read", 3},
    [SYS_write] = {"write", 3},
    [SYS_mprotect] = {"mprotect", 3},
    [SYS_openat] = {"openat", 3},
    [SYS_socket] = {"socket", 3},
    [SYS_newfstatat] = {"newfstatat", 4},
    [SYS_pread64] = {"pread64", 4},
    [SYS_prlimit64] = {"prlimit64", 4},
    [SYS_rseq] = {"rseq", 4},
    [SYS_sendfile] = {"sendfile", 4},
    [SYS_socketpair] = {"socketpair", 4},
    [SYS_mmap] = {"mmap", 6},
    [SYS_recvfrom] = {"recvfrom", 6},
    [SYS_sendto] = {"sendto", 6},
};

加载器将读取用户通过命令行参数提供的待追踪 ELF 文件的路径。然后，加载器会创建一个子进程，并使用 execve 来运行命令行参数中指定的程序。

父进程将处理加载和附加 eBPF 程序所需的所有设置。它还执行一项关键任务：通过 BPF 哈希映射将子进程的 ID 发送给 eBPF 程序。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
// loader.c
int main(int argc, char **argv)
{
  if (argc < 2)
  {
    fatal_error("Usage: ./beetrace <path_to_program>");
  }
  const char *file_path = argv[1];
  pid_t pid = fork();
  if (pid == 0)
  {
    // Child process
    int fd = open("/dev/null", O_WRONLY);
    if(fd==-1){
        // error
    }
    dup2(fd, 1); // disable stdout for the child process
    sleep(2); // wait for the parent process to do the required setup for tracing
    execve(file_path, NULL, NULL);
  }
  else{
    // Parent process
  }
}

要追踪系统调用，我们需要编写由 tracepoint/raw_syscalls/sys_enter 和 tracepoint/raw_syscalls/sys_exit 跟踪点触发的 eBPF 程序。这些跟踪点提供了对系统调用号和参数的访问。对于给定的系统调用，tracepoint/raw_syscalls/sys_enter 跟踪点总是在 tracepoint/raw_syscalls/sys_exit 跟踪点之前触发。我们可以使用前者获取系统调用参数，使用后者获取返回值。

此外，我们将使用 eBPF 映射在用户空间程序和我们的 eBPF 程序之间共享信息。具体来说，我们将使用两种类型的 eBPF 映射：哈希映射和环形缓冲区。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
// controller.c
// Hashmap
struct
{
  __uint(type, BPF_MAP_TYPE_HASH);
  __uint(key_size, 10);
  __uint(value_size, 4);
  __uint(max_entries, 256 * 1024);
} pid_hashmap SEC(".maps");
// Ring buffer
struct
{
  __uint(type, BPF_MAP_TYPE_RINGBUF);
  __uint(max_entries, 256 * 1024);
} syscall_info_buffer SEC(".maps");

确定了映射关系之后，我们就可以动手写代码了。首先，让我们来编写针对追踪点 tracepoint/raw_syscalls/sys_enter 的程序代码。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
// loader.c
SEC("tracepoint/raw_syscalls/sys_enter")
int detect_syscall_enter(struct trace_event_raw_sys_enter *ctx)
{
  // Retrieve the system call number
  long syscall_nr = ctx->id;
  const char *key = "child_pid";
  int target_pid;
  // Reading the process id of the child process in userland
  void *value = bpf_map_lookup_elem(&pid_hashmap, key);
  void *args[MAX_ARGS];
  if (value)
  {
    target_pid = *(int *)value;
    // PID of the process that executed the current system call
    pid_t pid = bpf_get_current_pid_tgid() & 0xffffffff;
    if (pid == target_pid && syscall_nr >= 0 && syscall_nr < MAX_SYSCALL_NR)
    {
      int idx = syscall_nr;
      // Reserve space in the ring buffer
      struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
      if (!info)
      {
        bpf_printk("bpf_ringbuf_reserve failed");
        return 1;
      }
      // Copy the syscall name into info->name
      bpf_probe_read_kernel_str(info->name, sizeof(syscalls[syscall_nr].name), syscalls[syscall_nr].name);
      for (int i = 0; i < MAX_ARGS; i++)
      {
        info->args[i] = (void *)BPF_CORE_READ(ctx, args[i]);
      }
      info->num_args = syscalls[syscall_nr].num_args;
      info->syscall_nr = syscall_nr;
      info->mode = SYS_ENTER;
      // Insert into ring buffer
      bpf_ringbuf_submit(info, 0);
    }
  }
  return 0;
}

同理，我们也能编写用于读取返回值并将其传递给用户态空间的程序代码。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
// controller.c
SEC("tracepoint/raw_syscalls/sys_exit")
int detect_syscall_exit(struct trace_event_raw_sys_exit *ctx)
{
  const char *key = "child_pid";
  void *value = bpf_map_lookup_elem(&pid_hashmap, key);
  pid_t pid, target_pid;
  if (value)
  {
    pid = bpf_get_current_pid_tgid() & 0xffffffff;
    target_pid = *(pid_t *)value;
    if (pid == target_pid)
    {
      struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
      if (!info)
      {
        bpf_printk("bpf_ringbuf_reserve failed");
        return 1;
      }
      info->mode = SYS_EXIT;
      info->retval = ctx->ret;
      bpf_ringbuf_submit(info, 0);
    }
  }
  return 0;
}

现在，让我们来完善加载器程序中父进程的功能部分。但在进行之前，我们需要理解几个关键函数的工作原理。
1、bpf_object__open: 通过打开由传递路径指向的 BPF ELF 对象文件并在内存中加载它，创建一个 bpf_object 结构体实例。

1
LIBBPF_API struct bpf_object *bpf_object__open(const char *path);

2、bpf_object__load: 将 BPF 对象加载到内核中。

1
LIBBPF_API int bpf_object__load(struct bpf_object *obj);

3、bpf_object__find_program_by_name: 返回指向有效 BPF 程序的指针。

1
LIBBPF_API struct bpf_program *bpf_object__find_program_by_name(const struct bpf_object *obj, const char *name);

4、bpf_program__attach: 根据自动检测的程序类型、附加类型和适用的额外参数，将 BPF 程序附加到内核。

1
LIBBPF_API struct bpf_link *bpf_program__attach(const struct bpf_program *prog);

5、bpf_map__update_elem: 允许在与提供的键对应的 BPF 映射中插入或更新值。

1
LIBBPF_API int bpf_map__update_elem(const struct bpf_map *map, const void *key, size_t key_sz, const void *value, size_t value_sz, __u64 flags);

6、bpf_object__find_map_fd_by_name: 给定一个 BPF 映射名称，返回该映射的文件描述符。

1
LIBBPF_API int bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name);

7、ring_buffer__new: 返回指向环形缓冲区的指针。

1
LIBBPF_API struct ring_buffer *ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, const struct ring_buffer_opts *opts);

第二个参数必须是一个可用于处理从环形缓冲区接收的数据的回调函数。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
bool initialized = false;
static int syscall_logger(void *ctx, void *data, size_t len)
{
  struct inner_syscall_info *info = (struct inner_syscall_info *)data;
  if (!info)
  {
    return -1;
  }
  if (info->mode == SYS_ENTER)
  {
    initialized = true;
    printf("%s(", info->name);
    for (int i = 0; i < info->num_args; i++)
    {
      printf("%p,", info->args[i]);
    }
    printf("\b) = ");
  }
  else if (info->mode == SYS_EXIT)
  {
    if (initialized)
    {
      printf("0x%lx\n", info->retval);
    }
  }
  return 0;
}

它会打印系统调用的名称和参数。

8、ring_buffer__consume: 此函数处理环形缓冲区中可用的事件。

1
LIBBPF_API int ring_buffer__consume(struct ring_buffer *rb);

现在我们有了编写加载器所需的一切要素。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
// loader.c
#include <bpf/libbpf.h>
#include "controller.h"
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <unistd.h>
void fatal_error(const char *message)
{
  puts(message);
  exit(1);
}
bool initialized = false;
static int syscall_logger(void *ctx, void *data, size_t len)
{
  struct inner_syscall_info *info = (struct inner_syscall_info *)data;
  if (!info)
  {
    return -1;
  }
  if (info->mode == SYS_ENTER)
  {
    initialized = true;
    printf("%s(", info->name);
    for (int i = 0; i < info->num_args; i++)
    {
      printf("%p,", info->args[i]);
    }
    printf("\b) = ");
  }
  else if (info->mode == SYS_EXIT)
  {
    if (initialized)
    {
      printf("0x%lx\n", info->retval);
    }
  }
  return 0;
}
int main(int argc, char **argv)
{
  int status;
  struct bpf_object *obj;
  struct bpf_program *enter_prog, *exit_prog;
  struct bpf_map *syscall_map;
  const char *obj_name = "controller.o";
  const char *map_name = "pid_hashmap";
  const char *enter_prog_name = "detect_syscall_enter"; 
  const char *exit_prog_name = "detect_syscall_exit";
  const char *syscall_info_bufname = "syscall_info_buffer";
  if (argc < 2)
  {
    fatal_error("Usage: ./beetrace <path_to_program>");
  }
  const char *file_path = argv[1];
  pid_t pid = fork();
  if (pid == 0)
  {
    int fd = open("/dev/null", O_WRONLY);
    if(fd==-1){
      fatal_error("failed to open /dev/null");
    }
    dup2(fd, 1);
    sleep(2);
    execve(file_path, NULL, NULL);
  }
  else
  {
    printf("Spawned child process with a PID of %d\n", pid);
    obj = bpf_object__open(obj_name);
    if (!obj)
    {
      fatal_error("failed to open the BPF object");
    }
    if (bpf_object__load(obj))
    {
      fatal_error("failed to load the BPF object into kernel");
    }
    enter_prog = bpf_object__find_program_by_name(obj, enter_prog_name);
    exit_prog = bpf_object__find_program_by_name(obj, exit_prog_name);
    if (!enter_prog || !exit_prog)
    {
      fatal_error("failed to find the BPF program");
    }
    if (!bpf_program__attach(enter_prog) || !bpf_program__attach(exit_prog))
    {
      fatal_error("failed to attach the BPF program");
    }
    syscall_map = bpf_object__find_map_by_name(obj, map_name);
    if (!syscall_map)
    {
      fatal_error("failed to find the BPF map");
    }
    const char *key = "child_pid";
    int err = bpf_map__update_elem(syscall_map, key, 10, (void *)&pid, sizeof(pid_t), 0);
    if (err)
    {
      printf("%d", err);
      fatal_error("failed to insert child pid into the ring buffer");
    }
    int rbFd = bpf_object__find_map_fd_by_name(obj, syscall_info_bufname);
    struct ring_buffer *rbuffer = ring_buffer__new(rbFd, syscall_logger, NULL, NULL);
    if (!rbuffer)
    {
      fatal_error("failed to allocate ring buffer");
    }
    if (wait(&status) == -1)
    {
      fatal_error("failed to wait for the child process");
    }
    while (1)
    {
      int e = ring_buffer__consume(rbuffer);
      if (!e)
      {
        break;
      }
      sleep(1);
    }
  }
  return 0;
}

以下便是 eBPF 程序的部分。所有的 C 语言源码最终会被编译整合成单一的对象文件。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
// controller.c
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
#include <sys/syscall.h>
#include "controller.h"
struct
{
  __uint(type, BPF_MAP_TYPE_HASH);
  __uint(key_size, 10);
  __uint(value_size, 4);
  __uint(max_entries, 256 * 1024);
} pid_hashmap SEC(".maps");
struct
{
  __uint(type, BPF_MAP_TYPE_RINGBUF);
  __uint(max_entries, 256 * 1024);
} syscall_info_buffer SEC(".maps");
SEC("tracepoint/raw_syscalls/sys_enter")
int detect_syscall_enter(struct trace_event_raw_sys_enter *ctx)
{
  // Retrieve the system call number
  long syscall_nr = ctx->id;
  const char *key = "child_pid";
  int target_pid;
  // Reading the process id of the child process in userland
  void *value = bpf_map_lookup_elem(&pid_hashmap, key);
  void *args[MAX_ARGS];
  if (value)
  {
    target_pid = *(int *)value;
    // PID of the process that executed the current system call
    pid_t pid = bpf_get_current_pid_tgid() & 0xffffffff;
    if (pid == target_pid && syscall_nr >= 0 && syscall_nr < MAX_SYSCALL_NR)
    {
      int idx = syscall_nr;
      // Reserve space in the ring buffer
      struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
      if (!info)
      {
        bpf_printk("bpf_ringbuf_reserve failed");
        return 1;
      }
      // Copy the syscall name into info->name
      bpf_probe_read_kernel_str(info->name, sizeof(syscalls[syscall_nr].name), syscalls[syscall_nr].name);
      for (int i = 0; i < MAX_ARGS; i++)
      {
        info->args[i] = (void *)BPF_CORE_READ(ctx, args[i]);
      }
      info->num_args = syscalls[syscall_nr].num_args;
      info->syscall_nr = syscall_nr;
      info->mode = SYS_ENTER;
      // Insert into ring buffer
      bpf_ringbuf_submit(info, 0);
    }
  }
  return 0;
}
SEC("tracepoint/raw_syscalls/sys_exit")
int detect_syscall_exit(struct trace_event_raw_sys_exit *ctx)
{
  const char *key = "child_pid";
  void *value = bpf_map_lookup_elem(&pid_hashmap, key);
  pid_t pid, target_pid;
  if (value)
  {
    pid = bpf_get_current_pid_tgid() & 0xffffffff;
    target_pid = *(pid_t *)value;
    if (pid == target_pid)
    {
      struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
      if (!info)
      {
        bpf_printk("bpf_ringbuf_reserve failed");
        return 1;
      }
      info->mode = SYS_EXIT;
      info->retval = ctx->ret;
      bpf_ringbuf_submit(info, 0);
    }
  }
  return 0;
}
char LICENSE[] SEC("license") = "GPL";

编译之前，我们不妨先构建一个测试程序，以便后续使用我们的工具对其进行追踪分析。

1
2
3
4
5
#include<stdio.h>
int main(){
    puts("tracer in action");
    return 0;
}

可以利用下面提供的 Makefile 来完成所有相关组件的编译工作。

1
2
3
compile:
	clang -O2 -g -Wall -I/usr/include -I/usr/include/bpf -o beetrace loader.c -lbpf
	clang -O2 -g -target bpf -c controller.c -o controller.o

整个代码可以在以下的GitHub仓库中找到：
https://github.com/0xSh4dy/bee_tracer

参考链接：