[译]strace的10个命令

strace是一个在类Unix操作系统如Linux上做debugging和trouble shooting的超级好用的工具。它可以捕获和记录进程的所有系统调用，以及这个进程接收的所有信号。

原文: 10 Strace Commands for Troubleshooting and Debugging Linux Processes

如果你的操作系统还没有安装strace，你可以运行下面的命令进行安装：

$ sudo apt install strace	#Debian/Ubuntu 
# yum install strace		#RHEL/CentOS
# dnf install strace		#Fedora 22+

如果一个程序崩溃或以一种出乎意料的方式运行，您可以通过它的系统调用来获得在执行过程中到底发生了什么的线索。我们将在后面看到，系统调用可以分为不同的事件：与进程管理相关的事件、以文件为参数的事件、涉及网络、内存映射、信号、IPC以及与文件描述符相关的系统调用。

可以使用strace运行程序/命令，也可以使用-p选项将PID传递给它，如下面示例所示。

追踪Linux系统调用

你可以通过运行下面的命令，追踪df命令的系统调用。

$ strace df -h
execve("/bin/df", ["df", "-h"], [/* 50 vars */]) = 0
brk(NULL)                               = 0x136e000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f82f78fd000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=147662, ...}) = 0
mmap(NULL, 147662, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f82f78d8000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f82f7310000
...

从上面的输出结果，我们可以看到几种类型的系统调用，比如

open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3

其中

• open: 系统调用的类型
• (“/etc/ld.so.cache”, O_RDONLY|O_CLOEXEC) : 系统调用的参数
• 3: 系统调用的返回结果

下面是df命令执行的时候write系统调用信息:

mmap(NULL, 26258, PROT_READ, MAP_SHARED, 3, 0) = 0x7f82f78f5000
close(3)                                = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1), ...}) = 0
write(1, "Filesystem      Size  Used Avail"..., 49Filesystem      Size  Used Avail Use% Mounted on
) = 49
write(1, "udev            3.9G     0  3.9G"..., 43udev            3.9G     0  3.9G   0% /dev
) = 43
write(1, "tmpfs           788M  9.6M  779M"..., 43tmpfs           788M  9.6M  779M   2% /run
) = 43
write(1, "/dev/sda10      324G  252G   56G"..., 40/dev/sda10      324G  252G   56G  82% /
) = 40
write(1, "tmpfs           3.9G  104M  3.8G"..., 47tmpfs           3.9G  104M  3.8G   3% /dev/shm
) = 47
write(1, "tmpfs           5.0M  4.0K  5.0M"..., 48tmpfs           5.0M  4.0K  5.0M   1% /run/lock
) = 48
write(1, "tmpfs           3.9G     0  3.9G"..., 53tmpfs           3.9G     0  3.9G   0% /sys/fs/cgroup
) = 53
write(1, "cgmfs           100K     0  100K"..., 56cgmfs           100K     0  100K   0% /run/cgmanager/fs
) = 56
write(1, "tmpfs           788M   36K  788M"..., 53tmpfs           788M   36K  788M   1% /run/user/1000
) = 53
close(1)                                = 0
close(2)                                = 0
exit_group(0)                           = ?
+++ exited with 0 +++

根据进程PID进行追踪

如果一个进程已经在运行，你可以通过它的pid进行追踪，它会显示追踪后这个进程的系统调用，使用CTRL+C退出。

$ sudo strace -p 3569
strace: Process 3569 attached
restart_syscall(<... resuming interrupted poll ...>) = 1
recvmsg(4, {msg_name(0)=NULL, msg_iov(1)=[{"U\2\24\300!\247\330\0\3\24\4\0\20\0\0\0\0\0\0\24\24\24\24\24\0\0\3\37%\2\0\0", 4096}], msg_controllen=0, msg_flags=0}, 0) = 32
recvmsg(4, 0x7ffee4dbf870, 0)           = -1 EAGAIN (Resource temporarily unavailable)
recvmsg(4, 0x7ffee4dbf850, 0)           = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}, {fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=10, events=POLLIN}, {fd=30, events=POLLIN}, {fd=31, events=POLLIN}], 6, -1) = 1 ([{fd=31, revents=POLLIN}])
read(31, "\372", 1)                     = 1
recvmsg(4, 0x7ffee4dbf850, 0)           = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}, {fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=10, events=POLLIN}, {fd=30, events=POLLIN}, {fd=31, events=POLLIN}], 6, 0) = 1 ([{fd=31, revents=POLLIN}])
read(31, "\372", 1)                     = 1
recvmsg(4, 0x7ffee4dbf850, 0)           = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}, {fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=10, events=POLLIN}, {fd=30, events=POLLIN}, {fd=31, events=POLLIN}], 6, 0) = 0 (Timeout)
mprotect(0x207faa20000, 8192, PROT_READ|PROT_WRITE) = 0
mprotect(0x207faa20000, 8192, PROT_READ|PROT_EXEC) = 0
mprotect(0x207faa21000, 4096, PROT_READ|PROT_WRITE) = 0
mprotect(0x207faa21000, 4096, PROT_READ|PROT_EXEC) = 0
...

得到进程的汇总信息

使用-c参数，可以得到追踪的每一种系统调用的耗时、次数和失败数，如下所示：

$ sudo strace -c -p 3569

strace: Process 3569 attached

^Cstrace: Process 3569 detached
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 99.73    0.016000           8      1971           poll
  0.16    0.000025           0       509        75 futex
  0.06    0.000010           0      1985      1966 recvmsg
  0.06    0.000009           0      2336           mprotect
  0.00    0.000000           0       478           read
  0.00    0.000000           0        13           write
  0.00    0.000000           0        29           mmap
  0.00    0.000000           0         9           munmap
  0.00    0.000000           0        18           writev
  0.00    0.000000           0       351           madvise
  0.00    0.000000           0         1           restart_syscall
------ ----------- ----------- --------- --------- ----------------
100.00    0.016044                  7700      2041 total

打印指令指针

-i可以显示每一次系统调用的时候的指令指针。

$ sudo strace -i df -h
[00007f0d7534c777] execve("/bin/df", ["df", "-h"], [/* 17 vars */]) = 0
[00007faf9cafa4b9] brk(NULL)            = 0x12f0000
[00007faf9cafb387] access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
[00007faf9cafb47a] mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7faf9cd03000
[00007faf9cafb387] access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
[00007faf9cafb327] open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
[00007faf9cafb2b4] fstat(3, {st_mode=S_IFREG|0644, st_size=147662, ...}) = 0
[00007faf9cafb47a] mmap(NULL, 147662, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7faf9ccde000
[00007faf9cafb427] close(3)             = 0
[00007faf9cafb387] access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
[00007faf9cafb327] open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
[00007faf9cafb347] read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\2\0\0\0\0\0"..., 832) = 832
[00007faf9cafb2b4] fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
[00007faf9cafb47a] mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7faf9c716000
[00007faf9cafb517] mprotect(0x7faf9c8d6000, 2097152, PROT_NONE) = 0
...

显示每一次调用的时间

-t参数可以显示时间戳。

$ sudo strace -t df -h
15:19:25 execve("/bin/df", ["df", "-h"], [/* 17 vars */]) = 0
15:19:25 brk(NULL)                      = 0x234c000
15:19:25 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
15:19:25 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8c7f1d9000
15:19:25 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
15:19:25 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
15:19:25 fstat(3, {st_mode=S_IFREG|0644, st_size=147662, ...}) = 0
15:19:25 mmap(NULL, 147662, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c7f1b4000
15:19:25 close(3)                       = 0
15:19:25 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
15:19:25 open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
15:19:25 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\2\0\0\0\0\0"..., 832) = 832
15:19:25 fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
15:19:25 mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f8c7ebec000
15:19:25 mprotect(0x7f8c7edac000, 2097152, PROT_NONE) = 0
...

显示系统调用的耗时

-T参数可以显示系统调用的耗时时间。

$ sudo strace -T df -h

execve("/bin/df", ["df", "-h"], [/* 17 vars */]) = 0 <0.000287>
brk(NULL)                               = 0xeca000 <0.000035>
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory) <0.000028>
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9aff2b1000 <0.000020>
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory) <0.000019>
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 <0.000022>
fstat(3, {st_mode=S_IFREG|0644, st_size=147662, ...}) = 0 <0.000015>
mmap(NULL, 147662, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9aff28c000 <0.000019>
close(3)                                = 0 <0.000014>
...

只追踪特定的系统调用

下面的命令中,trace=write是使用一个特定的表达式做筛选，只追踪特定的系统调用。表达式可以是signal, abbrev, verbose, raw, read 和 write等几种类型。

$ sudo strace -e trace=write df -h

write(1, "Filesystem      Size  Used Avail"..., 49Filesystem      Size  Used Avail Use% Mounted on
) = 49
write(1, "udev            3.9G     0  3.9G"..., 43udev            3.9G     0  3.9G   0% /dev
) = 43
write(1, "tmpfs           788M  9.6M  779M"..., 43tmpfs           788M  9.6M  779M   2% /run
) = 43
write(1, "/dev/sda10      324G  252G   56G"..., 40/dev/sda10      324G  252G   56G  82% /
) = 40
write(1, "tmpfs           3.9G  104M  3.8G"..., 47tmpfs           3.9G  104M  3.8G   3% /dev/shm
) = 47
write(1, "tmpfs           5.0M  4.0K  5.0M"..., 48tmpfs           5.0M  4.0K  5.0M   1% /run/lock
) = 48
write(1, "tmpfs           3.9G     0  3.9G"..., 53tmpfs           3.9G     0  3.9G   0% /sys/fs/cgroup
) = 53
write(1, "cgmfs           100K     0  100K"..., 56cgmfs           100K     0  100K   0% /run/cgmanager/fs
) = 56
write(1, "tmpfs           788M   28K  788M"..., 53tmpfs           788M   28K  788M   1% /run/user/1000
) = 53
+++ exited with 0 +++

又比如

$ sudo strace -e trace=open,close df -h
$ sudo strace -e trace=open,close,read,write df -h
$ sudo strace -e trace=all df -h

基于特定条件的系统调用

可以针对特定类型进行追踪(process、file、memory、network、signal)等等。

针对进行管理的追踪:

$ sudo strace -q -e trace=process df -h	

execve("/bin/df", ["df", "-h"], [/* 17 vars */]) = 0
arch_prctl(ARCH_SET_FS, 0x7fe2222ff700) = 0
Filesystem      Size  Used Avail Use% Mounted on
udev            3.9G     0  3.9G   0% /dev
tmpfs           788M  9.6M  779M   2% /run
/dev/sda10      324G  252G   56G  82% /
tmpfs           3.9G  104M  3.8G   3% /dev/shm
tmpfs           5.0M  4.0K  5.0M   1% /run/lock
tmpfs           3.9G     0  3.9G   0% /sys/fs/cgroup
cgmfs           100K     0  100K   0% /run/cgmanager/fs
tmpfs           788M   28K  788M   1% /run/user/1000
exit_group(0)                           = ?
+++ exited with 0 +++

针对文件系统调用的追踪：

$ sudo strace -q  -e trace=file df -h

execve("/bin/df", ["df", "-h"], [/* 17 vars */]) = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
...

针对内存的追踪:

$ sudo strace -q -e trace=memory df -h	

brk(NULL)                               = 0x77a000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe8f4658000
mmap(NULL, 147662, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fe8f4633000
mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fe8f406b000
mprotect(0x7fe8f422b000, 2097152, PROT_NONE) = 0
mmap(0x7fe8f442b000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7fe8f442b000
mmap(0x7fe8f4431000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fe8f4431000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe8f4632000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe8f4631000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe8f4630000
mprotect(0x7fe8f442b000, 16384, PROT_READ) = 0
mprotect(0x616000, 4096, PROT_READ)     = 0
mprotect(0x7fe8f465a000, 4096, PROT_READ) = 0
munmap(0x7fe8f4633000, 147662)          = 0
mmap(NULL, 2981280, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fe8f3d93000
brk(NULL)                               = 0x77a000
brk(0x79b000)                           = 0x79b000
mmap(NULL, 619, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fe8f4657000
mmap(NULL, 26258, PROT_READ, MAP_SHARED, 3, 0) = 0x7fe8f4650000
Filesystem      Size  Used Avail Use% Mounted on
udev            3.9G     0  3.9G   0% /dev
tmpfs           788M  9.6M  779M   2% /run
/dev/sda10      324G  252G   56G  82% /
tmpfs           3.9G  104M  3.8G   3% /dev/shm
tmpfs           5.0M  4.0K  5.0M   1% /run/lock
tmpfs           3.9G     0  3.9G   0% /sys/fs/cgroup
cgmfs           100K     0  100K   0% /run/cgmanager/fs
tmpfs           788M   28K  788M   1% /run/user/1000
+++ exited with 0 +++

以及针对网络和信号的追踪:

$ sudo strace -e trace=network df -h
$ sudo strace -e trace=signal df -h

将追踪结果写入到文件

-o参数将标准输出写入到文件:

$ sudo strace -o df_debug.txt df -h

Filesystem      Size  Used Avail Use% Mounted on
udev            3.9G     0  3.9G   0% /dev
tmpfs           788M  9.6M  779M   2% /run
/dev/sda10      324G  252G   56G  82% /
tmpfs           3.9G  104M  3.8G   3% /dev/shm
tmpfs           5.0M  4.0K  5.0M   1% /run/lock
tmpfs           3.9G     0  3.9G   0% /sys/fs/cgroup
cgmfs           100K     0  100K   0% /run/cgmanager/fs
tmpfs           788M   28K  788M   1% /run/user/1000

显示strace的debug信息

-d可以显示strace的debug信息。

更多的信息参考man strace。

其它一些参考资料:

• 20 Command Line Tools to Monitor Linux Performance
• Sysdig – A Powerful System Monitoring and Troubleshooting Tool for Linux
• How to Trace Execution of Commands in Shell Script with Shell Tracing
• BCC – Dynamic Tracing Tools for Linux Performance Monitoring, Networking and More