trace 以下 ftp 进程的 stack：

# ps -ef|grep ftp
root 20620 29427 0 12:02:45 ttypc 0:00 ftp bj71s020

在 live kernel 上运行 Q4：

# q4 /stand/vmunix /dev/mem
...
把进程列表 load 出来：
q4> load struct proc from proc_list next p_factp max nproc
loaded 142 struct procs as a linked list (stopped by null pointer)

只留下进程 20620：
q4> keep p_pid==20620
kept 1 of 142 struct proc's, discarded 141

看看 20620 的进程名字：
q4> print p_comm
p_comm
"ftp"

检查当前进程的 stack trace：

q4> trace pile
stack trace for process at 0x0`42854040 (pid 20620), thread at 0x0`4296c040
(tid
220896)
process was not running on any processor
_sleep+0x86c
sleep_spinunlock+0x70
pty1_sleep+0x58
canon+0xf0
ttread+0xd8
pty1_read+0xd8
spec_rdwr+0x204
vno_rw+0x1ac
read+0x10c
syscall+0x750
$syscallrtn+0x0
所以，该进程处于 sleep 状态，而且是等待终端输入。

实 验
# chmod g-x,g+s /tmp/top.out
# lockfile /tmp/top.out
# cat /tmp/top.out
目 标 ：
1 ， 跟 踪 cat 进 程 ， 找 出 它 sleep 的 原 因
2，继续跟踪锁的 owner，找到 lockfile 进程

q4> load struct proc from proc_list next p_factp max nproc
loaded 117 struct procs as a linked list (stopped by null pointer)
q4> keep p_pid==25002
kept 1 of 117 struct proc's, discarded 116
q4> print p_comm
p_comm
"cat"
q4> trace pile
stack trace for process at 0x0`43640040 (pid 25002), thread at 0x0`47c2b040
(tid
58447)
process was not running on any processor
_swtch+0xd0
_sleep+0x1cc
locked+0xd84
vx_rdwr+0x2b8
vno_rw+0x80
read+0x10c
syscall+0x768
$syscallrtn+0x0
q4> trace -u pile
......
LEVEL FUNC ARG0 ARG1

ARG2 ARG3 ARG4 ARG5 ARG6 ARG7

lev 0) _swtch+0xd0 0x0`000e93ec n/a

n/a n/a n/a n/a n/a n/a

lev 1) _sleep+0x1cc n/a 0x0`000002a8

n/a n/a n/a n/a n/a n/a

lev 2) locked+0xd84 0x0`00000001 0x0`4262d280

n/a n/a 0x0`00000001 0x0`00000000 0x0`00000000 n/a

lev 3) vx_rdwr+0x2b8 0x0`4262d280 0x400003ff`ffff0e08
0x0`0000
0001 0x0`00000001 0x0`42a3f3c0 n/a n/a n/a
lev 4) vno_rw+0x80 0x0`01f27598 0x0`00000001
0x400003ff`ffff
0e08 n/a n/a n/a 0x0`00621560 0x0`00ea40d0
lev 5) read+0x10c 0x400003ff`ffff03a0 n/a

n/a n/a n/a n/a n/a n/a

lev 6) syscall+0x768 n/a n/a

n/a n/a n/a n/a n/a n/a

lev 7) $syscallrtn+0x0 n/a n/a

n/a n/a n/a n/a n/a n/a

vno_rw(fp, rw, uiop)

struct file *fp;
enum uio_rw rw;
struct uio *uiop;
vx_rdwr(vnode*,...)
locked(?,vnode*,...)

q4> load struct vnode from 0x4262d280

loaded 1 struct vnode as an array (stopped by max count)
q4> print -tx
indexof 0
mapped 0x1
spaceof 0
addrof 0x4262d280
physaddrof 0x662d280
realmode 0
v_flag 0x8018
v_shlockc 0
v_exlockc 0
v_tcount 0
v_count 0x2
v_vfsmountedhere 0
v_op 0xf65230
v_socket 0
v_stream 0
v_vfsp 0x42461580
v_type VREG
v_rdev 0
v_data 0x425b5580
v_fstype VVXFS
v_vas 0
v_lock.b_lock 0
 v_lock.order 0x5a
v_lock.owner 0
v_cleanblkhd 0x47ca5f00
v_dirtyblkhd 0
v_writecount 0x1
v_locklist 0x1090af8
v_scount 0x1
v_nodeid 0x29
v_ncachedhd 0
v_ncachevhd 0x1f70690
v_pfdathd 0
v_bhash_origin 0

q4> load struct vx_inode from 0x425b5580

loaded 1 struct vx_inode as an array (stopped by max count)
q4> print -tx
......
i_dev 0x40000005
i_number 0x29
......

找到了! Major number 0x40=64，minor number 000005，即/dev/vg00/lvol5。"bdf"显示

是/tmp 文件系统。inode 节点号是 0x29=41，用"ls -i | grep 41"找到文件名是 top.out。

vnode 数据结构中包含了一个"v_locklist"字段，类型是 struct locklist*。

q4> load struct locklist from v_locklist

loaded 1 struct locklist as an array (stopped by max count)
q4> print -tx
indexof 0
mapped 0x1
spaceof 0
addrof 0x1090888
physaddrof 0x1090888
realmode 0
ll_link 0
ll_count 0x2
ll_flags 0x3
ll_proc 0x46e7d040
ll_kthreadp 0x4813b040
ll_start 0
ll_end 0x20000000000
ll_type 0x2
 ll_vp 0x426f2d00
ll_waitq 0
ll_fwd 0x1090888
ll_rev 0x1090888
ll_sib_fwd 0x1090888
ll_sib_rev 0x1090888

注意，struct locklist 中有一个字段 ll_proc 类型是 struct proc*。

q4> load struct proc from ll_proc

loaded 1 struct proc as an array (stopped by max count)
q4> print p_pid
p_pid
3602
q4> print p_comm
p_comm
"lockfile"

# ps -ef|grep 3602
root 3643 3633 3 17:35:46 ttyp4 0:00 grep 3602
root 3602 2021 0 17:26:23 ttyp3 0:00 ./lockfile /tmp/top.out

/*load 该 进 程 的 struct proc*/

...
q4> print p_highestfd
p_highestfd
3
/*文件描述符数量：3+1*/

q4> ex p_ofilep using L

0x46c7bdc0
q4> ex 0x46c7bdc0 for p_highestfd+1 using 8L
0x13463c8 0 0xc41a58 0 0x1f257b0 0x10000 0 0
0x13463c8 0 0xc41a58 0 0x1f257b0 0 0 0
0x13463c8 0 0xc41a58 0 0x1f257b0 0 0 0
0x13463c8 0 0xc41a58 0 0x1f27dd8 0 0 0
q4> load struct file from 0x1f27dd8
loaded 1 struct file as an array (stopped by max count)
/*完成*/
注：examine 的数据类型：
L for 64bit hex ， X for 32bit hex ， D for 32bit decimal ， s for string ， Y for
date，a for symbolic。Q4 online help：help ex。

实验：
(同前)
# lockfile /tmp/top.out
# cat /tmp/top.out

分析过程：

# ps -ef|grep cat
root 27616 26881 3 16:02:20 ttyp1 0:00 grep cat
root 26913 26888 0 10:35:34 ttyp3 0:00 cat /tmp/top.out

# q4 /stand/vmunix /dev/mem
...
/*找到 sleep 中的 cat 进程*/
q4> load struct proc from proc_list next p_factp max nproc
loaded 178 struct procs as a linked list (stopped by null pointer)
q4> keep p_pid==26913
kept 1 of 178 struct proc's, discarded 177
q4> print p_comm
p_comm
"cat"

q4> trace pile

stack trace for process at 0x0`46d8f040 (pid 26913), thread at 0x0`42238040
(tid
29832)
process was not running on any processor
_swtch+0xd0
_sleep+0x1cc
locked+0xd84
vx_rdwr+0x2b8
vno_rw+0x80
read+0x10c
syscall+0x768
$syscallrtn+0x0

/*找到线程*/
q4> load struct kthread from p_firstthreadp next kt_nextp max nkthread
loaded 1 struct kthread as a linked list (stopped by null pointer)
q4> print kt_tid
kt_tid
29832

/*线程的 wait channel*/

q4> print -tx kt_wchan /* 显 示 wchan 的 地 址 */
kt_wchan 0x1090888
q4> print kt_wchan %p /* 显 示 wchan 的 符 号 */
kt_wchan
locklist+0xd0

/*locklist 是 file lock 的系统变量，对应的 kernel 参数是 nflocks.

* 把 所 有 的 file lock 列 出 ： */
q4> load struct locklist from &locklist max nflocks
loaded 200 struct locklists as an array (stopped by max count)
/*ll_vp 是 vnode 指针，非空值表示该锁使用中；
* 或 者 用 ll_count>0 也 行 */
q4> keep ll_vp!=0
kept 9 of 200 struct locklist's, discarded 191
q4> keep addrof==0x1090888 /*wchan 的 地 址 */
kept 1 of 9 struct locklist's, discarded 8
q4> print -tx
indexof 0
mapped 0x1
spaceof 0
addrof 0x1090888
physaddrof 0x1090888
realmode 0
ll_link 0
ll_count 0x2
ll_flags 0x3
ll_proc 0x4673b040
ll_kthreadp 0x46f77040
ll_start 0
ll_end 0x20000000000
ll_type 0x2
ll_vp 0x45a0aa00
ll_waitq 0
ll_fwd 0x1090888
ll_rev 0x1090888
ll_sib_fwd 0x1090888
ll_sib_rev 0x1090888

/* 定 位 file lock 的 inode*/

q4> load struct vnode from ll_vp
loaded 1 struct vnode as an array (stopped by max count)
q4> print -tx
indexof 0
mapped 0x1
spaceof 0
 addrof 0x45a0aa00
physaddrof 0xaa0aa00
realmode 0
v_flag 0x8018
v_shlockc 0
v_exlockc 0
v_tcount 0
v_count 0x2
v_vfsmountedhere 0
v_op 0xf65230
v_socket 0
v_stream 0
v_vfsp 0x4244c040
v_type VREG
v_rdev 0
v_data 0x428b3ac0
v_fstype VVXFS
v_vas 0
v_lock.b_lock 0
v_lock.order 0x5a
v_lock.owner 0
v_cleanblkhd 0
v_dirtyblkhd 0
v_writecount 0x1
v_locklist 0x1090888
v_scount 0
v_nodeid 0x29
v_ncachedhd 0
v_ncachevhd 0
v_pfdathd 0
v_bhash_origin 0
q4> load struct vx_inode from v_data
loaded 1 struct vx_inode as an array (stopped by max count)
q4> print -tx i_dev i_number
i_dev 0x40000005
i_number 0x29
/*找到了：文件系统/dev/vg00/lvol5，inode 0x29*/

/*寻找拥有 file lock 的进程*/

q4> history
HIST NAME LAYOUT COUNT TYPE COMMENTS
1 <none> list 178 struct proc stopped by null pointer
2 <none> mixed? 1 struct proc subset of 1
3 <none> list 1 struct kthread stopped by null pointer
 4 <none> array 200 struct locklist stopped by max count
5 <none> mixed? 9 struct locklist subset of 4
6 <none> mixed? 1 struct locklist subset of 5
7 <none> array 1 struct vnode stopped by max count
8 <none> array 1 struct vx_inode stopped by max count
q4> recall 6
copied a pile
q4> print -tx
...
ll_proc 0x4673b040
ll_kthreadp 0x46f77040
...
q4> load struct proc from ll_proc
loaded 1 struct proc as an array (stopped by max count)
q4> print -tx p_pid p_comm
p_pid 0x6920
p_comm "lockfile"
/*找到了：进程号 0x6920，名字 lockfile*/

/*寻找线程*/
q4> history
HIST NAME LAYOUT COUNT TYPE COMMENTS
1 <none> list 178 struct proc stopped by null pointer
2 <none> mixed? 1 struct proc subset of 1
3 <none> list 1 struct kthread stopped by null pointer
4 <none> array 200 struct locklist stopped by max count
5 <none> mixed? 9 struct locklist subset of 4
6 <none> mixed? 1 struct locklist subset of 5
7 <none> array 1 struct vnode stopped by max count
8 <none> array 1 struct vx_inode stopped by max count
9 <none> mixed? 1 struct locklist copy of 6
10 <none> array 1 struct proc stopped by max count
11 <none> array 0 struct kthread stopped by null pointer
q4> forget 11
q4> forget 10
q4> load struct kthread from ll_kthreadp
loaded 1 struct kthread as an array (stopped by max count)
q4> print -tx kt_tid
kt_tid 0x7487
/*找到了：线程号 0x7487*/
# cat lswchan
#! /usr/bin/sh
ps -el | awk '{
mylen=length($11);
 if ( mylen==6 || mylen==7 )
{
print "0x" $11 "?r" | "adb64 -k /stand/vmunix /dev/mem | tail -1 > /tmp/t";
"cat /tmp/t" | getline s;
close("/tmp/t");
printf("%s\t%-16s%s\n", $4, $14, s);
}
else
{
printf("%s\t%-16s%s\n", $4, $14, $11);
}
}'
分享：

对此进程作一个 core dump，再检查 core file，关键字是 LANG、PATH、ARGS 等。

例：

# ps -ef|grep inetd
root 811 1 0 Sep 9 ? 1:53 /usr/sbin/inetd -l
root 4805 4306 2 11:35:56 pts/tf 0:00 grep inetd
# gdb /usr/sbin/inetd 811
...
(gdb) dumpcore
Dumping core to the core file core.811
(gdb) quit
...
# grep LANG core.811
...
^OM-+X^?^?@/usr/sbin/inetd-lLANG=CPATH=/sbin:/usr/sbin:/usr/binTZ=EAT-8NI
...