Linux jobworks 6.8.0-136-generic #136-Ubuntu SMP PREEMPT_DYNAMIC Wed Jul 1 21:53:05 UTC 2026 x86_64
Apache/2.4.58 (Ubuntu)
Server IP : 10.0.1.5 & Your IP : 216.73.217.52
Domains :
Cant Read [ /etc/named.conf ]
User : www-data
Terminal
Auto Root
Create File
Create Folder
Localroot Suggester
Backdoor Destroyer
Readme
/
usr /
share /
doc /
bpfcc-tools /
examples /
doc /
Delete
Unzip
Name
Size
Permission
Date
Action
lib
[ DIR ]
drwxr-xr-x
2025-11-05 21:34
argdist_example.txt
22.49
KB
-rw-r--r--
2023-12-08 15:36
bashreadline_example.txt
882
B
-rw-r--r--
2023-12-08 15:36
bindsnoop_example.txt
4.42
KB
-rw-r--r--
2023-12-08 15:36
biolatency_example.txt
23.46
KB
-rw-r--r--
2023-12-08 15:36
biolatpcts_example.txt
2.97
KB
-rw-r--r--
2023-12-08 15:36
biopattern_example.txt
1.37
KB
-rw-r--r--
2023-12-08 15:36
biosnoop_example.txt
3.47
KB
-rw-r--r--
2023-12-08 15:36
biotop_example.txt
9.11
KB
-rw-r--r--
2023-12-08 15:36
bitesize_example.txt
4.98
KB
-rw-r--r--
2023-12-08 15:36
bpflist_example.txt
2.13
KB
-rw-r--r--
2023-12-08 15:36
btrfsdist_example.txt
9.32
KB
-rw-r--r--
2023-12-08 15:36
btrfsslower_example.txt
6.65
KB
-rw-r--r--
2023-12-08 15:36
cachestat_example.txt
3.92
KB
-rw-r--r--
2023-12-08 15:36
cachetop_example.txt
3.83
KB
-rw-r--r--
2023-12-08 15:36
capable_example.txt
6.5
KB
-rw-r--r--
2023-12-08 15:36
cobjnew_example.txt
2.97
KB
-rw-r--r--
2023-12-08 15:36
compactsnoop_example.txt
9.92
KB
-rw-r--r--
2023-12-08 15:36
cpudist_example.txt
16.48
KB
-rw-r--r--
2023-12-08 15:36
cpuunclaimed_example.txt
15.2
KB
-rw-r--r--
2023-12-08 15:36
criticalstat_example.txt
4.81
KB
-rw-r--r--
2023-12-08 15:36
cthreads_example.txt
2.08
KB
-rw-r--r--
2023-12-08 15:36
dbslower_example.txt
3.89
KB
-rw-r--r--
2023-12-08 15:36
dbstat_example.txt
6.5
KB
-rw-r--r--
2023-12-08 15:36
dcsnoop_example.txt
4.27
KB
-rw-r--r--
2023-12-08 15:36
dcstat_example.txt
3.26
KB
-rw-r--r--
2023-12-08 15:36
deadlock_example.txt
16.25
KB
-rw-r--r--
2023-12-08 15:36
dirtop_example.txt
4.98
KB
-rw-r--r--
2023-12-08 15:36
drsnoop_example.txt
5
KB
-rw-r--r--
2023-12-08 15:36
execsnoop_example.txt
6.64
KB
-rw-r--r--
2023-12-08 15:36
exitsnoop_example.txt
6.22
KB
-rw-r--r--
2023-12-08 15:36
ext4dist_example.txt
8.78
KB
-rw-r--r--
2023-12-08 15:36
ext4slower_example.txt
11.07
KB
-rw-r--r--
2023-12-08 15:36
filegone_example.txt
743
B
-rw-r--r--
2023-12-08 15:36
filelife_example.txt
2.04
KB
-rw-r--r--
2023-12-08 15:36
fileslower_example.txt
5.58
KB
-rw-r--r--
2023-12-08 15:36
filetop_example.txt
6.8
KB
-rw-r--r--
2023-12-08 15:36
funccount_example.txt
13.29
KB
-rw-r--r--
2023-12-08 15:36
funcinterval_example.txt
15.28
KB
-rw-r--r--
2023-12-08 15:36
funclatency_example.txt
20.98
KB
-rw-r--r--
2023-12-08 15:36
funcslower_example.txt
6.63
KB
-rw-r--r--
2023-12-08 15:36
gethostlatency_example.txt
1.29
KB
-rw-r--r--
2023-12-08 15:36
hardirqs_example.txt
37.05
KB
-rw-r--r--
2023-12-08 15:36
inject_example.txt
6.67
KB
-rw-r--r--
2023-12-08 15:36
javacalls_example.txt
3.91
KB
-rw-r--r--
2023-12-08 15:36
javaflow_example.txt
5.88
KB
-rw-r--r--
2023-12-08 15:36
javagc_example.txt
3.78
KB
-rw-r--r--
2023-12-08 15:36
javaobjnew_example.txt
2.97
KB
-rw-r--r--
2023-12-08 15:36
javastat_example.txt
2.98
KB
-rw-r--r--
2023-12-08 15:36
javathreads_example.txt
2.08
KB
-rw-r--r--
2023-12-08 15:36
killsnoop_example.txt
1.31
KB
-rw-r--r--
2023-12-08 15:36
klockstat_example.txt
8.34
KB
-rw-r--r--
2023-12-08 15:36
kvmexit_example.txt
11.63
KB
-rw-r--r--
2023-12-08 15:36
llcstat_example.txt
3.24
KB
-rw-r--r--
2023-12-08 15:36
mdflush_example.txt
1.74
KB
-rw-r--r--
2023-12-08 15:36
memleak_example.txt
10.02
KB
-rw-r--r--
2023-12-08 15:36
mountsnoop_example.txt
1.45
KB
-rw-r--r--
2023-12-08 15:36
mysqld_qslower_example.txt
2.3
KB
-rw-r--r--
2023-12-08 15:36
netqtop_example.txt
12.2
KB
-rw-r--r--
2023-12-08 15:36
nfsdist_example.txt
8.31
KB
-rw-r--r--
2023-12-08 15:36
nfsslower_example.txt
7.68
KB
-rw-r--r--
2023-12-08 15:36
nodegc_example.txt
3.78
KB
-rw-r--r--
2023-12-08 15:36
nodestat_example.txt
2.98
KB
-rw-r--r--
2023-12-08 15:36
offcputime_example.txt
19.2
KB
-rw-r--r--
2023-12-08 15:36
offwaketime_example.txt
37.36
KB
-rw-r--r--
2023-12-08 15:36
oomkill_example.txt
1.88
KB
-rw-r--r--
2023-12-08 15:36
opensnoop_example.txt
10.33
KB
-rw-r--r--
2023-12-08 15:36
perlcalls_example.txt
3.91
KB
-rw-r--r--
2023-12-08 15:36
perlflow_example.txt
5.88
KB
-rw-r--r--
2023-12-08 15:36
perlstat_example.txt
2.98
KB
-rw-r--r--
2023-12-08 15:36
phpcalls_example.txt
3.91
KB
-rw-r--r--
2023-12-08 15:36
phpflow_example.txt
5.88
KB
-rw-r--r--
2023-12-08 15:36
phpstat_example.txt
2.98
KB
-rw-r--r--
2023-12-08 15:36
pidpersec_example.txt
677
B
-rw-r--r--
2023-12-08 15:36
ppchcalls_example.txt
6.93
KB
-rw-r--r--
2023-12-08 15:36
profile_example.txt
31.08
KB
-rw-r--r--
2023-12-08 15:36
pythoncalls_example.txt
3.91
KB
-rw-r--r--
2023-12-08 15:36
pythonflow_example.txt
5.88
KB
-rw-r--r--
2023-12-08 15:36
pythongc_example.txt
3.78
KB
-rw-r--r--
2023-12-08 15:36
pythonstat_example.txt
2.98
KB
-rw-r--r--
2023-12-08 15:36
rdmaucma_example.txt
1.94
KB
-rw-r--r--
2023-12-08 15:36
readahead_example.txt
3.17
KB
-rw-r--r--
2023-12-08 15:36
reset-trace_example.txt
9.15
KB
-rw-r--r--
2023-12-08 15:36
rubycalls_example.txt
3.91
KB
-rw-r--r--
2023-12-08 15:36
rubyflow_example.txt
5.88
KB
-rw-r--r--
2023-12-08 15:36
rubygc_example.txt
3.78
KB
-rw-r--r--
2023-12-08 15:36
rubyobjnew_example.txt
2.97
KB
-rw-r--r--
2023-12-08 15:36
rubystat_example.txt
2.98
KB
-rw-r--r--
2023-12-08 15:36
runqlat_example.txt
31.3
KB
-rw-r--r--
2023-12-08 15:36
runqlen_example.txt
11.85
KB
-rw-r--r--
2023-12-08 15:36
runqslower_example.txt
2.13
KB
-rw-r--r--
2023-12-08 15:36
shmsnoop_example.txt
2.73
KB
-rw-r--r--
2023-12-08 15:36
slabratetop_example.txt
5.22
KB
-rw-r--r--
2023-12-08 15:36
sofdsnoop_example.txt
3.14
KB
-rw-r--r--
2023-12-08 15:36
softirqs_example.txt
11.02
KB
-rw-r--r--
2023-12-08 15:36
solisten_example.txt
2.3
KB
-rw-r--r--
2023-12-08 15:36
sslsniff_example.txt
6.74
KB
-rw-r--r--
2023-12-08 15:36
stackcount_example.txt
21.45
KB
-rw-r--r--
2023-12-08 15:36
statsnoop_example.txt
3.02
KB
-rw-r--r--
2023-12-08 15:36
swapin.txt
2.57
KB
-rw-r--r--
2023-12-08 15:36
swapin_example.txt
1.39
KB
-rw-r--r--
2023-12-08 15:36
syncsnoop_example.txt
387
B
-rw-r--r--
2023-12-08 15:36
syscount_example.txt
6.27
KB
-rw-r--r--
2023-12-08 15:36
tclcalls_example.txt
3.91
KB
-rw-r--r--
2023-12-08 15:36
tclflow_example.txt
5.88
KB
-rw-r--r--
2023-12-08 15:36
tclobjnew_example.txt
2.97
KB
-rw-r--r--
2023-12-08 15:36
tclstat_example.txt
2.98
KB
-rw-r--r--
2023-12-08 15:36
tcpaccept_example.txt
2.76
KB
-rw-r--r--
2023-12-08 15:36
tcpcong_example.txt
33.31
KB
-rw-r--r--
2023-12-08 15:36
tcpconnect_example.txt
6.27
KB
-rw-r--r--
2023-12-08 15:36
tcpconnlat_example.txt
2.55
KB
-rw-r--r--
2023-12-08 15:36
tcpdrop_example.txt
1.95
KB
-rw-r--r--
2023-12-08 15:36
tcplife_example.txt
6.83
KB
-rw-r--r--
2023-12-08 15:36
tcpretrans_example.txt
3.85
KB
-rw-r--r--
2023-12-08 15:36
tcprtt_example.txt
9.83
KB
-rw-r--r--
2023-12-08 15:36
tcpstates_example.txt
2.84
KB
-rw-r--r--
2023-12-08 15:36
tcpsubnet_example.txt
5.37
KB
-rw-r--r--
2023-12-08 15:36
tcpsynbl_example.txt
1.15
KB
-rw-r--r--
2023-12-08 15:36
tcptop_example.txt
5.75
KB
-rw-r--r--
2023-12-08 15:36
tcptracer_example.txt
1.98
KB
-rw-r--r--
2023-12-08 15:36
threadsnoop_example.txt
1.07
KB
-rw-r--r--
2023-12-08 15:36
tplist_example.txt
4.4
KB
-rw-r--r--
2023-12-08 15:36
trace_example.txt
21.62
KB
-rw-r--r--
2023-12-08 15:36
ttysnoop_example.txt
3.24
KB
-rw-r--r--
2023-12-08 15:36
vfscount_example.txt
2.17
KB
-rw-r--r--
2023-12-08 15:36
vfsstat_example.txt
1.66
KB
-rw-r--r--
2023-12-08 15:36
virtiostat_example.txt
2.62
KB
-rw-r--r--
2023-12-08 15:36
wakeuptime_example.txt
33.25
KB
-rw-r--r--
2023-12-08 15:36
xfsdist_example.txt
6.77
KB
-rw-r--r--
2023-12-08 15:36
xfsslower_example.txt
6.91
KB
-rw-r--r--
2023-12-08 15:36
zfsdist_example.txt
9.52
KB
-rw-r--r--
2023-12-08 15:36
zfsslower_example.txt
7.37
KB
-rw-r--r--
2023-12-08 15:36
Save
Rename
Demonstrations of runqlat, the Linux eBPF/bcc version. This program summarizes scheduler run queue latency as a histogram, showing how long tasks spent waiting their turn to run on-CPU. Here is a heavily loaded system: # ./runqlat Tracing run queue latency... Hit Ctrl-C to end. ^C usecs : count distribution 0 -> 1 : 233 |*********** | 2 -> 3 : 742 |************************************ | 4 -> 7 : 203 |********** | 8 -> 15 : 173 |******** | 16 -> 31 : 24 |* | 32 -> 63 : 0 | | 64 -> 127 : 30 |* | 128 -> 255 : 6 | | 256 -> 511 : 3 | | 512 -> 1023 : 5 | | 1024 -> 2047 : 27 |* | 2048 -> 4095 : 30 |* | 4096 -> 8191 : 20 | | 8192 -> 16383 : 29 |* | 16384 -> 32767 : 809 |****************************************| 32768 -> 65535 : 64 |*** | The distribution is bimodal, with one mode between 0 and 15 microseconds, and another between 16 and 65 milliseconds. These modes are visible as the spikes in the ASCII distribution (which is merely a visual representation of the "count" column). As an example of reading one line: 809 events fell into the 16384 to 32767 microsecond range (16 to 32 ms) while tracing. I would expect the two modes to be due the workload: 16 hot CPU-bound threads, and many other mostly idle threads doing occasional work. I suspect the mostly idle threads will run with a higher priority when they wake up, and are the reason for the low latency mode. The high latency mode will be the CPU-bound threads. More analysis with this and other tools can confirm. A -m option can be used to show milliseconds instead, as well as an interval and a count. For example, showing three x five second summary in milliseconds: # ./runqlat -m 5 3 Tracing run queue latency... Hit Ctrl-C to end. msecs : count distribution 0 -> 1 : 3818 |****************************************| 2 -> 3 : 39 | | 4 -> 7 : 39 | | 8 -> 15 : 62 | | 16 -> 31 : 2214 |*********************** | 32 -> 63 : 226 |** | msecs : count distribution 0 -> 1 : 3775 |****************************************| 2 -> 3 : 52 | | 4 -> 7 : 37 | | 8 -> 15 : 65 | | 16 -> 31 : 2230 |*********************** | 32 -> 63 : 212 |** | msecs : count distribution 0 -> 1 : 3816 |****************************************| 2 -> 3 : 49 | | 4 -> 7 : 40 | | 8 -> 15 : 53 | | 16 -> 31 : 2228 |*********************** | 32 -> 63 : 221 |** | This shows a similar distribution across the three summaries. A -p option can be used to show one PID only, which is filtered in kernel for efficiency. For example, PID 4505, and one second summaries: # ./runqlat -mp 4505 1 Tracing run queue latency... Hit Ctrl-C to end. msecs : count distribution 0 -> 1 : 1 |* | 2 -> 3 : 2 |*** | 4 -> 7 : 1 |* | 8 -> 15 : 0 | | 16 -> 31 : 25 |****************************************| 32 -> 63 : 3 |**** | msecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 2 |** | 4 -> 7 : 0 | | 8 -> 15 : 1 |* | 16 -> 31 : 30 |****************************************| 32 -> 63 : 1 |* | msecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 28 |****************************************| 32 -> 63 : 2 |** | msecs : count distribution 0 -> 1 : 1 |* | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 27 |****************************************| 32 -> 63 : 4 |***** | [...] For comparison, here is pidstat(1) for that process: # pidstat -p 4505 1 Linux 4.4.0-virtual (bgregg-xxxxxxxx) 02/08/2016 _x86_64_ (8 CPU) 08:56:11 AM UID PID %usr %system %guest %CPU CPU Command 08:56:12 AM 0 4505 9.00 3.00 0.00 12.00 0 bash 08:56:13 AM 0 4505 7.00 5.00 0.00 12.00 0 bash 08:56:14 AM 0 4505 10.00 2.00 0.00 12.00 0 bash 08:56:15 AM 0 4505 11.00 2.00 0.00 13.00 0 bash 08:56:16 AM 0 4505 9.00 3.00 0.00 12.00 0 bash [...] This is a synthetic workload that is CPU bound. It's only spending 12% on-CPU each second because of high CPU demand on this server: the remaining time is spent waiting on a run queue, as visualized by runqlat. Here is the same system, but when it is CPU idle: # ./runqlat 5 1 Tracing run queue latency... Hit Ctrl-C to end. usecs : count distribution 0 -> 1 : 2250 |******************************** | 2 -> 3 : 2340 |********************************** | 4 -> 7 : 2746 |****************************************| 8 -> 15 : 418 |****** | 16 -> 31 : 93 |* | 32 -> 63 : 28 | | 64 -> 127 : 119 |* | 128 -> 255 : 9 | | 256 -> 511 : 4 | | 512 -> 1023 : 20 | | 1024 -> 2047 : 22 | | 2048 -> 4095 : 5 | | 4096 -> 8191 : 2 | | Back to a microsecond scale, this time there is little run queue latency past 1 millisecond, as would be expected. Now 16 threads are performing heavy disk I/O: # ./runqlat 5 1 Tracing run queue latency... Hit Ctrl-C to end. usecs : count distribution 0 -> 1 : 204 | | 2 -> 3 : 944 |* | 4 -> 7 : 16315 |********************* | 8 -> 15 : 29897 |****************************************| 16 -> 31 : 1044 |* | 32 -> 63 : 23 | | 64 -> 127 : 128 | | 128 -> 255 : 24 | | 256 -> 511 : 5 | | 512 -> 1023 : 13 | | 1024 -> 2047 : 15 | | 2048 -> 4095 : 13 | | 4096 -> 8191 : 10 | | The distribution hasn't changed too much. While the disks are 100% busy, there is still plenty of CPU headroom, and threads still don't spend much time waiting their turn. A -P option will print a distribution for each PID: # ./runqlat -P Tracing run queue latency... Hit Ctrl-C to end. ^C pid = 0 usecs : count distribution 0 -> 1 : 351 |******************************** | 2 -> 3 : 96 |******** | 4 -> 7 : 437 |****************************************| 8 -> 15 : 12 |* | 16 -> 31 : 10 | | 32 -> 63 : 0 | | 64 -> 127 : 16 |* | 128 -> 255 : 0 | | 256 -> 511 : 0 | | 512 -> 1023 : 0 | | 1024 -> 2047 : 0 | | 2048 -> 4095 : 0 | | 4096 -> 8191 : 0 | | 8192 -> 16383 : 1 | | pid = 12929 usecs : count distribution 0 -> 1 : 1 |****************************************| 2 -> 3 : 0 | | 4 -> 7 : 1 |****************************************| pid = 12930 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 1 |****************************************| 32 -> 63 : 0 | | 64 -> 127 : 1 |****************************************| pid = 12931 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 1 |******************** | 8 -> 15 : 0 | | 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 0 | | 256 -> 511 : 0 | | 512 -> 1023 : 2 |****************************************| pid = 12932 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 1 |****************************************| 256 -> 511 : 0 | | 512 -> 1023 : 1 |****************************************| pid = 7 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 426 |************************************* | 4 -> 7 : 457 |****************************************| 8 -> 15 : 16 |* | pid = 9 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 425 |****************************************| 8 -> 15 : 16 |* | pid = 11 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 10 |****************************************| pid = 14 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 8 |****************************************| 4 -> 7 : 2 |********** | pid = 18 usecs : count distribution 0 -> 1 : 414 |****************************************| 2 -> 3 : 0 | | 4 -> 7 : 20 |* | 8 -> 15 : 8 | | pid = 12928 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 1 |****************************************| 8 -> 15 : 0 | | 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 1 |****************************************| pid = 1867 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 15 |****************************************| 16 -> 31 : 1 |** | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 4 |********** | pid = 1871 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 2 |****************************************| 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 0 | | 256 -> 511 : 0 | | 512 -> 1023 : 1 |******************** | pid = 1876 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 1 |****************************************| 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 0 | | 256 -> 511 : 1 |****************************************| pid = 1878 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 3 |****************************************| pid = 1880 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 3 |****************************************| pid = 9307 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 1 |****************************************| pid = 1886 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 1 |******************** | 8 -> 15 : 2 |****************************************| pid = 1888 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 3 |****************************************| pid = 3297 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 1 |****************************************| pid = 1892 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 1 |******************** | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 0 | | 256 -> 511 : 0 | | 512 -> 1023 : 2 |****************************************| pid = 7024 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 4 |****************************************| pid = 16468 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 3 |****************************************| pid = 12922 usecs : count distribution 0 -> 1 : 1 |****************************************| 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 1 |****************************************| 16 -> 31 : 1 |****************************************| 32 -> 63 : 0 | | 64 -> 127 : 1 |****************************************| pid = 12923 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 1 |******************** | 8 -> 15 : 0 | | 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 2 |****************************************| 128 -> 255 : 0 | | 256 -> 511 : 0 | | 512 -> 1023 : 1 |******************** | 1024 -> 2047 : 1 |******************** | pid = 12924 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 2 |******************** | 8 -> 15 : 4 |****************************************| 16 -> 31 : 1 |********** | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 0 | | 256 -> 511 : 0 | | 512 -> 1023 : 0 | | 1024 -> 2047 : 1 |********** | pid = 12925 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 1 |****************************************| pid = 12926 usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 1 |****************************************| 4 -> 7 : 0 | | 8 -> 15 : 1 |****************************************| 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 0 | | 128 -> 255 : 0 | | 256 -> 511 : 0 | | 512 -> 1023 : 1 |****************************************| pid = 12927 usecs : count distribution 0 -> 1 : 1 |****************************************| 2 -> 3 : 0 | | 4 -> 7 : 1 |****************************************| A -L option will print a distribution for each TID: # ./runqlat -L Tracing run queue latency... Hit Ctrl-C to end. ^C tid = 0 usecs : count distribution 0 -> 1 : 593 |**************************** | 2 -> 3 : 829 |****************************************| 4 -> 7 : 300 |************** | 8 -> 15 : 321 |*************** | 16 -> 31 : 132 |****** | 32 -> 63 : 58 |** | 64 -> 127 : 0 | | 128 -> 255 : 0 | | 256 -> 511 : 13 | | tid = 7 usecs : count distribution 0 -> 1 : 8 |******** | 2 -> 3 : 19 |******************** | 4 -> 7 : 37 |****************************************| [...] And a --pidnss option (short for PID namespaces) will print for each PID namespace, for analyzing container performance: # ./runqlat --pidnss -m Tracing run queue latency... Hit Ctrl-C to end. ^C pidns = 4026532870 msecs : count distribution 0 -> 1 : 40 |****************************************| 2 -> 3 : 1 |* | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 0 | | 32 -> 63 : 2 |** | 64 -> 127 : 5 |***** | pidns = 4026532809 msecs : count distribution 0 -> 1 : 67 |****************************************| pidns = 4026532748 msecs : count distribution 0 -> 1 : 63 |****************************************| pidns = 4026532687 msecs : count distribution 0 -> 1 : 7 |****************************************| pidns = 4026532626 msecs : count distribution 0 -> 1 : 45 |****************************************| 2 -> 3 : 0 | | 4 -> 7 : 0 | | 8 -> 15 : 0 | | 16 -> 31 : 0 | | 32 -> 63 : 0 | | 64 -> 127 : 3 |** | pidns = 4026531836 msecs : count distribution 0 -> 1 : 314 |****************************************| 2 -> 3 : 1 | | 4 -> 7 : 11 |* | 8 -> 15 : 28 |*** | 16 -> 31 : 137 |***************** | 32 -> 63 : 86 |********** | 64 -> 127 : 1 | | pidns = 4026532382 msecs : count distribution 0 -> 1 : 285 |****************************************| 2 -> 3 : 5 | | 4 -> 7 : 16 |** | 8 -> 15 : 9 |* | 16 -> 31 : 69 |********* | 32 -> 63 : 25 |*** | Many of these distributions have two modes: the second, in this case, is caused by capping CPU usage via CPU shares. USAGE message: # ./runqlat -h usage: runqlat.py [-h] [-T] [-m] [-P] [--pidnss] [-L] [-p PID] [interval] [count] Summarize run queue (scheduler) latency as a histogram positional arguments: interval output interval, in seconds count number of outputs optional arguments: -h, --help show this help message and exit -T, --timestamp include timestamp on output -m, --milliseconds millisecond histogram -P, --pids print a histogram per process ID --pidnss print a histogram per PID namespace -L, --tids print a histogram per thread ID -p PID, --pid PID trace this PID only examples: ./runqlat # summarize run queue latency as a histogram ./runqlat 1 10 # print 1 second summaries, 10 times ./runqlat -mT 1 # 1s summaries, milliseconds, and timestamps ./runqlat -P # show each PID separately ./runqlat -p 185 # trace PID 185 only