/usr/sbin
NameSizeModeActions
aa-load396800755editdlrm
aa-remove-unknown32250755editdlrm
aa-status400000755editdlrm
aa-teardown1370755editdlrm
accessdb149040755editdlrm
acpid539200755editdlrm
add-shell10530755editdlrm
addgnupghome30750755editdlrm
addgroup551910755editdlrm
adduser551910755editdlrm
agetty609920755editdlrm
apparmor_parser16298480755editdlrm
apparmor_status400000755editdlrm
applygnupgdefaults22170755editdlrm
argdist-bpfcc368620755editdlrm
arpd269600755editdlrm
arptables2244240755editdlrm
arptables-nft2244240755editdlrm
arptables-nft-restore2244240755editdlrm
arptables-nft-save2244240755editdlrm
arptables-restore2244240755editdlrm
arptables-save2244240755editdlrm
badblocks351440755editdlrm
bashreadline-bpfcc23800755editdlrm
bashreadline.bt6980755editdlrm
bcache-super-show146480755editdlrm
bindsnoop-bpfcc163460755editdlrm
biolatency-bpfcc113650755editdlrm
biolatency-kp.bt6640755editdlrm
biolatency.bt6810755editdlrm
biolatpcts-bpfcc102480755editdlrm
biopattern-bpfcc39570755editdlrm
biosdecode278560755editdlrm
biosnoop-bpfcc108330755editdlrm
biosnoop.bt11480755editdlrm
biostacks.bt9150755editdlrm
biotop-bpfcc95670755editdlrm
bitesize-bpfcc11660755editdlrm
bitesize.bt5670755editdlrm
blkdeactivate163510755editdlrm
blkdiscard229120755editdlrm
blkid557200755editdlrm
blkzone352000755editdlrm
blockdev352000755editdlrm
bpflist-bpfcc26010755editdlrm
bpftool16220755editdlrm
bridge1110960755editdlrm
btrfsdist-bpfcc66270755editdlrm
btrfsslower-bpfcc99850755editdlrm
cachestat-bpfcc65310755editdlrm
cachetop-bpfcc93670755editdlrm
cache_check14305680755editdlrm
cache_dump14305680755editdlrm
cache_metadata_size14305680755editdlrm
cache_repair14305680755editdlrm
cache_restore14305680755editdlrm
cache_writeback14305680755editdlrm
capable-bpfcc84770755editdlrm
capable.bt19260755editdlrm
capsh584560755editdlrm
cfdisk970080755editdlrm
cgdisk1704800755editdlrm
chcpu311040755editdlrm
chgpasswd597200755editdlrm
chmem352000755editdlrm
chpasswd557360755editdlrm
chronyd3062320755editdlrm
chroot394320755editdlrm
cobjnew-bpfcc530755editdlrm
compactsnoop-bpfcc113690755editdlrm
cpgr496080755editdlrm
cppw496080755editdlrm
cpudist-bpfcc70130755editdlrm
cpuunclaimed-bpfcc149380755editdlrm
cpuwalk.bt4970755editdlrm
criticalstat-bpfcc86070755editdlrm
cron600800755editdlrm
cryptdisks_start15440755editdlrm
cryptdisks_stop8440755editdlrm
cryptsetup2313200755editdlrm
ctrlaltdel147200755editdlrm
dbslower-bpfcc73910755editdlrm
dbstat-bpfcc37900755editdlrm
dcb824480755editdlrm
dcsnoop-bpfcc41260755editdlrm
dcsnoop.bt12610755editdlrm
dcstat-bpfcc38590755editdlrm
deadlock-bpfcc209430755editdlrm
debugfs2312880755editdlrm
delgroup189770755editdlrm
deluser189770755editdlrm
depmod1743280755editdlrm
devlink1544800755editdlrm
dhcpcd4048880755editdlrm
dirtop-bpfcc85680755editdlrm
dmeventd515920755editdlrm
dmidecode1384960755editdlrm
dmsetup1751600755editdlrm
dmstats1751600755editdlrm
dosfsck802640755editdlrm
dosfslabel393040755editdlrm
dpkg-preconfigure43560755editdlrm
dpkg-reconfigure45360755editdlrm
drsnoop-bpfcc68890755editdlrm
dumpe2fs351360755editdlrm
e2freefrag187440755editdlrm
e2fsck3730800755editdlrm
e2image433280755editdlrm
e2label1132160755editdlrm
e2mmpstatus351360755editdlrm
e2scrub72940755editdlrm
e2scrub_all53940755editdlrm
e2undo228400755editdlrm
e4crypt311040755editdlrm
e4defrag351280755editdlrm
ebtables2244240755editdlrm
ebtables-nft2244240755editdlrm
ebtables-nft-restore2244240755editdlrm
ebtables-nft-save2244240755editdlrm
ebtables-restore2244240755editdlrm
ebtables-save2244240755editdlrm
ebtables-translate2244240755editdlrm
era_check14305680755editdlrm
era_dump14305680755editdlrm
era_invalidate14305680755editdlrm
era_restore14305680755editdlrm
ethtool6673200755editdlrm
execsnoop-bpfcc100590755editdlrm
execsnoop.bt9280755editdlrm
exitsnoop-bpfcc96460755editdlrm
ext4dist-bpfcc66830755editdlrm
ext4slower-bpfcc99480755editdlrm
faillock228480755editdlrm
fatlabel393040755editdlrm
fdisk1171680755editdlrm
filefrag187600755editdlrm
filegone-bpfcc57760755editdlrm
filelife-bpfcc65280755editdlrm
fileslower-bpfcc73720755editdlrm
filetop-bpfcc64990755editdlrm
findfs147200755editdlrm
fixparts598800755editdlrm
fsadm245800755editdlrm
fsck434400755editdlrm
fsck.btrfs11850755editdlrm
fsck.cramfs311680755editdlrm
fsck.ext23730800755editdlrm
fsck.ext33730800755editdlrm
fsck.ext43730800755editdlrm
fsck.fat802640755editdlrm
fsck.minix557120755editdlrm
fsck.msdos802640755editdlrm
fsck.vfat802640755editdlrm
fsck.xfs25660755editdlrm
fsfreeze147200755editdlrm
fstab-decode146480755editdlrm
fstrim433920755editdlrm
funccount-bpfcc129810755editdlrm
funcinterval-bpfcc55900755editdlrm
funclatency-bpfcc115490755editdlrm
funcslower-bpfcc106300755editdlrm
gdisk2032480755editdlrm
genl1234720755editdlrm
getcap146480755editdlrm
gethostlatency-bpfcc39140755editdlrm
gethostlatency.bt12220755editdlrm
getpcaps146480755editdlrm
getty609920755editdlrm
groupadd728400755editdlrm
groupdel644240755editdlrm
groupmems597440755editdlrm
groupmod727440755editdlrm
grpck597200755editdlrm
grpconv513680755editdlrm
grpunconv513680755editdlrm
grub-bios-setup9815520755editdlrm
grub-install12258240755editdlrm
grub-macbless9691280755editdlrm
grub-mkconfig88400755editdlrm
grub-mkdevicemap723840755editdlrm
grub-probe9775760755editdlrm
grub-reboot48410755editdlrm
grub-set-default35560755editdlrm
halt15013040755editdlrm
hardirqs-bpfcc70190755editdlrm
hdparm1427760755editdlrm
iconvconfig352960755editdlrm
init1008160755editdlrm
inject-bpfcc164450755editdlrm
insmod1743280755editdlrm
install-sgmlcatalog45510755editdlrm
installkernel26590755editdlrm
integritysetup686720755editdlrm
invoke-rc.d165170755editdlrm
ip7729200755editdlrm
ip6tables2244240755editdlrm
ip6tables-apply70520755editdlrm
ip6tables-legacy951760755editdlrm
ip6tables-legacy-restore951760755editdlrm
ip6tables-legacy-save951760755editdlrm
ip6tables-nft2244240755editdlrm
ip6tables-nft-restore2244240755editdlrm
ip6tables-nft-save2244240755editdlrm
ip6tables-restore2244240755editdlrm
ip6tables-restore-translate2244240755editdlrm
ip6tables-save2244240755editdlrm
ip6tables-translate2244240755editdlrm
iptables2244240755editdlrm
iptables-apply70520755editdlrm
iptables-legacy951760755editdlrm
iptables-legacy-restore951760755editdlrm
iptables-legacy-save951760755editdlrm
iptables-nft2244240755editdlrm
iptables-nft-restore2244240755editdlrm
iptables-nft-save2244240755editdlrm
iptables-restore2244240755editdlrm
iptables-restore-translate2244240755editdlrm
iptables-save2244240755editdlrm
iptables-translate2244240755editdlrm
irqbalance602720755editdlrm
irqbalance-ui393040755editdlrm
iscsi-iname187440755editdlrm
iscsiadm3793200755editdlrm
iscsid2934320755editdlrm
iscsistart2810800755editdlrm
iscsi_discovery52930755editdlrm
isosize147200755editdlrm
iucode-tool556400755editdlrm
iucode_tool556400755editdlrm
javacalls-bpfcc550755editdlrm
javaflow-bpfcc540755editdlrm
javagc-bpfcc520755editdlrm
javaobjnew-bpfcc560755editdlrm
javastat-bpfcc540755editdlrm
javathreads-bpfcc570755editdlrm
kbdrate187520755editdlrm
killall5268560755editdlrm
killsnoop-bpfcc45540755editdlrm
killsnoop.bt8730755editdlrm
klockstat-bpfcc133560755editdlrm
kpartx431760755editdlrm
kvmexit-bpfcc114610755editdlrm
ldattach270080755editdlrm
ldconfig3870755editdlrm
ldconfig.real10512800755editdlrm
llcstat-bpfcc45840755editdlrm
loads.bt11270755editdlrm
locale-gen43160755editdlrm
logrotate965040755editdlrm
logsave144960755editdlrm
losetup763040755editdlrm
lsmod1743280755editdlrm
luksformat34010755editdlrm
lvchange31567120755editdlrm
lvconvert31567120755editdlrm
lvcreate31567120755editdlrm
lvdisplay31567120755editdlrm
lvextend31567120755editdlrm
lvm31567120755editdlrm
lvmconfig31567120755editdlrm
lvmdiskscan31567120755editdlrm
lvmdump103640755editdlrm
lvmpolld2416320755editdlrm
lvmsadc31567120755editdlrm
lvmsar31567120755editdlrm
lvreduce31567120755editdlrm
lvremove31567120755editdlrm
lvrename31567120755editdlrm
lvresize31567120755editdlrm
lvs31567120755editdlrm
lvscan31567120755editdlrm
lxc5890755editdlrm
lxd5890755editdlrm
make-bcache229120755editdlrm
mdadm6371440755editdlrm
mdflush-bpfcc22950755editdlrm
mdflush.bt7750755editdlrm
mdmon2650160755editdlrm
memleak-bpfcc212980755editdlrm
mkdosfs520480755editdlrm
mke2fs1337520755editdlrm
mkfs147200755editdlrm
mkfs.bfs229120755editdlrm
mkfs.btrfs5737520755editdlrm
mkfs.cramfs351440755editdlrm
mkfs.ext21337520755editdlrm
mkfs.ext31337520755editdlrm
mkfs.ext41337520755editdlrm
mkfs.fat520480755editdlrm
mkfs.minix434080755editdlrm
mkfs.msdos520480755editdlrm
mkfs.ntfs679760755editdlrm
mkfs.vfat520480755editdlrm
mkfs.xfs4495280755editdlrm
mkhomedir_helper228720755editdlrm
mkinitramfs157620755editdlrm
mklost+found146480755editdlrm
mkntfs679760755editdlrm
mkswap515920755editdlrm
ModemManager21655760755editdlrm
modinfo1743280755editdlrm
modprobe1743280755editdlrm
mount.fuse187360755editdlrm
mount.fuse3187360755editdlrm
mount.lowntfs-3g1218320755editdlrm
mount.ntfs1628240755editdlrm
mount.ntfs-3g1628240755editdlrm
mountsnoop-bpfcc149680755editdlrm
mpathpersist319600755editdlrm
multipath351280755editdlrm
multipathc187440755editdlrm
multipathd1458800755editdlrm
mysqld554679440755editdlrm
mysqld_qslower-bpfcc31250755editdlrm
naptime.bt10350755editdlrm
needrestart410920755editdlrm
netplan8020755editdlrm
netqtop-bpfcc57250755editdlrm
newusers890480755editdlrm
nfnl_osf187360755editdlrm
nfsdist-bpfcc50680755editdlrm
nfsslower-bpfcc139390755editdlrm
nft268560755editdlrm
nginx13141680755editdlrm
nodegc-bpfcc520755editdlrm
nodestat-bpfcc540755editdlrm
nologin146400755editdlrm
ntfsclone515920755editdlrm
ntfscp311040755editdlrm
ntfslabel229120755editdlrm
ntfsresize638880755editdlrm
ntfsundelete515920755editdlrm
offcputime-bpfcc137790755editdlrm
offwaketime-bpfcc156760755editdlrm
on_ac_power37880755editdlrm
oomkill-bpfcc20840755editdlrm
oomkill.bt11980755editdlrm
opensnoop-bpfcc145810755editdlrm
opensnoop.bt9530755editdlrm
overlayroot-chroot25100755editdlrm
ownership147920755editdlrm
pam-auth-update214650755editdlrm
pam_extrausers_chkpwd269442755editdlrm
pam_extrausers_update351360755editdlrm
pam_getenv28900755editdlrm
pam_namespace_helper4670755editdlrm
pam_timestamp_check146560755editdlrm
parted966640755editdlrm
partprobe147200755editdlrm
pdata_tools14305680755editdlrm
perlcalls-bpfcc550755editdlrm
perlflow-bpfcc540755editdlrm
perlstat-bpfcc540755editdlrm
php-fpm7.448033760755editdlrm
phpcalls-bpfcc540755editdlrm
phpdismod72780755editdlrm
phpenmod72780755editdlrm
phpflow-bpfcc530755editdlrm
phpquery63890755editdlrm
phpstat-bpfcc530755editdlrm
pidpersec-bpfcc11110755editdlrm
pidpersec.bt6280755editdlrm
pivot_root147200755editdlrm
plymouthd1500880755editdlrm
poweroff15013040755editdlrm
ppchcalls-bpfcc142240755editdlrm
profile-bpfcc147560755editdlrm
pvchange31567120755editdlrm
pvck31567120755editdlrm
pvcreate31567120755editdlrm
pvdisplay31567120755editdlrm
pvmove31567120755editdlrm
pvremove31567120755editdlrm
pvresize31567120755editdlrm
pvs31567120755editdlrm
pvscan31567120755editdlrm
pwck555920755editdlrm
pwconv472720755editdlrm
pwhistory_helper228480755editdlrm
pwunconv472720755editdlrm
pythoncalls-bpfcc570755editdlrm
pythonflow-bpfcc560755editdlrm
pythongc-bpfcc540755editdlrm
pythonstat-bpfcc560755editdlrm
rdmaucma-bpfcc50730755editdlrm
readahead-bpfcc66940755editdlrm
readprofile229440755editdlrm
reboot15013040755editdlrm
remove-shell11110755editdlrm
reset-trace-bpfcc34980755editdlrm
resize2fs719920755editdlrm
resolvconf1624800755editdlrm
rmmod1743280755editdlrm
rmt560240755editdlrm
rmt-tar560240755editdlrm
rsyslogd7901920755editdlrm
rtacct289920755editdlrm
rtcwake352000755editdlrm
rtmon1193120755editdlrm
rubycalls-bpfcc550755editdlrm
rubyflow-bpfcc540755editdlrm
rubygc-bpfcc520755editdlrm
rubyobjnew-bpfcc560755editdlrm
rubystat-bpfcc540755editdlrm
runlevel15013040755editdlrm
runqlat-bpfcc95260755editdlrm
runqlat.bt7880755editdlrm
runqlen-bpfcc82480755editdlrm
runqlen.bt10370755editdlrm
runqslower-bpfcc92250755editdlrm
runuser556800755editdlrm
service91040755editdlrm
setcap146480755editdlrm
setuids.bt17990755editdlrm
setvesablank147120755editdlrm
setvtrgb147760755editdlrm
sfdisk1089280755editdlrm
sgdisk1827680755editdlrm
shadowconfig22730755editdlrm
shmsnoop-bpfcc79850755editdlrm
shutdown15013040755editdlrm
slabratetop-bpfcc65370755editdlrm
sofdsnoop-bpfcc82550755editdlrm
softirqs-bpfcc57230755editdlrm
solisten-bpfcc61030755editdlrm
sshd9214160755editdlrm
ssllatency.bt21330755editdlrm
sslsniff-bpfcc140130755editdlrm
sslsnoop.bt20410755editdlrm
stackcount-bpfcc166550755editdlrm
start-stop-daemon486320755editdlrm
statsnoop-bpfcc50410755editdlrm
statsnoop.bt12940755editdlrm
sudo_logsrvd2544640755editdlrm
sudo_sendlog1348320755editdlrm
sulogin433920755editdlrm
swapin.bt6000755editdlrm
swaplabel188160755editdlrm
swapoff229120755editdlrm
swapon433920755editdlrm
switch_root229120755editdlrm
syncsnoop-bpfcc13000755editdlrm
syncsnoop.bt8390755editdlrm
syscount-bpfcc87710755editdlrm
syscount.bt8720755editdlrm
sysctl311120755editdlrm
tarcat9360755editdlrm
tc6452000755editdlrm
tclcalls-bpfcc540755editdlrm
tclflow-bpfcc530755editdlrm
tclobjnew-bpfcc550755editdlrm
tclstat-bpfcc530755editdlrm
tcpaccept-bpfcc92200755editdlrm
tcpaccept.bt17560755editdlrm
tcpcong-bpfcc205910755editdlrm
tcpconnect-bpfcc189020755editdlrm
tcpconnect.bt16150755editdlrm
tcpconnlat-bpfcc92850755editdlrm
tcpdrop-bpfcc76160755editdlrm
tcpdrop.bt24690755editdlrm
tcplife-bpfcc169490755editdlrm
tcplife.bt27850755editdlrm
tcpretrans-bpfcc140980755editdlrm
tcpretrans.bt21200755editdlrm
tcprtt-bpfcc89130755editdlrm
tcpstates-bpfcc140640755editdlrm
tcpsubnet-bpfcc78140755editdlrm
tcpsynbl-bpfcc21750755editdlrm
tcpsynbl.bt9620755editdlrm
tcptop-bpfcc129430755editdlrm
tcptracer-bpfcc181300755editdlrm
telinit15013040755editdlrm
thin_check14305680755editdlrm
thin_delta14305680755editdlrm
thin_dump14305680755editdlrm
thin_ls14305680755editdlrm
thin_metadata_size14305680755editdlrm
thin_repair14305680755editdlrm
thin_restore14305680755editdlrm
thin_rmap14305680755editdlrm
thin_trim14305680755editdlrm
threadsnoop-bpfcc18580755editdlrm
threadsnoop.bt7120755editdlrm
tipc926880755editdlrm
tplist-bpfcc41570755editdlrm
trace-bpfcc438920755editdlrm
ttysnoop-bpfcc76940755editdlrm
tune2fs1132160755editdlrm
ucalls119700755editdlrm
uflow81150755editdlrm
ufw49590755editdlrm
ugc78230755editdlrm
umount.udisks2146400755editdlrm
undump.bt7890755editdlrm
unix_chkpwd310402755editdlrm
unix_update351360755editdlrm
uobjnew61810755editdlrm
update-ca-certificates54460755editdlrm
update-catalog93930755editdlrm
update-grub640755editdlrm
update-grub-gfxpayload3010755editdlrm
update-grub2640755editdlrm
update-ieee-data34920755editdlrm
update-info-dir17000755editdlrm
update-initramfs69080755editdlrm
update-locale30570755editdlrm
update-passwd353920755editdlrm
update-pciids17780755editdlrm
update-rc.d181470755editdlrm
update-secureboot-policy76050755editdlrm
update-shells39840755editdlrm
update-xmlcatalog172840755editdlrm
upgrade-from-grub-legacy15960755editdlrm
usb_modeswitch610960755editdlrm
usb_modeswitch_dispatcher274180755editdlrm
useradd1432320755editdlrm
userdel931920755editdlrm
usermod1307120755editdlrm
ustat124120755editdlrm
uthreads41000755editdlrm
uuidd316240755editdlrm
validlocale17730755editdlrm
vcstime146400755editdlrm
vdpa353920755editdlrm
veritysetup449920755editdlrm
vfscount-bpfcc13900755editdlrm
vfscount.bt5150755editdlrm
vfsstat-bpfcc41590755editdlrm
vfsstat.bt7210755editdlrm
vgcfgbackup31567120755editdlrm
vgcfgrestore31567120755editdlrm
vgchange31567120755editdlrm
vgck31567120755editdlrm
vgconvert31567120755editdlrm
vgcreate31567120755editdlrm
vgdisplay31567120755editdlrm
vgexport31567120755editdlrm
vgextend31567120755editdlrm
vgimport31567120755editdlrm
vgimportclone31567120755editdlrm
vgmerge31567120755editdlrm
vgmknodes31567120755editdlrm
vgreduce31567120755editdlrm
vgremove31567120755editdlrm
vgrename31567120755editdlrm
vgs31567120755editdlrm
vgscan31567120755editdlrm
vgsplit31567120755editdlrm
vigr621440755editdlrm
vipw621440755editdlrm
virtiostat-bpfcc89000755editdlrm
visudo2587760755editdlrm
vpddecode149280755editdlrm
wakeuptime-bpfcc82910755editdlrm
wipefs392960755editdlrm
writeback.bt16990755editdlrm
xfsdist-bpfcc47230755editdlrm
xfsdist.bt9720755editdlrm
xfsslower-bpfcc79640755editdlrm
xfs_admin21740755editdlrm
xfs_bmap6950755editdlrm
xfs_copy926080755editdlrm
xfs_db7050880755editdlrm
xfs_estimate145040755editdlrm
xfs_freeze8000755editdlrm
xfs_fsr431920755editdlrm
xfs_growfs391520755editdlrm
xfs_info12940755editdlrm
xfs_io2085360755editdlrm
xfs_logprint801440755editdlrm
xfs_mdrestore350480755editdlrm
xfs_metadump8160755editdlrm
xfs_mkfile10400755editdlrm
xfs_ncheck6850755editdlrm
xfs_quota923280755editdlrm
xfs_repair6587600755editdlrm
xfs_rtcp185840755editdlrm
xfs_scrub1088160755editdlrm
xfs_scrub_all78410755editdlrm
xfs_spaceman433200755editdlrm
xtables-legacy-multi951760755editdlrm
xtables-monitor2244240755editdlrm
xtables-nft-multi2244240755editdlrm
zerofree144880755editdlrm
zfsdist-bpfcc54270755editdlrm
zfsslower-bpfcc86570755editdlrm
zic679840755editdlrm
zramctl558240755editdlrm
Edit: /usr/sbin/argdist-bpfcc (36862B)
#! /usr/bin/python3 # # argdist Trace a function and display a distribution of its # parameter values as a histogram or frequency count. # # USAGE: argdist [-h] [-p PID] [-z STRING_SIZE] [-i INTERVAL] [-n COUNT] [-v] # [-c] [-T TOP] [-C specifier] [-H specifier] [-I header] # [-t TID] # # Licensed under the Apache License, Version 2.0 (the "License") # Copyright (C) 2016 Sasha Goldshtein. from bcc import BPF, USDT, StrcmpRewrite from time import sleep, strftime import argparse import re import traceback import os import sys class Probe(object): next_probe_index = 0 streq_index = 0 aliases = {"$PID": "(bpf_get_current_pid_tgid() >> 32)", "$COMM": "&val.name"} def _substitute_aliases(self, expr): if expr is None: return expr for alias, subst in Probe.aliases.items(): expr = expr.replace(alias, subst) return expr def _parse_signature(self): params = map(str.strip, self.signature.split(',')) self.param_types = {} for param in params: # If the type is a pointer, the * can be next to the # param name. Other complex types like arrays are not # supported right now. index = param.rfind('*') index = index if index != -1 else param.rfind(' ') param_type = param[0:index + 1].strip() param_name = param[index + 1:].strip() self.param_types[param_name] = param_type # Maintain list of user params. Then later decide to # switch to bpf_probe_read_kernel or bpf_probe_read_user. if "__user" in param_type.split(): self.probe_user_list.add(param_name) def _generate_entry(self): self.entry_probe_func = self.probe_func_name + "_entry" text = """ int PROBENAME(struct pt_regs *ctx SIGNATURE) { u64 __pid_tgid = bpf_get_current_pid_tgid(); u32 __pid = __pid_tgid; // lower 32 bits u32 __tgid = __pid_tgid >> 32; // upper 32 bits PID_FILTER TID_FILTER COLLECT return 0; } """ text = text.replace("PROBENAME", self.entry_probe_func) text = text.replace("SIGNATURE", "" if len(self.signature) == 0 else ", " + self.signature) text = text.replace("PID_FILTER", self._generate_pid_filter()) text = text.replace("TID_FILTER", self._generate_tid_filter()) collect = "" for pname in self.args_to_probe: param_hash = self.hashname_prefix + pname if pname == "__latency": collect += """ u64 __time = bpf_ktime_get_ns(); %s.update(&__pid, &__time); """ % param_hash else: collect += "%s.update(&__pid, &%s);\n" % \ (param_hash, pname) text = text.replace("COLLECT", collect) return text def _generate_entry_probe(self): # Any $entry(name) expressions result in saving that argument # when entering the function. self.args_to_probe = set() regex = r"\$entry\((\w+)\)" for expr in self.exprs: for arg in re.finditer(regex, expr): self.args_to_probe.add(arg.group(1)) for arg in re.finditer(regex, self.filter): self.args_to_probe.add(arg.group(1)) if any(map(lambda expr: "$latency" in expr, self.exprs)) or \ "$latency" in self.filter: self.args_to_probe.add("__latency") self.param_types["__latency"] = "u64" # nanoseconds for pname in self.args_to_probe: if pname not in self.param_types: raise ValueError("$entry(%s): no such param" % arg) self.hashname_prefix = "%s_param_" % self.probe_hash_name text = "" for pname in self.args_to_probe: # Each argument is stored in a separate hash that is # keyed by pid. text += "BPF_HASH(%s, u32, %s);\n" % \ (self.hashname_prefix + pname, self.param_types[pname]) text += self._generate_entry() return text def _generate_retprobe_prefix(self): # After we're done here, there are __%s_val variables for each # argument we needed to probe using $entry(name), and they all # have values (which isn't necessarily the case if we missed # the method entry probe). text = "" self.param_val_names = {} for pname in self.args_to_probe: val_name = "__%s_val" % pname text += "%s *%s = %s.lookup(&__pid);\n" % \ (self.param_types[pname], val_name, self.hashname_prefix + pname) text += "if (%s == 0) { return 0 ; }\n" % val_name self.param_val_names[pname] = val_name return text def _generate_comm_prefix(self): text = """ struct val_t { u32 pid; char name[sizeof(struct __string_t)]; }; struct val_t val = {.pid = (bpf_get_current_pid_tgid() >> 32) }; bpf_get_current_comm(&val.name, sizeof(val.name)); """ return text def _replace_entry_exprs(self): for pname, vname in self.param_val_names.items(): if pname == "__latency": entry_expr = "$latency" val_expr = "(bpf_ktime_get_ns() - *%s)" % vname else: entry_expr = "$entry(%s)" % pname val_expr = "(*%s)" % vname for i in range(0, len(self.exprs)): self.exprs[i] = self.exprs[i].replace( entry_expr, val_expr) self.filter = self.filter.replace(entry_expr, val_expr) def _attach_entry_probe(self): if self.is_user: self.bpf.attach_uprobe(name=self.library, sym=self.function, fn_name=self.entry_probe_func, pid=self.pid or -1) else: self.bpf.attach_kprobe(event=self.function, fn_name=self.entry_probe_func) def _bail(self, error): raise ValueError("error parsing probe '%s': %s" % (self.raw_spec, error)) def _validate_specifier(self): # Everything after '#' is the probe label, ignore it spec = self.raw_spec.split('#')[0] parts = spec.strip().split(':') if len(parts) < 3: self._bail("at least the probe type, library, and " + "function signature must be specified") if len(parts) > 6: self._bail("extraneous ':'-separated parts detected") if parts[0] not in ["r", "p", "t", "u"]: self._bail("probe type must be 'p', 'r', 't', or 'u'" + " but got '%s'" % parts[0]) if re.match(r"\S+\(.*\)", parts[2]) is None: self._bail(("function signature '%s' has an invalid " + "format") % parts[2]) def _parse_expr_types(self, expr_types): if len(expr_types) == 0: self._bail("no expr types specified") self.expr_types = expr_types.split(',') def _parse_exprs(self, exprs): if len(exprs) == 0: self._bail("no exprs specified") self.exprs = exprs.split(',') def _make_valid_identifier(self, ident): return re.sub(r'[^A-Za-z0-9_]', '_', ident) def __init__(self, tool, type, specifier): self.usdt_ctx = None self.streq_functions = "" self.pid = tool.args.pid self.tid = tool.args.tid self.cumulative = tool.args.cumulative or False self.raw_spec = specifier self.probe_user_list = set() self.bin_cmp = False self._validate_specifier() spec_and_label = specifier.split('#') self.label = spec_and_label[1] \ if len(spec_and_label) == 2 else None parts = spec_and_label[0].strip().split(':') self.type = type # hist or freq self.probe_type = parts[0] fparts = parts[2].split('(') self.function = fparts[0].strip() if self.probe_type == "t": self.library = "" # kernel self.tp_category = parts[1] self.tp_event = self.function elif self.probe_type == "u": self.library = parts[1] self.probe_func_name = self._make_valid_identifier( "%s_probe%d" % (self.function, Probe.next_probe_index)) self._enable_usdt_probe() else: self.library = parts[1] self.is_user = len(self.library) > 0 self.signature = fparts[1].strip()[:-1] self._parse_signature() # If the user didn't specify an expression to probe, we probe # the retval in a ret probe, or simply the value "1" otherwise. self.is_default_expr = len(parts) < 5 if not self.is_default_expr: self._parse_expr_types(parts[3]) self._parse_exprs(parts[4]) if len(self.exprs) != len(self.expr_types): self._bail("mismatched # of exprs and types") if self.type == "hist" and len(self.expr_types) > 1: self._bail("histograms can only have 1 expr") else: if not self.probe_type == "r" and self.type == "hist": self._bail("histograms must have expr") self.expr_types = \ ["u64" if not self.probe_type == "r" else "int"] self.exprs = \ ["1" if not self.probe_type == "r" else "$retval"] self.filter = "" if len(parts) != 6 else parts[5] self._substitute_exprs() # Do we need to attach an entry probe so that we can collect an # argument that is required for an exit (return) probe? def check(expr): keywords = ["$entry", "$latency"] return any(map(lambda kw: kw in expr, keywords)) self.entry_probe_required = self.probe_type == "r" and \ (any(map(check, self.exprs)) or check(self.filter)) self.probe_func_name = self._make_valid_identifier( "%s_probe%d" % (self.function, Probe.next_probe_index)) self.probe_hash_name = self._make_valid_identifier( "%s_hash%d" % (self.function, Probe.next_probe_index)) Probe.next_probe_index += 1 def _enable_usdt_probe(self): self.usdt_ctx = USDT(path=self.library, pid=self.pid) self.usdt_ctx.enable_probe( self.function, self.probe_func_name) def _substitute_exprs(self): def repl(expr): expr = self._substitute_aliases(expr) rdict = StrcmpRewrite.rewrite_expr(expr, self.bin_cmp, self.library, self.probe_user_list, self.streq_functions, Probe.streq_index) expr = rdict["expr"] self.streq_functions = rdict["streq_functions"] Probe.streq_index = rdict["probeid"] return expr.replace("$retval", "PT_REGS_RC(ctx)") for i in range(0, len(self.exprs)): self.exprs[i] = repl(self.exprs[i]) self.filter = repl(self.filter) def _is_string(self, expr_type): return expr_type == "char*" or expr_type == "char *" def _generate_hash_field(self, i): if self._is_string(self.expr_types[i]): return "struct __string_t v%d;\n" % i else: return "%s v%d;\n" % (self.expr_types[i], i) def _generate_usdt_arg_assignment(self, i): expr = self.exprs[i] if self.probe_type == "u" and expr[0:3] == "arg": arg_index = int(expr[3]) arg_ctype = self.usdt_ctx.get_probe_arg_ctype( self.function, arg_index - 1) return (" %s %s = 0;\n" + " bpf_usdt_readarg(%s, ctx, &%s);\n") \ % (arg_ctype, expr, expr[3], expr) else: return "" def _generate_field_assignment(self, i): text = self._generate_usdt_arg_assignment(i) if self._is_string(self.expr_types[i]): if self.is_user or \ self.exprs[i] in self.probe_user_list: probe_readfunc = "bpf_probe_read_user" else: probe_readfunc = "bpf_probe_read_kernel" return (text + " %s(&__key.v%d.s," + " sizeof(__key.v%d.s), (void *)%s);\n") % \ (probe_readfunc, i, i, self.exprs[i]) else: return text + " __key.v%d = %s;\n" % \ (i, self.exprs[i]) def _generate_hash_decl(self): if self.type == "hist": return "BPF_HISTOGRAM(%s, %s);" % \ (self.probe_hash_name, self.expr_types[0]) else: text = "struct %s_key_t {\n" % self.probe_hash_name for i in range(0, len(self.expr_types)): text += self._generate_hash_field(i) text += "};\n" text += "BPF_HASH(%s, struct %s_key_t, u64);\n" % \ (self.probe_hash_name, self.probe_hash_name) return text def _generate_key_assignment(self): if self.type == "hist": return self._generate_usdt_arg_assignment(0) + \ ("%s __key = %s;\n" % (self.expr_types[0], self.exprs[0])) else: text = "struct %s_key_t __key = {};\n" % \ self.probe_hash_name for i in range(0, len(self.exprs)): text += self._generate_field_assignment(i) return text def _generate_hash_update(self): if self.type == "hist": return "%s.atomic_increment(bpf_log2l(__key));" % \ self.probe_hash_name else: return "%s.atomic_increment(__key);" % \ self.probe_hash_name def _generate_pid_filter(self): # Kernel probes need to explicitly filter pid, because the # attach interface doesn't support pid filtering if self.pid is not None and not self.is_user: return "if (__tgid != %d) { return 0; }" % self.pid else: return "" def _generate_tid_filter(self): if self.tid is not None and not self.is_user: return "if (__pid != %d) { return 0; }" % self.tid else: return "" def generate_text(self): program = "" probe_text = """ DATA_DECL """ + ( "TRACEPOINT_PROBE(%s, %s)" % (self.tp_category, self.tp_event) if self.probe_type == "t" else "int PROBENAME(struct pt_regs *ctx SIGNATURE)") + """ { u64 __pid_tgid = bpf_get_current_pid_tgid(); u32 __pid = __pid_tgid; // lower 32 bits u32 __tgid = __pid_tgid >> 32; // upper 32 bits PID_FILTER TID_FILTER PREFIX KEY_EXPR if (!(FILTER)) return 0; COLLECT return 0; } """ prefix = "" signature = "" # If any entry arguments are probed in a ret probe, we need # to generate an entry probe to collect them if self.entry_probe_required: program += self._generate_entry_probe() prefix += self._generate_retprobe_prefix() # Replace $entry(paramname) with a reference to the # value we collected when entering the function: self._replace_entry_exprs() if self.probe_type == "p" and len(self.signature) > 0: # Only entry uprobes/kprobes can have user-specified # signatures. Other probes force it to (). signature = ", " + self.signature # If COMM is specified prefix with code to get process name if self.exprs.count(self.aliases['$COMM']): prefix += self._generate_comm_prefix() program += probe_text.replace("PROBENAME", self.probe_func_name) program = program.replace("SIGNATURE", signature) program = program.replace("PID_FILTER", self._generate_pid_filter()) program = program.replace("TID_FILTER", self._generate_tid_filter()) decl = self._generate_hash_decl() key_expr = self._generate_key_assignment() collect = self._generate_hash_update() program = program.replace("DATA_DECL", decl) program = program.replace("KEY_EXPR", key_expr) program = program.replace("FILTER", "1" if len(self.filter) == 0 else self.filter) program = program.replace("COLLECT", collect) program = program.replace("PREFIX", prefix) return self.streq_functions + program def _attach_u(self): libpath = BPF.find_library(self.library) if libpath is None: libpath = BPF.find_exe(self.library) if libpath is None or len(libpath) == 0: self._bail("unable to find library %s" % self.library) if self.probe_type == "r": self.bpf.attach_uretprobe(name=libpath, sym=self.function, fn_name=self.probe_func_name, pid=self.pid or -1) else: self.bpf.attach_uprobe(name=libpath, sym=self.function, fn_name=self.probe_func_name, pid=self.pid or -1) def _attach_k(self): if self.probe_type == "t": pass # Nothing to do for tracepoints elif self.probe_type == "r": self.bpf.attach_kretprobe(event=self.function, fn_name=self.probe_func_name) else: self.bpf.attach_kprobe(event=self.function, fn_name=self.probe_func_name) def attach(self, bpf): self.bpf = bpf if self.probe_type == "u": return if self.is_user: self._attach_u() else: self._attach_k() if self.entry_probe_required: self._attach_entry_probe() # Check whether hash table batch ops is supported if self.type == "freq" and self.bpf.kernel_struct_has_field( b'bpf_map_ops', b'map_lookup_and_delete_batch') == 1: self.htab_batch_ops = True else: self.htab_batch_ops = False def _v2s(self, v): # Most fields can be converted with plain str(), but strings # are wrapped in a __string_t which has an .s field if "__string_t" in type(v).__name__: return str(v.s) return str(v) def _display_expr(self, i): # Replace ugly latency calculation with $latency expr = self.exprs[i].replace( "(bpf_ktime_get_ns() - *____latency_val)", "$latency") # Replace alias values back with the alias name for alias, subst in Probe.aliases.items(): expr = expr.replace(subst, alias) # Replace retval expression with $retval expr = expr.replace("PT_REGS_RC(ctx)", "$retval") # Replace ugly (*__param_val) expressions with param name return re.sub(r"\(\*__(\w+)_val\)", r"\1", expr) def _display_key(self, key): if self.is_default_expr: if not self.probe_type == "r": return "total calls" else: return "retval = %s" % str(key.v0) else: # The key object has v0, ..., vk fields containing # the values of the expressions from self.exprs def str_i(i): key_i = self._v2s(getattr(key, "v%d" % i)) return "%s = %s" % \ (self._display_expr(i), key_i) return ", ".join(map(str_i, range(0, len(self.exprs)))) def display(self, top): data = self.bpf.get_table(self.probe_hash_name) if self.type == "freq": print(self.label or self.raw_spec) print("\t%-10s %s" % ("COUNT", "EVENT")) sdata = sorted(data.items_lookup_batch() if self.htab_batch_ops else data.items(), key=lambda p: p[1].value) if top is not None: sdata = sdata[-top:] for key, value in sdata: # Print some nice values if the user didn't # specify an expression to probe if self.is_default_expr: if not self.probe_type == "r": key_str = "total calls" else: key_str = "retval = %s" % \ self._v2s(key.v0) else: key_str = self._display_key(key) print("\t%-10s %s" % (str(value.value), key_str)) elif self.type == "hist": label = self.label or (self._display_expr(0) if not self.is_default_expr else "retval") data.print_log2_hist(val_type=label) if not self.cumulative: if self.htab_batch_ops: data.items_delete_batch() else: data.clear() def __str__(self): return self.label or self.raw_spec class Tool(object): examples = """ Probe specifier syntax: {p,r,t,u}:{[library],category}:function(signature):type[,type...]:expr[,expr...][:filter]][#label] Where: p,r,t,u -- probe at function entry, function exit, kernel tracepoint, or USDT probe in exit probes: can use $retval, $entry(param), $latency library -- the library that contains the function (leave empty for kernel functions) category -- the category of the kernel tracepoint (e.g. net, sched) function -- the function name to trace (or tracepoint name) signature -- the function's parameters, as in the C header type -- the type of the expression to collect (supports multiple) expr -- the expression to collect (supports multiple) filter -- the filter that is applied to collected values label -- the label for this probe in the resulting output EXAMPLES: argdist -H 'p::__kmalloc(u64 size):u64:size' Print a histogram of allocation sizes passed to kmalloc argdist -p 1005 -C 'p:c:malloc(size_t size):size_t:size:size==16' Print a frequency count of how many times process 1005 called malloc with an allocation size of 16 bytes argdist -C 'r:c:gets():char*:(char*)$retval#snooped strings' Snoop on all strings returned by gets() argdist -H 'r::__kmalloc(size_t size):u64:$latency/$entry(size)#ns per byte' Print a histogram of nanoseconds per byte from kmalloc allocations argdist -C 'p::__kmalloc(size_t sz, gfp_t flags):size_t:sz:flags&GFP_ATOMIC' Print frequency count of kmalloc allocation sizes that have GFP_ATOMIC argdist -p 1005 -C 'p:c:write(int fd):int:fd' -T 5 Print frequency counts of how many times writes were issued to a particular file descriptor number, in process 1005, but only show the top 5 busiest fds argdist -p 1005 -H 'r:c:read()' Print a histogram of results (sizes) returned by read() in process 1005 argdist -C 'r::__vfs_read():u32:$PID:$latency > 100000' Print frequency of reads by process where the latency was >0.1ms argdist -C 'r::__vfs_read():u32:$COMM:$latency > 100000' Print frequency of reads by process name where the latency was >0.1ms argdist -H 'r::__vfs_read(void *file, void *buf, size_t count):size_t: $entry(count):$latency > 1000000' Print a histogram of read sizes that were longer than 1ms argdist -H \\ 'p:c:write(int fd, const void *buf, size_t count):size_t:count:fd==1' Print a histogram of buffer sizes passed to write() across all processes, where the file descriptor was 1 (STDOUT) argdist -C 'p:c:fork()#fork calls' Count fork() calls in libc across all processes Can also use funccount.py, which is easier and more flexible argdist -H 't:block:block_rq_complete():u32:args->nr_sector' Print histogram of number of sectors in completing block I/O requests argdist -C 't:irq:irq_handler_entry():int:args->irq' Aggregate interrupts by interrupt request (IRQ) argdist -C 'u:pthread:pthread_start():u64:arg2' -p 1337 Print frequency of function addresses used as a pthread start function, relying on the USDT pthread_start probe in process 1337 argdist -H 'p:c:sleep(u32 seconds):u32:seconds' \\ -H 'p:c:nanosleep(struct timespec *req):long:req->tv_nsec' Print histograms of sleep() and nanosleep() parameter values argdist -p 2780 -z 120 \\ -C 'p:c:write(int fd, char* buf, size_t len):char*:buf:fd==1' Spy on writes to STDOUT performed by process 2780, up to a string size of 120 characters argdist -I 'kernel/sched/sched.h' \\ -C 'p::__account_cfs_rq_runtime(struct cfs_rq *cfs_rq):s64:cfs_rq->runtime_remaining' Trace on the cfs scheduling runqueue remaining runtime. The struct cfs_rq is defined in kernel/sched/sched.h which is in kernel source tree and not in kernel-devel package. So this command needs to run at the kernel source tree root directory so that the added header file can be found by the compiler. """ def __init__(self): parser = argparse.ArgumentParser(description="Trace a " + "function and display a summary of its parameter values.", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=Tool.examples) parser.add_argument("-p", "--pid", type=int, help="id of the process to trace (optional)") parser.add_argument("-t", "--tid", type=int, help="id of the thread to trace (optional)") parser.add_argument("-z", "--string-size", default=80, type=int, help="maximum string size to read from char* arguments") parser.add_argument("-i", "--interval", default=1, type=int, help="output interval, in seconds (default 1 second)") parser.add_argument("-d", "--duration", type=int, help="total duration of trace, in seconds") parser.add_argument("-n", "--number", type=int, dest="count", help="number of outputs") parser.add_argument("-v", "--verbose", action="store_true", help="print resulting BPF program code before executing") parser.add_argument("-c", "--cumulative", action="store_true", help="do not clear histograms and freq counts at " + "each interval") parser.add_argument("-T", "--top", type=int, help="number of top results to show (not applicable to " + "histograms)") parser.add_argument("-H", "--histogram", action="append", dest="histspecifier", metavar="specifier", help="probe specifier to capture histogram of " + "(see examples below)") parser.add_argument("-C", "--count", action="append", dest="countspecifier", metavar="specifier", help="probe specifier to capture count of " + "(see examples below)") parser.add_argument("-I", "--include", action="append", metavar="header", help="additional header files to include in the BPF program " "as either full path, " "or relative to relative to current working directory, " "or relative to default kernel header search path") parser.add_argument("--ebpf", action="store_true", help=argparse.SUPPRESS) self.args = parser.parse_args() self.usdt_ctx = None def _create_probes(self): self.probes = [] for specifier in (self.args.countspecifier or []): self.probes.append(Probe(self, "freq", specifier)) for histspecifier in (self.args.histspecifier or []): self.probes.append(Probe(self, "hist", histspecifier)) if len(self.probes) == 0: print("at least one specifier is required") exit(1) def _generate_program(self): bpf_source = """ struct __string_t { char s[%d]; }; #include """ % self.args.string_size for include in (self.args.include or []): if include.startswith((".", "/")): include = os.path.abspath(include) bpf_source += "#include \"%s\"\n" % include else: bpf_source += "#include <%s>\n" % include bpf_source += BPF.generate_auto_includes( map(lambda p: p.raw_spec, self.probes)) for probe in self.probes: bpf_source += probe.generate_text() if self.args.verbose: for text in [probe.usdt_ctx.get_text() for probe in self.probes if probe.usdt_ctx]: print(text) if self.args.verbose or self.args.ebpf: print(bpf_source) if self.args.ebpf: exit() usdt_contexts = [probe.usdt_ctx for probe in self.probes if probe.usdt_ctx] self.bpf = BPF(text=bpf_source, usdt_contexts=usdt_contexts) def _attach(self): for probe in self.probes: probe.attach(self.bpf) if self.args.verbose: print("open uprobes: %s" % list(self.bpf.uprobe_fds.keys())) print("open kprobes: %s" % list(self.bpf.kprobe_fds.keys())) def _main_loop(self): count_so_far = 0 seconds = 0 while True: try: sleep(self.args.interval) seconds += self.args.interval except KeyboardInterrupt: exit() print("[%s]" % strftime("%H:%M:%S")) for probe in self.probes: probe.display(self.args.top) count_so_far += 1 if self.args.count is not None and \ count_so_far >= self.args.count: exit() if self.args.duration and \ seconds >= self.args.duration: exit() def run(self): try: self._create_probes() self._generate_program() self._attach() self._main_loop() except: exc_info = sys.exc_info() sys_exit = exc_info[0] is SystemExit if self.args.verbose: traceback.print_exc() elif not sys_exit: print(exc_info[1]) exit(0 if sys_exit else 1) if __name__ == "__main__": Tool().run()