# debugging
--------------------------------------------------------------------------------

#
# strace(1)
#

strace [OPTS] [ELF]
  -f .......... follow child processes on fork(2)
  -p <pid> .... attach to running process
  -s <size> ... max string size (default: 32)
  -e <expr> ... expression for trace filtering
  -o <file> ... log output into <file>
  -c .......... dump syscall statitics at the end

useful <expr>:
  trace=syscall[,syscall] .... trace only syscall listed
  trace=file ................. trace all syscall that take a filename as arg
  trace=process .............. trace process management related syscalls
  trace=signal ............... trace signal related syscalls
  signal ..................... trace signals delivered to the process

use cases:
- trace 'open & socket' syscalls for a running process + childs
strace -f -p <pid> -e trace=open,socket

- trace signals delivered to a running process
strace -f -p <pid> -e signal


#
# lsof(8)
#

lsof
  -a ......... AND slection filters instead ORing (OR: default)
  -p <pid> ... list open file descriptors for process
  +fg ........ show file flags for file descripros
  -n ......... don't convert network addr to hostnames
  -P ......... dont' convert network port to know service names
  -i <@h[:p]>. show connections to h (hostname|ip addr) with optional port p

file flags:
  R/W/RW ..... read/write/read-write
  CR ......... create
  AP ......... append
  TR ......... truncate

uase cases:
- show open files with file flags
lsof +fg -p <pid>

- show open tcp connections from user
lsof -a -u $USER -i tcp

- show open connections to 'localhost' for user
lsof -a -u $USER -i @localhost

#
# pmap(1)
#

pmap <pid>
  ............. dump virtual memory map of process.
                compared to /proc/<pid>/maps it shows the size of the mappings


#
# pstack(1)
#

pstack <pid>
  ............. dump current stack of process + threads


#
# pidstat(1)
#

# trace minor/major page faults
pidstat -r -p <pid> [interval]
  minor_pagefault: happens when the page needed is already in memory but not
                   allocated to the faulting process, in that case the kernel
                   only has to create a new page-table entry pointing to the
                   shared physical page
  major_pagefault: happends when the page needed is NOT in memory, the kernel
                   has to create a new page-table entry and populate the
                   physical page


#
# /usr/bin/time(1)
#

# statistics of process run
/usr/bin/time -v <cmd>


#
# perf(1)
#

# get supported events
perf list
  ......... show supported hw/sw events

perf stat
  -p <pid> .. show stats for running process
  -I <ms> ... show stats periodically over interval <ms>
  -e <ev> ... filter for events

perf top
  -p <pid> .. show stats for running process
  -F <hz> ... sampling frequency
  -K ........ hide kernel threads

perf record
  -p <pid> ............... record stats for running process
  -F <hz> ................ sampling frequency
  --call-graph <method> .. [fp, dwarf, lbr] method how to caputre backtrace
                           fp   : use frame-pointer, need -fno-omit-frame-pointer
                           dwarf: use .cfi debug information
                           lbr  : use hardware last branch record facility
  -g ..................... short-hand for --call-graph fp
  -e <ev> ................ filter for events

perf report
  -n .................... annotate symbols with nr of samples
  --stdio ............... report to stdio, if not presen tui mode
  -g graph,0.5,caller ... show caller based call chains with value >0.5

useful <ev>:
  page-faults
  minor-faults
  major-faults
  cpu-cycles`
  task-clock


#
# flamegraph(https://github.com/brendangregg/FlameGraph)
#

# flamegraph for single event trace
perf record -g -p <pid> -e cpu-cycles
perf script | FlameGraph/stackcollapse-perf.pl | FlameGraph/flamegraph.pl > cycles-flamegraph.svg

# flamegraphs for multiple events trace
perf record -g -p <pid> -e cpu-cycles,page-faults
perf script --per-event-dump
# fold & generate as above
#

#
# OProfile
#

operf -g -p <pid>
  -g ...... caputre call-graph information

opreport [opt] FILE
  NOOPT ... show time spent per binary image
  -l ...... show time spent per symbol
  -c ...... show callgraph information (see below)
  -a ...... add column with time spent accumulated over child nodes

ophelp
  NOOPT ... show supported hw/sw events

--------------------------------------------------------------------------------
vim:ft=help:sts=2:et:tw=80:cc=80:fo+=t