Slide 37
Slide 37 text
GPUprobe: Tracing CUDA Workloads with eBPF
sudo bpftrace -e '
uprobe:/usr/lib/x86_64-linux-gnu/libcuda.so:cuLaunchKernel
{
@kernel_launches[comm] = count();
printf("CUDA kernel launch: process=%s pid=%d\n", comm, pid);
}
uprobe:/usr/lib/x86_64-linux-gnu/libcudart.so:cudaMemcpy
{
@memcpy_calls[comm] = count();
printf("CUDA memory copy: process=%s pid=%d\n", comm, pid);
}
tracepoint:syscalls:sys_exit_ioctl
/comm -= "python"/
{
@ioctl_latency = hist(nsecs);
}
'
CUDA kernel launch: process=python pid=1842
CUDA memory copy: process=python pid=1842
CUDA kernel launch: process=python pid=1842
CUDA kernel launch: process=python pid=1842
@kernel_launches[python]: 3421
@memcpy_calls[python]: 891
@ioctl_latency:
[1K, 2K) 120
[2K, 4K) 387
[4K, 8K) 912
[8K, 16K) 240
[16K, 32K) 42
@doniacld