Brendan Gregg brendangregg

Testing of Xen vPMU filter options.

Three tests are executed on a simple program (noploop, which runs NOP in a loop). The first two show Linux "perf stat" summaries, and the third tests specific counters: the IPC ones, an architectural one, and a few extras. Which of these will work depends on the filter mode.

I'll begin by showing off and on, the current modes, and then the new modes: ipc and arch.

vpmu=off (default)

root@lgud-bgregg:~> xl dmesg | grep vpmu

	struct task_struct {
	volatile long int state; /* 0 8 */
	void * stack; /* 8 8 */
	atomic_t usage; /* 16 4 */
	unsigned int flags; /* 20 4 */
	unsigned int ptrace; /* 24 4 */

	/* XXX 4 bytes hole, try to pack */

	struct llist_node wake_entry; /* 32 8 */

	#!/usr/bin/python
	#
	# irqdist Summarize irq event time as a histogram.
	# For Linux, uses BCC, eBPF.
	#
	# USAGE: irqdist [-h] [-T] [-Q] [-m] [-D] [interval] [count]
	#
	# Copyright (c) 2015 Brendan Gregg.
	# Licensed under the Apache License, Version 2.0 (the "License")
	#

	#!/usr/bin/python
	#
	# biolatency Summarize block device I/O latency as a histogram.
	# For Linux, uses BCC, eBPF.
	#
	# USAGE: biolatency [-h] [-T] [-Q] [-m] [interval] [count]
	#
	# Copyright (c) 2015 Brendan Gregg.
	# Licensed under the Apache License, Version 2.0 (the "License")
	#

	#!/usr/bin/python
	#
	# opensnoop Trace open() syscalls.
	# For Linux, uses BCC, eBPF. Embedded C.
	#
	# USAGE: opensnoop [-h] [-t] [-x] [-p PID]
	#
	# Copyright (c) 2015 Brendan Gregg.
	# Licensed under the Apache License, Version 2.0 (the "License")
	#

	To get https://github.com/iovisor/bcc running (eBPF front-end)

	# do a normal linux 4.3+ build, or get 4.3+ pkgs (https://github.com/iovisor/bcc/blob/master/INSTALL.md#ubuntu---binary)
	sudo apt-get -y install bison build-essential cmake flex git libedit-dev python zlib1g-dev
	# Build LLVM and Clang development libs
	git clone --depth 1 http://llvm.org/git/llvm.git
	cd llvm/tools; git clone --depth 1 http://llvm.org/git/clang.git
	cd ..; mkdir -p build/install; cd build
	cmake -G "Unix Makefiles" -DLLVM_TARGETS_TO_BUILD="BPF;X86" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PWD/install ..
	make -j4

	# /apps/perf-tools/bin/funcslower -t spa_sync 100000
	Tracing "spa_sync" slower than 100000 us... Ctrl-C to end.
	4445151.315130 \| 0) ! 151766.8 us \| } /* spa_sync [zfs] */
	4445166.267272 \| 8) ! 104154.9 us \| } /* spa_sync [zfs] */
	4445191.382823 \| 7) ! 216032.0 us \| } /* spa_sync [zfs] */
	4445196.291821 \| 7) ! 125089.3 us \| } /* spa_sync [zfs] */
	4445201.371173 \| 7) ! 204526.1 us \| } /* spa_sync [zfs] */
	^C
	Ending tracing...

	# /apps/perf-tools/bin/kprobe -s 'p:taskq_dispatch +0(+16(%di)):string' \| head -20
	taskq_dispatch [spl]
	Tracing kprobe taskq_dispatch. Ctrl-C to end.
	<...>-68258 [007] d... 1242008.371281: taskq_dispatch: (taskq_dispatch+0x0/0x170 [spl]) arg1="metaslab_group_taskq"
	<...>-68258 [007] d... 1242008.371289: <stack trace>
	=> vdev_sync_done
	=> spa_sync
	=> txg_sync_thread
	=> thread_generic_wrapper
	=> kthread

	<idle>-0 [002] dNh. 5790376.199595: offcpu_sched_wakeup: waker_comm=swapper/2 (0), wakee_comm=sleep (29321), wakee_offcpu_delay=6000061382, flag=0, waker_stack=ttwu_do_wakeup+0xb2/0xc0
	ttwu_do_activate.constprop.74+0x5d/0x70
	try_to_wake_up+0x1d2/0x2c0
	wake_up_process+0x23/0x40
	hrtimer_wakeup+0x22/0x40
	__run_hrtimer+0x77/0x1d0
	hrtimer_interrupt+0xef/0x230
	local_apic_timer_interrupt+0x37/0x60
	smp_apic_timer_interr
	<idle>-0 [002] d... 5790376.199618: offcpu_sched_switch: comm=sleep, pid=29321, delay=6000101834, flag=0, stack=schedule+0x29/0x70

	#!/usr/bin/stap
	/*
	* chaintime-nd.stp Measure off-CPU time by stack trace & wakeup stack.
	* For Linux, uses SystemTap (non-debuginfo).
	*
	* USAGE: ./chaintime-nd.stp [duration]
	*
	* This script uses the kernel tracepoint sched:sched_switch to measure off-CPU
	* time along with the blocked stack trace, and sched:sched_wakeup to measure
	* the wakeup stack for sleeping threads, to associate with the blocked stack.