Skip to content

Instantly share code, notes, and snippets.

@pkhuong

pkhuong/cycle.h

Last active November 7, 2018 04:38
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save pkhuong/f63c16dcf60f7f9f066ca0b1eeba3495 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save pkhuong/f63c16dcf60f7f9f066ca0b1eeba3495 to your computer and use it in GitHub Desktop.
Download ZIP
interrupt abuse
Raw
cycle.h
/*
* Copyright (c) 2003, 2007 Matteo Frigo
* Copyright (c) 2003, 2007 Massachusetts Institute of Technology
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
*/
/* machine-dependent cycle counters code. Needs to be inlined. */
/***************************************************************************/
/* To use the cycle counters in your code, simply #include "cycle.h" (this
file), and then use the functions/macros:
ticks getticks(void);
ticks is an opaque typedef defined below, representing the current time.
You extract the elapsed time between two calls to gettick() via:
double elapsed(ticks t1, ticks t0);
which returns a double-precision variable in arbitrary units. You
are not expected to convert this into human units like seconds; it
is intended only for *comparisons* of time intervals.
(In order to use some of the OS-dependent timer routines like
Solaris' gethrtime, you need to paste the autoconf snippet below
into your configure.ac file and #include "config.h" before cycle.h,
or define the relevant macros manually if you are not using autoconf.)
*/
/***************************************************************************/
/* This file uses macros like HAVE_GETHRTIME that are assumed to be
defined according to whether the corresponding function/type/header
is available on your system. The necessary macros are most
conveniently defined if you are using GNU autoconf, via the tests:
dnl ---------------------------------------------------------------------
AC_C_INLINE
AC_HEADER_TIME
AC_CHECK_HEADERS([sys/time.h c_asm.h intrinsics.h mach/mach_time.h])
AC_CHECK_TYPE([hrtime_t],[AC_DEFINE(HAVE_HRTIME_T, 1, [Define to 1 if hrtime_t is defined in <sys/time.h>])],,[#if HAVE_SYS_TIME_H
#include <sys/time.h>
#endif])
AC_CHECK_FUNCS([gethrtime read_real_time time_base_to_time clock_gettime mach_absolute_time])
dnl Cray UNICOS _rtc() (real-time clock) intrinsic
AC_MSG_CHECKING([for _rtc intrinsic])
rtc_ok=yes
AC_TRY_LINK([#ifdef HAVE_INTRINSICS_H
#include <intrinsics.h>
#endif], [_rtc()], [AC_DEFINE(HAVE__RTC,1,[Define if you have the UNICOS _rtc() intrinsic.])], [rtc_ok=no])
AC_MSG_RESULT($rtc_ok)
dnl ---------------------------------------------------------------------
*/
/***************************************************************************/
#if TIME_WITH_SYS_TIME
# include <sys/time.h>
# include <time.h>
#else
# if HAVE_SYS_TIME_H
# include <sys/time.h>
# else
# include <time.h>
# endif
#endif
#define INLINE_ELAPSED(INL) static INL double elapsed(ticks t1, ticks t0) \
{ \
return (double)t1 - (double)t0; \
}
/*----------------------------------------------------------------*/
/* Solaris */
#if defined(HAVE_GETHRTIME) && defined(HAVE_HRTIME_T) && !defined(HAVE_TICK_COUNTER)
typedef hrtime_t ticks;
#define getticks gethrtime
INLINE_ELAPSED(inline)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/* AIX v. 4+ routines to read the real-time clock or time-base register */
#if defined(HAVE_READ_REAL_TIME) && defined(HAVE_TIME_BASE_TO_TIME) && !defined(HAVE_TICK_COUNTER)
typedef timebasestruct_t ticks;
static __inline ticks getticks(void)
{
ticks t;
read_real_time(&t, TIMEBASE_SZ);
return t;
}
static __inline double elapsed(ticks t1, ticks t0) /* time in nanoseconds */
{
time_base_to_time(&t1, TIMEBASE_SZ);
time_base_to_time(&t0, TIMEBASE_SZ);
return (((double)t1.tb_high - (double)t0.tb_high) * 1.0e9 +
((double)t1.tb_low - (double)t0.tb_low));
}
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/*
* PowerPC ``cycle'' counter using the time base register.
*/
#if ((((defined(__GNUC__) && (defined(__powerpc__) || defined(__ppc__))) || (defined(__MWERKS__) && defined(macintosh)))) || (defined(__IBM_GCC_ASM) && (defined(__powerpc__) || defined(__ppc__)))) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long long ticks;
static __inline__ ticks getticks(void)
{
unsigned int tbl, tbu0, tbu1;
do {
__asm__ __volatile__ ("mftbu %0" : "=r"(tbu0));
__asm__ __volatile__ ("mftb %0" : "=r"(tbl));
__asm__ __volatile__ ("mftbu %0" : "=r"(tbu1));
} while (tbu0 != tbu1);
return (((unsigned long long)tbu0) << 32) | tbl;
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/* MacOS/Mach (Darwin) time-base register interface (unlike UpTime,
from Carbon, requires no additional libraries to be linked). */
#if defined(HAVE_MACH_ABSOLUTE_TIME) && defined(HAVE_MACH_MACH_TIME_H) && !defined(HAVE_TICK_COUNTER)
#include <mach/mach_time.h>
typedef uint64_t ticks;
#define getticks mach_absolute_time
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/*
* Pentium cycle counter
*/
#if (defined(__GNUC__) || defined(__ICC)) && defined(__i386__) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long long ticks;
static __inline__ ticks getticks(void)
{
ticks ret;
__asm__ __volatile__("rdtscp": "=A" (ret) :: "ecx");
/* no input, nothing else clobbered */
return ret;
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#define TIME_MIN 5000.0 /* unreliable pentium IV cycle counter */
#endif
/* Visual C++ -- thanks to Morten Nissov for his help with this */
#if _MSC_VER >= 1200 && _M_IX86 >= 500 && !defined(HAVE_TICK_COUNTER)
#include <windows.h>
typedef LARGE_INTEGER ticks;
#define RDTSC __asm __emit 0fh __asm __emit 031h /* hack for VC++ 5.0 */
static __inline ticks getticks(void)
{
ticks retval;
__asm {
RDTSC
mov retval.HighPart, edx
mov retval.LowPart, eax
}
return retval;
}
static __inline double elapsed(ticks t1, ticks t0)
{
return (double)t1.QuadPart - (double)t0.QuadPart;
}
#define HAVE_TICK_COUNTER
#define TIME_MIN 5000.0 /* unreliable pentium IV cycle counter */
#endif
/*----------------------------------------------------------------*/
/*
* X86-64 cycle counter
*/
#if (defined(__GNUC__) || defined(__ICC) || defined(__SUNPRO_C)) && defined(__x86_64__) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long long ticks;
static __inline__ ticks getticks(void)
{
unsigned a, d;
asm volatile("rdtscp" : "=a" (a), "=d" (d) :: "rcx");
return ((ticks)a) | (((ticks)d) << 32);
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/* PGI compiler, courtesy Cristiano Calonaci, Andrea Tarsi, & Roberto Gori.
NOTE: this code will fail to link unless you use the -Masmkeyword compiler
option (grrr). */
#if defined(__PGI) && defined(__x86_64__) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long long ticks;
static ticks getticks(void)
{
asm(" rdtsc; shl $0x20,%rdx; mov %eax,%eax; or %rdx,%rax; ");
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/* Visual C++, courtesy of Dirk Michaelis */
#if _MSC_VER >= 1400 && (defined(_M_AMD64) || defined(_M_X64)) && !defined(HAVE_TICK_COUNTER)
#include <intrin.h>
#pragma intrinsic(__rdtsc)
typedef unsigned __int64 ticks;
#define getticks __rdtsc
INLINE_ELAPSED(__inline)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/*
* IA64 cycle counter
*/
/* intel's icc/ecc compiler */
#if (defined(__EDG_VERSION) || defined(__ECC)) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long ticks;
#include <ia64intrin.h>
static __inline__ ticks getticks(void)
{
return __getReg(_IA64_REG_AR_ITC);
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/* gcc */
#if defined(__GNUC__) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long ticks;
static __inline__ ticks getticks(void)
{
ticks ret;
__asm__ __volatile__ ("mov %0=ar.itc" : "=r"(ret));
return ret;
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/* HP/UX IA64 compiler, courtesy Teresa L. Johnson: */
#if defined(__hpux) && defined(__ia64) && !defined(HAVE_TICK_COUNTER)
#include <machine/sys/inline.h>
typedef unsigned long ticks;
static inline ticks getticks(void)
{
ticks ret;
ret = _Asm_mov_from_ar (_AREG_ITC);
return ret;
}
INLINE_ELAPSED(inline)
#define HAVE_TICK_COUNTER
#endif
/* Microsoft Visual C++ */
#if defined(_MSC_VER) && defined(_M_IA64) && !defined(HAVE_TICK_COUNTER)
typedef unsigned __int64 ticks;
# ifdef __cplusplus
extern "C"
# endif
ticks __getReg(int whichReg);
#pragma intrinsic(__getReg)
static __inline ticks getticks(void)
{
volatile ticks temp;
temp = __getReg(3116);
return temp;
}
INLINE_ELAPSED(inline)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/*
* PA-RISC cycle counter
*/
#if defined(__hppa__) || defined(__hppa) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long ticks;
# ifdef __GNUC__
static __inline__ ticks getticks(void)
{
ticks ret;
__asm__ __volatile__("mfctl 16, %0": "=r" (ret));
/* no input, nothing else clobbered */
return ret;
}
# else
# include <machine/inline.h>
static inline unsigned long getticks(void)
{
register ticks ret;
_MFCTL(16, ret);
return ret;
}
# endif
INLINE_ELAPSED(inline)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/* S390, courtesy of James Treacy */
#if defined(__GNUC__) && defined(__s390__) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long long ticks;
static __inline__ ticks getticks(void)
{
ticks cycles;
__asm__("stck 0(%0)" : : "a" (&(cycles)) : "memory", "cc");
return cycles;
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
#if defined(__GNUC__) && defined(__alpha__) && !defined(HAVE_TICK_COUNTER)
/*
* The 32-bit cycle counter on alpha overflows pretty quickly,
* unfortunately. A 1GHz machine overflows in 4 seconds.
*/
typedef unsigned int ticks;
static __inline__ ticks getticks(void)
{
unsigned long cc;
__asm__ __volatile__ ("rpcc %0" : "=r"(cc));
return (cc & 0xFFFFFFFF);
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
#if defined(__GNUC__) && defined(__sparc_v9__) && !defined(HAVE_TICK_COUNTER)
typedef unsigned long ticks;
static __inline__ ticks getticks(void)
{
ticks ret;
__asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
return ret;
}
INLINE_ELAPSED(__inline__)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
#if (defined(__DECC) || defined(__DECCXX)) && defined(__alpha) && defined(HAVE_C_ASM_H) && !defined(HAVE_TICK_COUNTER)
# include <c_asm.h>
typedef unsigned int ticks;
static __inline ticks getticks(void)
{
unsigned long cc;
cc = asm("rpcc %v0");
return (cc & 0xFFFFFFFF);
}
INLINE_ELAPSED(__inline)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/* SGI/Irix */
#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_SGI_CYCLE) && !defined(HAVE_TICK_COUNTER)
typedef struct timespec ticks;
static inline ticks getticks(void)
{
struct timespec t;
clock_gettime(CLOCK_SGI_CYCLE, &t);
return t;
}
static inline double elapsed(ticks t1, ticks t0)
{
return ((double)t1.tv_sec - (double)t0.tv_sec) * 1.0E9 +
((double)t1.tv_nsec - (double)t0.tv_nsec);
}
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/* Cray UNICOS _rtc() intrinsic function */
#if defined(HAVE__RTC) && !defined(HAVE_TICK_COUNTER)
#ifdef HAVE_INTRINSICS_H
# include <intrinsics.h>
#endif
typedef long long ticks;
#define getticks _rtc
INLINE_ELAPSED(inline)
#define HAVE_TICK_COUNTER
#endif
/*----------------------------------------------------------------*/
/* MIPS ZBus */
#if HAVE_MIPS_ZBUS_TIMER
#if defined(__mips__) && !defined(HAVE_TICK_COUNTER)
#include <sys/mman.h>
#include <unistd.h>
#include <fcntl.h>
typedef uint64_t ticks;
static inline ticks getticks(void)
{
static uint64_t* addr = 0;
if (addr == 0)
{
uint32_t rq_addr = 0x10030000;
int fd;
int pgsize;
pgsize = getpagesize();
fd = open ("/dev/mem", O_RDONLY | O_SYNC, 0);
if (fd < 0) {
perror("open");
return NULL;
}
addr = mmap(0, pgsize, PROT_READ, MAP_SHARED, fd, rq_addr);
close(fd);
if (addr == (uint64_t *)-1) {
perror("mmap");
return NULL;
}
}
return *addr;
}
INLINE_ELAPSED(inline)
#define HAVE_TICK_COUNTER
#endif
#endif /* HAVE_MIPS_ZBUS_TIMER */
Raw
libbpf-macros.h
/*
* Macros are lifted from libbpf.h in iovisor's BCC.
*
* Copyright (c) 2015 PLUMgrid, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LIBBPF_MACROS_H
#define LIBBPF_MACROS_H
#include <linux/bpf.h>
#define BPF_ALU64_REG(OP, DST, SRC) \
((struct bpf_insn) { \
.code = BPF_ALU64 | BPF_OP(OP) | BPF_X, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = 0, \
.imm = 0 })
#define BPF_ALU32_REG(OP, DST, SRC) \
((struct bpf_insn) { \
.code = BPF_ALU | BPF_OP(OP) | BPF_X, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = 0, \
.imm = 0 })
/* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */
#define BPF_ALU64_IMM(OP, DST, IMM) \
((struct bpf_insn) { \
.code = BPF_ALU64 | BPF_OP(OP) | BPF_K, \
.dst_reg = DST, \
.src_reg = 0, \
.off = 0, \
.imm = IMM })
#define BPF_ALU32_IMM(OP, DST, IMM) \
((struct bpf_insn) { \
.code = BPF_ALU | BPF_OP(OP) | BPF_K, \
.dst_reg = DST, \
.src_reg = 0, \
.off = 0, \
.imm = IMM })
/* Short form of mov, dst_reg = src_reg */
#define BPF_MOV64_REG(DST, SRC) \
((struct bpf_insn) { \
.code = BPF_ALU64 | BPF_MOV | BPF_X, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = 0, \
.imm = 0 })
/* Short form of mov, dst_reg = imm32 */
#define BPF_MOV64_IMM(DST, IMM) \
((struct bpf_insn) { \
.code = BPF_ALU64 | BPF_MOV | BPF_K, \
.dst_reg = DST, \
.src_reg = 0, \
.off = 0, \
.imm = IMM })
/* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
#define BPF_LD_IMM64(DST, IMM) \
BPF_LD_IMM64_RAW(DST, 0, IMM)
#define BPF_LD_IMM64_RAW(DST, SRC, IMM) \
((struct bpf_insn) { \
.code = BPF_LD | BPF_DW | BPF_IMM, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = 0, \
.imm = (__u32) (IMM) }), \
((struct bpf_insn) { \
.code = 0, /* zero is reserved opcode */ \
.dst_reg = 0, \
.src_reg = 0, \
.off = 0, \
.imm = ((__u64) (IMM)) >> 32 })
#define BPF_PSEUDO_MAP_FD 1
/* pseudo BPF_LD_IMM64 insn used to refer to process-local map_fd */
#define BPF_LD_MAP_FD(DST, MAP_FD) \
BPF_LD_IMM64_RAW(DST, BPF_PSEUDO_MAP_FD, MAP_FD)
/* Direct packet access, R0 = *(uint *) (skb->data + imm32) */
#define BPF_LD_ABS(SIZE, IMM) \
((struct bpf_insn) { \
.code = BPF_LD | BPF_SIZE(SIZE) | BPF_ABS, \
.dst_reg = 0, \
.src_reg = 0, \
.off = 0, \
.imm = IMM })
/* Memory load, dst_reg = *(uint *) (src_reg + off16) */
#define BPF_LDX_MEM(SIZE, DST, SRC, OFF) \
((struct bpf_insn) { \
.code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = OFF, \
.imm = 0 })
/* Memory store, *(uint *) (dst_reg + off16) = src_reg */
#define BPF_STX_MEM(SIZE, DST, SRC, OFF) \
((struct bpf_insn) { \
.code = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = OFF, \
.imm = 0 })
/* Memory store, *(uint *) (dst_reg + off16) = imm32 */
#define BPF_ST_MEM(SIZE, DST, OFF, IMM) \
((struct bpf_insn) { \
.code = BPF_ST | BPF_SIZE(SIZE) | BPF_MEM, \
.dst_reg = DST, \
.src_reg = 0, \
.off = OFF, \
.imm = IMM })
/* Conditional jumps against registers, if (dst_reg 'op' src_reg) goto pc + off16 */
#define BPF_JMP_REG(OP, DST, SRC, OFF) \
((struct bpf_insn) { \
.code = BPF_JMP | BPF_OP(OP) | BPF_X, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = OFF, \
.imm = 0 })
/* Conditional jumps against immediates, if (dst_reg 'op' imm32) goto pc + off16 */
#define BPF_JMP_IMM(OP, DST, IMM, OFF) \
((struct bpf_insn) { \
.code = BPF_JMP | BPF_OP(OP) | BPF_K, \
.dst_reg = DST, \
.src_reg = 0, \
.off = OFF, \
.imm = IMM })
/* Raw code statement block */
#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM) \
((struct bpf_insn) { \
.code = CODE, \
.dst_reg = DST, \
.src_reg = SRC, \
.off = OFF, \
.imm = IMM })
/* Program exit */
#define BPF_EXIT_INSN() \
((struct bpf_insn) { \
.code = BPF_JMP | BPF_EXIT, \
.dst_reg = 0, \
.src_reg = 0, \
.off = 0, \
.imm = 0 })
#endif /* !LIBBPF_MACROS_H */
Raw
load_signal.c
/*
* barrierd is a daemon that keeps track of the most recent time we
* know for sure that each core was *not* executing userspace code.
*
* Let t_qmin be the oldest such time across all CPUs. Knowing t_qmin,
* userspace code can assume that any instruction still in flight
* began execution after that time, and thus, under the x86-TSO memory
* model, that any of write that retired before t_qmin must be
* globally visible. This approach is particularly interesting for an
* out-of-kernel implementation because lost events do not affect
* correctness, only how quickly we conclude global visibility.
*
* I considered obtaining this information by polling various /proc
* files. The problem is that the files that are available are either
* slow to read, suffer from coarse granularity, or track too few
* cases of non-userspace execution.
*
* barrierd tracks t_qmin with a per-CPU array updated by an eBPF
* program that is attached to an arbitrary set of tracepoint (eBPF
* code runs in the kernel, so any time the program runs, userspace
* was interrupted).
*
* Having eBPF update a per-CPU uint64_t[1] suffices to determine
* t_qmin. However, polling on the BPF array is less than ideal. The
* eBPF program also pushes a nonsense perf event the first time that
* each CPU update its timestamp above the previous t_qmin (or any
* other watermark set by userspace). barrierd can now simply track
* the POLLIN status of the corresponding perf file descriptors to
* know when to re-read the BPF array. epoll gives us a trivial
* solution, given that we don't care about the polled fds' data.
*
* TODO: make sure we have a story for torn reads.
*
* TODO: expose a useful interface for apps.
*
* 1. read-only mmap-able file.
* 2. futex on updates to t_qmin
* 3. have a mode that works on versions instead of CLOCK_MONOTONIC.
* 4. consider array of futexes to get more precise wakeup conditions
* 5. add a minimum wait between array reads
* 6. let apps signal something (futex?) to turn off min wait until
* t_qmin is > time-of-signal.
* -> Could do that by making the bpf code signal when old <=
* watermark and newc > watermark, then setting the watermark to
* the desired t_qmin, but that loses updates. Set the watermark
* to min(desired t_qmin, now + min_wait).
*/
#define _GNU_SOURCE
#include <asm/unistd.h>
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/perf_event.h>
#include <linux/version.h>
#include <poll.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include "cycle.h"
#include "libbpf-macros.h"
/* XXX: consider parsing /proc/mount. */
#define DEBUGFS "/sys/kernel/debug/tracing/"
#define RING_PAGE_CNT 1
struct state {
uint32_t ncpu;
int per_cpu_map_fd;
int trigger_map_fd;
int perf_map_fd;
int prog_fd;
struct pollfd perf_event_fds[1024];
};
static inline int bpf(enum bpf_cmd cmd, union bpf_attr *attr,
unsigned int size)
{
return syscall(__NR_bpf, cmd, attr, size);
}
static inline int perf_event_open(struct perf_event_attr *attr,
pid_t pid, int cpu, int group_fd,
unsigned long flags)
{
return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);
}
static int create_map(enum bpf_map_type type, uint32_t key_size,
uint32_t value_size, uint32_t max_entries)
{
union bpf_attr attr = {
.map_type = type,
.key_size = key_size,
.value_size = value_size,
.max_entries = max_entries,
};
int fd;
fd = bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
perror("map");
return fd;
}
static int load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
uint32_t insn_cnt, const char *license,
char *log_buf, uint32_t log_buf_sz)
{
int fd;
union bpf_attr attr = {
.prog_type = type,
.insn_cnt = insn_cnt,
.insns = (uintptr_t)insns,
.license = (uintptr_t)license,
.log_buf = (uintptr_t)log_buf,
.log_size = log_buf_sz,
.log_level = 1,
.kern_version = LINUX_VERSION_CODE,
.prog_flags = BPF_F_STRICT_ALIGNMENT,
};
if (log_buf != NULL) {
log_buf[0] = '\0';
}
fd = bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
perror("foo");
printf("Log\n%.*s\n", (int)log_buf_sz, log_buf);
return fd;
}
static int create_perf_fd(struct state *state, size_t cpu)
{
struct perf_event_attr attr = {
.sample_type = PERF_SAMPLE_RAW,
.type = PERF_TYPE_SOFTWARE,
.config = PERF_COUNT_SW_BPF_OUTPUT,
.sample_period = 1,
.wakeup_events = 1,
};
int r;
assert(cpu < state->ncpu);
r = perf_event_open(&attr, -1, cpu, -1, PERF_FLAG_FD_CLOEXEC);
if (r < 0) {
perror("create_perf_fd:");
return r;
}
state->perf_event_fds[cpu].fd = r;
return 0;
}
static int map_perf_fd(const struct state *state, size_t cpu)
{
uint32_t key = cpu;
union bpf_attr attr = {
.map_fd = state->perf_map_fd,
.key = (uintptr_t)&key,
.value = (uintptr_t)&state->perf_event_fds[cpu].fd,
.flags = BPF_ANY,
};
int r;
assert(mmap(NULL, (1 + 1) * 4096, PROT_READ, MAP_SHARED,
state->perf_event_fds[cpu].fd, 0)
!= MAP_FAILED);
r = ioctl(state->perf_event_fds[cpu].fd, PERF_EVENT_IOC_ENABLE, 0);
if (r < 0) {
perror("map_perf_fd ioctl:");
return r;
}
r = bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
if (r < 0) {
perror("map_perf_fd: ");
return r;
}
return 0;
}
static int populate_perf_fds(struct state *state)
{
int r;
for (size_t i = 0; i < state->ncpu; i++) {
r = create_perf_fd(state, i);
if (r < 0) {
return r;
}
r = map_perf_fd(state, i);
if (r < 0) {
return r;
}
}
return 0;
}
static int populate_state_maps(struct state *state)
{
int r;
r = create_map(BPF_MAP_TYPE_PERCPU_ARRAY,
sizeof(uint32_t), sizeof(uint64_t),
1);
if (r < 0) {
perror("create_map BPF_MAP_TYPE_PERCPU_ARRAY:");
return r;
}
state->per_cpu_map_fd = r;
r = create_map(BPF_MAP_TYPE_ARRAY,
sizeof(uint32_t), sizeof(uint64_t),
1);
if (r < 0) {
perror("create_map BPF_MAP_TYPE_ARRAY:");
return r;
}
state->trigger_map_fd = r;
r = create_map(BPF_MAP_TYPE_PERF_EVENT_ARRAY,
sizeof(uint32_t), sizeof(int),
state->ncpu);
if (r < 0) {
perror("create_map BPF_MAP_TYPE_PERF_EVENT_ARRAY");
return r;
}
state->perf_map_fd = r;
return 0;
}
static int populate_state_prog(struct state *state)
{
char buf[65536] = { 0 };
int per_cpu_map_fd = state->per_cpu_map_fd;
int trigger_map_fd = state->trigger_map_fd;
int perf_map_fd = state->perf_map_fd;
int r;
const struct bpf_insn prog[] = {
# include "signal.epbf.inc"
};
r = load_program(BPF_PROG_TYPE_TRACEPOINT,
prog, sizeof(prog) / sizeof(prog[0]),
"Dual BSD/GPL", buf, sizeof(buf));
if (r < 0) {
perror("load_program BPF_PROG_TYPE_TRACEPOINT: ");
return r;
}
state->prog_fd = r;
return 0;
}
static int populate_state(struct state *state)
{
int r;
r = populate_state_maps(state);
if (r < 0) {
return r;
}
r = populate_perf_fds(state);
if (r < 0) {
return r;
}
return populate_state_prog(state);
}
static int id_for_event(const char *event)
{
char buf[128];
char *path = NULL;
int id_fd = -1;
int r;
r = asprintf(&path, "%s/events/%s/id", DEBUGFS, event);
if (r < 0) {
perror("asprintf: ");
goto out;
}
id_fd = open(path, O_RDONLY, 0);
if (id_fd < 0) {
perror("open: ");
goto out;
}
r = read(id_fd, buf, sizeof(buf));
if ((size_t)r >= sizeof(buf)) {
printf("Bad read for event %s\n", event);
r = -1;
goto out;
}
r = atoi(buf);
out:
if (id_fd >= 0) {
close(id_fd);
}
free(path);
return r;
}
static int attach_filter_to_tracepoint(const struct state *state,
const char *event)
{
struct perf_event_attr attr = {
.type = PERF_TYPE_TRACEPOINT,
.sample_type = PERF_SAMPLE_RAW,
.sample_period = 1,
.wakeup_events = 1,
};
int perf_fd;
int r;
r = id_for_event(event);
if (r < 0) {
return r;
}
attr.config = r;
/* tracepoint events always disregard the CPU(really?). */
perf_fd = perf_event_open(&attr, /*pid=*/-1, /*cpu=*/0,
/*group=*/-1, /*flags=*/0);
if (perf_fd < 0) {
perror("perf_event_open: ");
return perf_fd;
}
r = ioctl(perf_fd, PERF_EVENT_IOC_SET_BPF, state->prog_fd);
if (r < 0) {
perror("ioctl PERF_EVENT_IOC_SET_BPF: ");
return r;
}
r = ioctl(perf_fd, PERF_EVENT_IOC_ENABLE, 0);
if (r < 0) {
perror("ioctl PERF_EVENT_IOC_ENABLE: ");
return r;
}
return 0;
}
static int state_set_trigger(const struct state *state, uint64_t ts)
{
uint32_t key = 0;
union bpf_attr attr = {
.map_fd = state->trigger_map_fd,
.key = (uintptr_t)&key,
.value = (uintptr_t)&ts,
.flags = BPF_ANY,
};
int r;
r = bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
if (r < 0) {
perror("state_set_trigger: ");
return r;
}
return 0;
}
static int print_now(void)
{
struct timespec now;
int r;
r = clock_gettime(CLOCK_MONOTONIC, &now);
if (r < 0) {
perror("clock_gettime: ");
return r;
}
printf("Now: %.3f %"PRIu64"\n",
now.tv_sec + 1e-9 * now.tv_nsec,
now.tv_sec * (1000 * 1000 * 1000UL) + now.tv_nsec);
return 0;
}
static int state_read_values(const struct state *state,
uint64_t *OUT_min_value)
{
uint64_t values[1024];
uint64_t min_value = UINT64_MAX;
uint32_t key = 0;
union bpf_attr attr = {
.map_fd = state->per_cpu_map_fd,
.key = (uintptr_t)&key,
.value = (uintptr_t)values,
};
int r;
ticks begin = getticks();
r = bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
if (r < 0) {
perror("state_read_values: ");
return r;
}
for (size_t i = 0; i < state->ncpu; i++) {
if (values[i] != 0 && 0) {
printf("%zu: %"PRIu64", ", i, values[i]);
}
if (values[i] < min_value) {
min_value = values[i];
}
}
printf("Read time: %.0f\n", elapsed(getticks(), begin));
printf("\t min: %"PRIu64"\n", min_value);
*OUT_min_value = min_value;
return 0;
}
static void drain_fd(int fd)
{
char buf[4096];
int r;
r = read(fd, buf, sizeof(buf));
/* printf("drained %i %"PRIu64"\n", r, *(uint64_t*)buf); */
assert((size_t)r < sizeof(buf));
assert(r >= 0 || errno == EINTR);
}
static void wait_for_events(struct state *state)
{
int r;
for (size_t i = 0; i < state->ncpu; i++) {
state->perf_event_fds[i].events = POLLIN;
state->perf_event_fds[i].revents = 0;
}
r = poll(state->perf_event_fds, state->ncpu, 1000);
assert(r >= 0 || errno == EINTR);
for (size_t i = 0; i < state->ncpu; i++) {
if (state->perf_event_fds[i].revents != 0 && 0) {
/* printf("revents %zu 0x%x\t", */
/* i, */
/* state->perf_event_fds[i].revents); */
drain_fd(state->perf_event_fds[i].fd);
}
}
}
int main ()
{
/* XXX: parse /sys/devices/system/cpu/possible */
struct state state = { .ncpu = 24 };
uint64_t min_ts = 1;
assert(populate_state(&state) == 0);
assert(state_set_trigger(&state, min_ts) == 0);
assert(attach_filter_to_tracepoint(
&state, "irq/softirq_entry") == 0);
assert(attach_filter_to_tracepoint(
&state, "irq_vectors/local_timer_entry") == 0);
assert(attach_filter_to_tracepoint(
&state, "raw_syscalls/sys_enter") == 0);
for (size_t i = 0; i < 10; i++) {
wait_for_events(&state);
assert(print_now() == 0);
assert(state_read_values(&state, &min_ts) == 0);
assert(state_set_trigger(&state, min_ts) == 0);
}
usleep(2000);
assert(print_now() == 0);
assert(state_read_values(&state, &min_ts) == 0);
return 0;
}
Raw
signal.ebpf.inc
/*
* On any event: stores the current timestamp in the CPU's array, and
* enqueues a perf event if the previous timestamp was less than or
* equal to trigger_map[0].
*
* The event is conditional to let userspace define the set of CPUs it
* cares about: we only want to be woken up for a CPU the first time
* its timestamp crosses the previous min quiescent time.
*
* This additional complexity in the eBPF program means userspace is
* free to ask for a wakeup on every event, without risking event
* storms.
*/
#define BPF_CALL_FN(function) \
BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_##function)
/*
* Result: r0
* Args: r1, r2, r3, r4, r5
* Callee-save: r6, r7, r8, r9
* frame pointer: r10 (read-only).
*
* ctxp: context from arg1 into r6
* nilp: pointer to 0 in r7
* prev: previous ts in r8
* curp: pointer to current ts in r9
*/
#define res BPF_REG_0
#define arg1 BPF_REG_1
#define arg2 BPF_REG_2
#define arg3 BPF_REG_3
#define arg4 BPF_REG_4
#define arg5 BPF_REG_5
#define fp BPF_REG_10
#define ctxp BPF_REG_6
#define nilp BPF_REG_7
#define prev BPF_REG_8
#define curp BPF_REG_9
/*
* # Stash the context
* 000: mov64 ctxp, arg1
*/
BPF_MOV64_REG(ctxp, arg1),
/*
* # Set up the pointer to 0
* 001: st_dw fp[-8], 0
* 002: mov64 nilp, fp
* 003: add64 nilp, -8
*/
BPF_ST_MEM(BPF_DW, fp, -8, 0),
BPF_MOV64_REG(nilp, fp),
BPF_ALU64_IMM(BPF_ADD, nilp, -8),
/*
* # curp = &bpf_ktime_get_ns
* 004: call ktime_get_ns
* 005: st_dw fp[-16], res
* 006: mov64 curp, fp
* 007: add64 curp, -16
*/
BPF_CALL_FN(ktime_get_ns),
BPF_STX_MEM(BPF_DW, fp, res, -16),
BPF_MOV64_REG(curp, fp),
BPF_ALU64_IMM(BPF_ADD, curp, -16),
/*
* # prev = *bpf_map_lookup_elem(per_cpu_map_fd, nilp)
* 008: ld_map_fd arg1, $per_cpu_map_fd
* 010: mov64 arg2, nilp
* 011: call bpf_map_lookup_elem # ($per_cpu_map_fd, nilp)
* 012: jeq res, 0, OUT # offset: 28 - 12 - 1 = 16
* 013: ld_dw prev, res
*/
BPF_LD_MAP_FD(arg1, per_cpu_map_fd),
BPF_MOV64_REG(arg2, nilp),
BPF_CALL_FN(map_lookup_elem),
BPF_JMP_IMM(BPF_JEQ, res, 0, 15),
BPF_LDX_MEM(BPF_DW, prev, res, 0),
/*
* # bpf_map_update_elem(per_cpu_map_fd, &nilp, &now, BPF_ANY);
* 014: ld_map_fd arg1, $per_cpu_map_fd
* 016: mov64 arg2, nilp
* 017: mov64 arg3, curp
* 018: mov64 arg4, BPF_ANY
* 019: call bpf_map_update_elem
*/
BPF_LD_MAP_FD(arg1, per_cpu_map_fd),
BPF_MOV64_REG(arg2, nilp),
BPF_MOV64_REG(arg3, curp),
BPF_MOV64_IMM(arg4, BPF_ANY),
BPF_CALL_FN(map_update_elem),
/*
* # prev = bpf_map_lookup_elem(trigger_map_fd, &nilp)
* # if prev <= trigger, signal perf
* 020: ld_map_fd arg1, $trigger_map_fd
* 022: mov64 arg2, nilp
* 023: call bpf_map_lookup_elem # (trigger_map_fd, nilp)
* 024: jeq res, 0, OUT # offset: 4 - 1
* 025: ld_dw res, res
* 026: jle prev, res, SIGNAL # offset: 3 - 1
* 027: mov64 res, 0
* OUT
* 028: exit
*/
BPF_LD_MAP_FD(arg1, trigger_map_fd),
BPF_MOV64_REG(arg2, nilp),
BPF_CALL_FN(map_lookup_elem),
BPF_JMP_IMM(BPF_JEQ, res, 0, 3),
BPF_LDX_MEM(BPF_DW, res, res, 0),
BPF_JMP_REG(BPF_JLE, prev, res, 2),
BPF_MOV64_IMM(res, 0),
BPF_EXIT_INSN(),
/* SIGNAL
* # bpf_perf_event_output(ctx, perf_map_fd, BPF_F_CURRENT_CPU,
* curp, sizeof(u64))
* 029: mov64 arg1, ctxp
* 030: ld_map_fd arg2, $perf_map_fd
* 032: mov64 arg3, BPF_F_CURRENT_CPU
* 033: mov64 arg4, curp
* 034: mov64 arg5, 8
* 035: call bpf_perf_event_output
* 036: exit
*/
BPF_MOV64_REG(arg1, ctxp),
BPF_LD_MAP_FD(arg2, perf_map_fd),
BPF_LD_IMM64(arg3, BPF_F_CURRENT_CPU),
BPF_MOV64_REG(arg4, curp),
BPF_MOV64_IMM(arg5, sizeof(uint64_t)),
BPF_CALL_FN(perf_event_output),
BPF_EXIT_INSN(),
#undef res
#undef arg1
#undef arg2
#undef arg3
#undef arg4
#undef arg5
#undef fp
#undef ctxp
#undef nilp
#undef prev
#undef curp
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment