Using perf_event_open to monitor Docker containers - performance

Question

I wrote a program in C to retrieve performance events such as cpu-cycles of Docker containers.
I mean, a user space program at host level (host level monitoring, not inside docker). I give pid of the docker container as pid entry of perf_event_open(), However, I have always 0 as returned value. I have tested the program for other non-docker pids for example firefox and it works very well.
I set PERF_FLAG_PID_CGROUP as flag, nothing changes!
Here is the code:
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <linux/perf_event.h>
#include <asm/unistd.h>
static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags)
{
int ret;
ret = syscall(__NR_perf_event_open, hw_event, pid, cpu,
group_fd, flags);
return ret;
}
int
main(int argc, char **argv)
{
struct perf_event_attr pe;
long long count;
int fd;
fd = open("/sys/fs/cgroup/perf_event/docker/f42c13cd9dd700544fe670e30d0b3216bdceaf01ddc370405618fdecfd10b26d", O_RDONLY);
if (fd == -1)
return 0;
memset(&pe, 0, sizeof(struct perf_event_attr));
pe.type = PERF_TYPE_HARDWARE;
pe.size = sizeof(struct perf_event_attr);
pe.config = PERF_COUNT_HW_CPU_CYCLES;
pe.disabled = 1;
pe.exclude_kernel = 0;
pe.exclude_hv = 0;
fd = perf_event_open(&pe, fd, -1, -1, PERF_FLAG_PID_CGROUP);
if (fd == -1) {
fprintf(stderr, "Error opening leader %llx\n", pe.config);
exit(EXIT_FAILURE);
}
ioctl(fd, PERF_EVENT_IOC_RESET, 0);
ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
usleep(100);
ioctl(fd, PERF_EVENT_IOC_DISABLE, 0);
read(fd, &count, sizeof(long long));
printf("Used %lld instructions\n", count);
close(fd);
}
According to the Man page of perf_event_open(), I also give the fd opened on the directory of docker container in groupfs. Doesn't work !
Would you please help me to solve the problem?
Thanks
Update:
I have checked with other events for example PERF_COUNT_HW_CACHE_REFERENCES,
I see 0 as returned value!
OS: Ubuntu 16.04
Kernel: 4.15.0-041500-generic
Architecture: X86_64

Answer 1

You have not specified which linux kernel version you are using. I will base my answer as per the latest linux kernel version.
The parameters that you passed to perf_event_open syscall look correct, except for one.
In your case, you pass cpu = -1 as a parameter to perf_event_open.
While this usually works in normal perf event filtering (i.e. per-cpu or per-thread), passing cpu = -1 will not work in cgroup based filtering for perf. In cgroup mode, the pid argument is used to pass the fd opened to the cgroup directory in cgroupfs (which you seemed to have passed correctly). The cpu argument designates the cpu on which to monitor threads from that cgroup. And when cpu=-1 it means the perf event measures the specified process/thread on any CPU (without regarding whether that CPU belongs to the cgroup you are measuring on).
This is how it is depicted in the latest Linux code.
if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
return -EINVAL;
You can clearly see that if either PID=-1 or CPU=-1, the syscall method will return error.
Working Example
From the perf_event_open documentation, it is quite clear that --
cgroup monitoring is available only for system-wide events and may therefore require extra permissions.
Since we are doing cgroup monitoring in this case, which is already understood as we are monitoring the perf-events of a container, we will have to monitor the event system-wide. This means the monitoring happens for ALL the available CPUs in the system.
Working code
I have 4 cores in my system and hence I use CPUs - 0,1,2,3
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <linux/perf_event.h>
#include <asm/unistd.h>
#include <errno.h>
static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags)
{
int ret;
ret = syscall(__NR_perf_event_open, hw_event, pid, cpu,
group_fd, flags);
return ret;
}
int
main(int argc, char **argv)
{
struct perf_event_attr pe;
long long count, count1, count2, count3;
int fd, fd1, fd2, fd3, fd4;
fd1 = open("/sys/fs/cgroup/perf_event/docker/001706b1a71617b0ce9d340f706d901e00ee398091dd62aded2a1863fc8c274a", O_RDONLY);
if (fd1 == -1)
return 0;
memset(&pe, 0, sizeof(struct perf_event_attr));
pe.type = PERF_TYPE_HARDWARE;
pe.size = sizeof(struct perf_event_attr);
pe.config = PERF_COUNT_HW_INSTRUCTIONS;
pe.disabled = 1;
pe.exclude_kernel = 0;
pe.exclude_hv = 0;
fd = perf_event_open(&pe, fd1, 0, -1, PERF_FLAG_PID_CGROUP|PERF_FLAG_FD_CLOEXEC);
if (fd == -1) {
fprintf(stderr, "Error opening leader: %s\n", strerror(errno));
exit(EXIT_FAILURE);
}
fd2 = perf_event_open(&pe, fd1, 1, -1, PERF_FLAG_PID_CGROUP|PERF_FLAG_FD_CLOEXEC);
if (fd2 == -1) {
fprintf(stderr, "Error: %s\n", strerror(errno));
exit(EXIT_FAILURE);
}
fd3 = perf_event_open(&pe, fd1, 2, -1, PERF_FLAG_PID_CGROUP|PERF_FLAG_FD_CLOEXEC);
if (fd3 == -1) {
fprintf(stderr, "Error: %s\n", strerror(errno));
exit(EXIT_FAILURE);
}
fd4 = perf_event_open(&pe, fd1, 3, -1, PERF_FLAG_PID_CGROUP|PERF_FLAG_FD_CLOEXEC);
if (fd4 == -1) {
fprintf(stderr, "Error: %s\n", strerror(errno));
exit(EXIT_FAILURE);
}
ioctl(fd, PERF_EVENT_IOC_RESET, 0);
ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
ioctl(fd2, PERF_EVENT_IOC_RESET, 0);
ioctl(fd2, PERF_EVENT_IOC_ENABLE, 0);
ioctl(fd3, PERF_EVENT_IOC_RESET, 0);
ioctl(fd3, PERF_EVENT_IOC_ENABLE, 0);
ioctl(fd4, PERF_EVENT_IOC_RESET, 0);
ioctl(fd4, PERF_EVENT_IOC_ENABLE, 0);
sleep(10); // using sleep(10) to actually observe instructions
ioctl(fd, PERF_EVENT_IOC_DISABLE, 0);
ioctl(fd2, PERF_EVENT_IOC_DISABLE, 0);
ioctl(fd3, PERF_EVENT_IOC_DISABLE, 0);
ioctl(fd4, PERF_EVENT_IOC_DISABLE, 0);
read(fd, &count, sizeof(long long));
read(fd2, &count1, sizeof(long long));
read(fd3, &count2, sizeof(long long));
read(fd4, &count3, sizeof(long long));
printf("Used %lld instructions\n", count+count1+count2+count3);
close(fd);
close(fd2);
close(fd3);
close(fd4);
}
`
Output : Used 55174 instructions