Sample program that exhibits extremely slow msync() on ZFS.

msync-multisize.c

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
#include <unistd.h>

static const size_t MAX_FILE_SIZE = 32 * 1024 * 1024;
static const size_t STEP_MULTIPLIER = 2;

static double get_time()
{
        struct timeval tv;
        gettimeofday(&tv, NULL);
        return ((double)tv.tv_sec) + (((double)tv.tv_usec) / 1000000);
}

int main(int argc, char** argv)
{
        const char* name = (argc >= 1) ? argv[0] : "msync";

        if (argc < 2)
        {
                fprintf(stderr, "usage: %s <file>\n", name);
                return EXIT_FAILURE;
        }

        const char* path = argv[1];

        for (size_t size = 1; size <= MAX_FILE_SIZE; size *= STEP_MULTIPLIER) {
                int fd = open(path, O_RDWR | O_CREAT, 0777);
                if (fd == -1)
                {
                        perror(name);
                        return EXIT_FAILURE;
                }

                void* mapping = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, 0);
                if (mapping == MAP_FAILED)
                {
                        perror(name);
                        return EXIT_FAILURE;
                }

                for (size_t pos = 0; pos < size; ++pos)
                        ((uint8_t*)mapping)[pos] = 0xff;

                printf("msync(%10lu bytes) ", (unsigned long) size);
                double start = get_time();
                if (msync(mapping, size, MS_SYNC) != 0)
                {
                        perror(name);
                        return EXIT_FAILURE;
                }
                printf("%10.3lf seconds\n", get_time() - start);

                if (close(fd) != 0)
                {
                        perror(name);
                        return EXIT_FAILURE;
                }
        }

        return EXIT_SUCCESS;
}