Created
January 30, 2020 15:35
-
-
Save elazarl/aaf45f52dbca7e2cc6df33feebb22a91 to your computer and use it in GitHub Desktop.
Check if a simple transform before applying zstd reduce file size
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <stdio.h> | |
#include <stdlib.h> | |
#include <stdint.h> | |
#include <fcntl.h> | |
#include <sys/types.h> | |
#include <sys/stat.h> | |
#include <time.h> | |
#include <unistd.h> | |
char buf[1024*1024]; | |
volatile uint64_t sink; | |
#define USEC_IN_SEC 1000000ULL | |
#define NSEC_IN_USEC 1000ULL | |
int read_all(char* buf, size_t buf_size, char *file_name); | |
int main(int argc, char** argv) { | |
int narg; | |
if (argc < 2) { | |
fputs("too little argumetns\n", stdout); | |
return 1; | |
} | |
uint64_t total_usec = 0; | |
uint64_t total_bytes = 0; | |
for (narg = 1; narg < argc; narg++) { | |
uint64_t i; | |
int file_size_bytes = read_all(buf, sizeof(buf), argv[narg]); | |
uint64_t* ptr = (uint64_t*)&buf[0]; | |
struct timespec before, after; | |
printf("running %d iterations\n", file_size_bytes); | |
clock_gettime(CLOCK_MONOTONIC, &before); | |
__sync_synchronize(); | |
for (i=1; i<file_size_bytes/sizeof(*ptr); i++) { | |
ptr[i] = ptr[i-1]-ptr[i]; | |
} | |
__sync_synchronize(); | |
clock_gettime(CLOCK_MONOTONIC, &after); | |
total_bytes += file_size_bytes; | |
total_usec += ((uint64_t)after.tv_sec - before.tv_sec) * USEC_IN_SEC + | |
((uint64_t)after.tv_nsec - before.tv_nsec) / 1000; | |
// make sure compile is obligated to do the above change | |
for (i=0; i<file_size_bytes/sizeof(*ptr); i++) { | |
sink = ptr[i]; | |
} | |
} | |
const double bytes_in_MB = 1024 * 1024; | |
printf("%lu bytes (%.03f MB) took %lu usecs\n", total_bytes, total_bytes / bytes_in_MB, total_usec); | |
double bytes_in_sec = ((double)total_bytes / (double)total_usec) * USEC_IN_SEC; | |
printf("%.03f MB in sec\n", bytes_in_sec / bytes_in_MB); | |
} | |
int read_all(char* buf, size_t buf_size, char *file_name) { | |
int fd = open(file_name, O_RDONLY); | |
if (fd < 0) { | |
perror("error opening file"); | |
exit(2); | |
} | |
size_t bytes_read_so_far = 0; | |
ssize_t bytes_read; | |
while ((bytes_read = read(fd, buf+bytes_read_so_far, buf_size-bytes_read_so_far)) != 0) { | |
if (bytes_read < 0) { | |
perror("error reading file"); | |
close(fd); | |
exit(1); | |
} | |
bytes_read_so_far += bytes_read; | |
} | |
close(fd); | |
return bytes_read_so_far; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment