Nginx proxy cache config to *not* cache content for bulk requesters such as Google, given limited disk space for cache.

gistfile1.txt

# Create a variable $bulk_agent by matching against $http_user_agent

map $http_user_agent $bulk_agent {
  default   0;
  ~*Google* 1;
  ~*Wget*   1;
  ~*Ruby*   1;
  ~*Java*   1;
}

# Log the result of the above for debugging purposes (see $bulk_agent in log_format).
log_format cache '$remote_addr [$time_local] '
             '$upstream_cache_status '
             'Cache-Control: $upstream_http_cache_control '
             'Expires: $upstream_http_expires '
             '"$request" ($status) '
             '"$http_user_agent" $bulk_agent';

location / {
  proxy_pass http://static;
  proxy_cache main;
  proxy_cache_key $cache_key;
  proxy_cache_valid 30m; 
  proxy_cache_use_stale error
        timeout
        invalid_header
        http_500
        http_502
        http_504
        http_404;
  #proxy_cache_bypass $bulk_agent; # Do not serve cached data
  proxy_no_cache $bulk_agent;      # Do not cache the response
}