test_whitelist.rb

# this tests a critical regex used in the 'Whitelist Query Params' config in
# Fastly, to transform request URLs into a normalized version that strips
# 'traffic partner' tracking params and other noise, but leaves any valid keys
# that match the params we actually use. A lot hinges on the proper operation
# of this regex to keep hit ratio high but leave needed params intact.
#
# This version leaves trailing ? and &, which is unavoidable now but are pulled
# out by a simple second sweep.
#
# http://regex101.com/r/oJ2gI6/10
# map of original urls to transformed version. If value is nil it should be unchanged.
URLS = {
  'http://ovens.reviewed.com/?utm_source=VSW&utm_medium=PPC' =>
  'http://ovens.reviewed.com/',

  '/blah/foo?1234-1234_6&brand=foo&utm_source=1234&type=camera&page=5&utm_medium=blah&foo&1234-1234_0' =>
  '/blah/foo?brand=foo&type=camera&page=5',

  '/blah/foo?nonsense=42&fooparam' =>
  '/blah/foo',

  '/blah/foo?&2938120-12923_3&utm_source=GatesOfHell&1=2' =>
  '/blah/foo',

  '/blah/foo?tags=Ice+Dispenser&page=42&fooparam=19&fifi' =>
  '/blah/foo?tags=Ice+Dispenser&page=42',

  'http://staging.espressomakerinfo.com/how_we_test?&11163888-449553_6&brand=123&n23=12&xxxxxxxxxxxxxxx&page=23' =>
  'http://staging.espressomakerinfo.com/how_we_test?&brand=123&page=23',

  'http://dishwashers.reviewed.com/features/goodbye-granite-the-6-hottest-countertop-finishes?utm_source=taboola&utm_medium=USAT%20Recirc&' =>
  'http://dishwashers.reviewed.com/features/goodbye-granite-the-6-hottest-countertop-finishes',

  'http://refrigerators.reviewed.com/features/seriously-stop-refrigerating-these-foods?utm_source=TB_paid&utm_medium=cpc&' =>
  'http://refrigerators.reviewed.com/features/seriously-stop-refrigerating-these-foods',

  'http://ovens.reviewed.com/news/with-one-chip-ge-creates-diy-smart-appliances?utm_source=usat&utm_medium=referral&utm_campaign=collab&' =>
  'http://ovens.reviewed.com/news/with-one-chip-ge-creates-diy-smart-appliances',

  '/content/lg-65ec9700-4k-oled-tv-first-impressions-review?utm_medium=referral&utm_source=feedblitz&utm_campaign=usatoday-techtopstories' =>
  '/content/lg-65ec9700-4k-oled-tv-first-impressions-review',

  'http://televisions.reviewed.com/features/the-ultimate-movie-player-starts-at-4000?utm_medium=referral&utm_source=dlvr.it.test&dlvrit=384245' =>
  'http://televisions.reviewed.com/features/the-ultimate-movie-player-starts-at-4000',

  'http://laundry.reviewed.com/?utm_source=Adwords&utm_medium=PPC-intoff&gclid=CjwKEAjwkf-gBRCd-b2m2aOo0EQSJABMeQDkU6Wmd-QCSogFgFbcYZzVrk9c99AtrRdj43rZzJ67pRoCtqnw_wcB' =>
  'http://laundry.reviewed.com/',

  '/features/bone-skulls-and-harmony-check-out-these-french-speakers?amp;utm_medium=referral&utm_campaign=collab' =>
  '/features/bone-skulls-and-harmony-check-out-these-french-speakers',

  '/features/bone-skulls-and-harmony-check-out-these-french-speakers?amp;utm_medium' =>
  '/features/bone-skulls-and-harmony-check-out-these-french-speakers',

  '/seriously-stop-refrigerating-these-foods?utm_source=TB_paid&utm_medium=cpc&' =>
  '/seriously-stop-refrigerating-these-foods',

  'http://laundry.reviewed.com/types/front-loading?page=1&_escaped_fragment_=' => nil,

  '/ads/interstitial_1?adid=stv_01' => nil,

  'http://www.reviewed.com/' => nil,

  'http://vacuums.reviewed.com//d/reviews/hoover-duros.htm?order_by=msrp%252Cdesc' => nil,

  '/ratings.htm?keywords=Daewoo+Steam+4.5CuFt+Washer%2C+7.3CuFt+GAS+Dryer+1%2C300+Spin+Speed%2C+Smart+Detergent+System&order_by=rating%2Cdesc' => nil,

  '/something.com/articles/12875/attachments.json?tags=Front+Photo%2CHandle+Photo%2CFingerprints+Photo%2CWater%2FIce+Dispenser+Photo%2CControls+Photo%2CWater%2FIce+Dispenser+Controls+Photo%2CInterior+Photo%2CRefrigerator+Main+1+Image%2CRefrigerator+Main+2+Image%2CRefrigerator+Main+3+Image%2CVegetable+Drawer+Photo%2CRefrigerator+Door+1-1+Image%2CIce+Maker+Photo%2CWater+Filter+Photo%2CRefrigerator+Door+2-1+Image%2CFreezer+Main+1+Image%2CFreezer+Door+1+Image%2CSides+Photo%2CBack+Photo&page=1&per_page=6' => nil,

  '/brands/hifiman?brand=hifiman' => nil,

}



tests, expectations = ["",""]
URLS.each do |source, res|
  tests << "\ntxreq -url \"#{source}\""
  expectations << "\nrxreq\nexpect req.url == \"#{res || source}\"\n"
end

vcltest = <<EOt

varnishtest "#1030"

server s1 {
        #{expectations}
} -start

varnish v1 -vcl+backend {
        sub vcl_recv {
		  if( req.url ) {
			set req.url = regsuball(req.url, "(^|&|\\?)(?!_escaped_fragment_|brand|branch|type|min_msrp|max_msrp|_type|keywords|page|per_page|tags|name|order_by|adid|return_url|event|classifications|zip|website_ids)([^&\\/=]+)(=[^&]+)?(?:(&|$))", "&");
			set req.url = regsuball(req.url, "&{2,}", "");
			if (req.url ~ "&" && !(req.url ~ "\\?")) {
				set req.url = regsub(req.url, "&", "?");
			}
		  }
        }
} -start

client c1 {
        #{tests}
} -run

EOt

filename = 'test-whitelist.vcl'

File.write("/tmp/#{filename}", vcltest)

system("sudo docker run --rm -v /tmp/#{filename}:/opt/#{filename}:ro zenedith/varnish /bin/sh -c 'ldconfig; varnishtest /opt/#{filename}'")