This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# HTTP crawler inside nginx. | |
# | |
# start crawling by curl http://127.0.0.1:18080/?host=<START URL> | |
# | |
worker_processes auto; | |
events { | |
worker_connections 16384; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16:09:47.418536 IP 10.1.25.156.58918 > 10.1.25.1.53: 64736+ A? releases.hashicorp.com. (40) | |
0x0000: 6c3b 6b8c e70a a45e 60d7 176d 0800 4500 l;k....^`..m..E. | |
0x0010: 0044 682a 0000 4011 cbe0 0a01 199c 0a01 .Dh*..@......... | |
0x0020: 1901 e626 0035 0030 25ee fce0 0100 0001 ...&.5.0%....... | |
0x0030: 0000 0000 0000 0872 656c 6561 7365 7309 .......releases. | |
0x0040: 6861 7368 6963 6f72 7003 636f 6d00 0001 hashicorp.com... | |
0x0050: 0001 .. | |
16:09:47.421645 IP 10.1.25.1.53 > 10.1.25.156.58918: 64736 5/13/13 CNAME dualstack.s.shared.global.fastly.net., A 151.101.129.183, A 151.101.193.183, A 151.101.1.183, A 151.101.65.183 (570) | |
0x0000: a45e 60d7 176d 6c3b 6b8c e70a 0800 4500 .^`..ml;k.....E. | |
0x0010: 0256 78fa 0000 4011 b8fe 0a01 1901 0a01 .Vx...@......... |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# --------------- cacher ---------------------------------- | |
server { | |
listen 80 ; | |
server_name static ; | |
keepalive_timeout 70; | |
access_log /dev/null ; |