-
-
Save hamishforbes/ea0e9446c3b6add0df559ba4bce2cc12 to your computer and use it in GitHub Desktop.
error_log logs/error.log debug; | |
events { | |
worker_connections 1024; | |
} | |
http { | |
default_type text/html; | |
lua_package_path "/Users/hamish/lua-ffi-zlib/lib/?.lua;;"; | |
gunzip off; | |
server { | |
listen 80; | |
location / { | |
header_filter_by_lua_block { | |
local ffi_zlib = require("ffi-zlib") | |
local stream, inbuf, outbuf = ffi_zlib.createStream(8192) | |
local ok = ffi_zlib.initInflate(stream) | |
if ok ~= 0 then | |
ngx.log(ngx.ERR, "Could not init stream: ", ffi_zlib.zlib_err(ok)) | |
end | |
local ctx = ngx.ctx | |
ctx.stream, ctx.inbuf, ctx.outbuf = stream, inbuf, outbuf | |
ngx.log(ngx.DEBUG, "Stream initialised") | |
} | |
body_filter_by_lua_block { | |
local ctx = ngx.ctx | |
if not ctx.stream then | |
return | |
end | |
local ffi_zlib = require("ffi-zlib") | |
local body_chunk = ngx.arg[1] | |
local str_sub = string.sub | |
local tbl_insert = table.insert | |
ngx.log(ngx.DEBUG, "BODY FILTER") | |
local input = function(bufsize) | |
local input_chunk = str_sub(body_chunk, 1, bufsize) | |
body_chunk = str_sub(body_chunk, bufsize, -1) | |
ngx.log(ngx.DEBUG, "INPUT: ", bufsize, " ", #input_chunk) | |
if #input_chunk == 0 then | |
return nil | |
end | |
return input_chunk | |
end | |
local inflated = {} | |
local output = function(out_chunk) | |
ngx.log(ngx.DEBUG, "OUTPUT: ", #out_chunk) | |
tbl_insert(inflated, out_chunk) | |
end | |
local ctx = ngx.ctx | |
--local ok, err = ffi_zlib.flate(ffi_zlib.inflate, ffi_zlib.inflateEnd, input, output, 8192, ctx.stream, ctx.inbuf, ctx.outbuf) | |
local stream, inbuf, outbuf = ctx.stream, ctx.inbuf, ctx.outbuf | |
local zlib_inflate = ffi_zlib.zlib.inflate | |
local bufsize = 8192 | |
local ffi = require("ffi") | |
local ffi_copy = ffi.copy | |
local ffi_str = ffi.string | |
local function flushOutput(stream, bufsize, output, outbuf) | |
-- Calculate available output bytes | |
local out_sz = bufsize - stream.avail_out | |
if out_sz == 0 then | |
return | |
end | |
-- Read bytes from output buffer and pass to output function | |
output(ffi_str(outbuf, out_sz)) | |
end | |
local err = 0 | |
local mode = ffi_zlib.zlib.Z_NO_FLUSH | |
repeat | |
-- Read some input | |
local data = input(bufsize) | |
if data ~= nil then | |
ffi_copy(inbuf, data) | |
stream.next_in, stream.avail_in = inbuf, #data | |
end | |
if ngx.arg[2] then | |
-- EOF, try and finish up if last chunk in response | |
mode = ffi_zlib.zlib.Z_FINISH | |
stream.avail_in = 0 | |
end | |
if ngx.arg[2] or data ~= nil then | |
-- While the output buffer is being filled completely just keep going | |
repeat | |
stream.next_out = outbuf | |
stream.avail_out = bufsize | |
-- Process the stream | |
err = zlib_inflate(stream, mode) | |
if err < ffi_zlib.zlib.Z_OK then | |
-- Error, clean up and return | |
ffi_zlib.zlib.inflateEnd(stream) | |
ngx.log(ngx.DEBUG, "FLATE: "..ffi_zlib.zlib_err(err)) | |
return false, "FLATE: "..ffi_zlib.zlib_err(err), stream | |
end | |
-- Write the data out | |
flushOutput(stream, bufsize, output, outbuf) | |
until stream.avail_out ~= 0 | |
else | |
ngx.log(ngx.DEBUG, "Nil input and not EOF") | |
end | |
until data == nil or err == ffi_zlib.zlib.Z_STREAM_END | |
if err == ffi_zlib.zlib.Z_STREAM_END then | |
-- Stream finished, clean up and return | |
ngx.log(ngx.DEBUG, "Ending stream") | |
ffi_zlib.zlib.inflateEnd(stream) | |
end | |
ngx.arg[1] = table.concat(inflated, "") | |
} | |
proxy_set_header 'Accept-Encoding' 'gzip'; | |
proxy_pass http://127.0.0.1:81/; | |
} | |
} | |
server { | |
listen 81; | |
location / { | |
gzip on; | |
gzip_min_length 1; | |
gzip_proxied any; | |
gzip_http_version 1.0; | |
chunked_transfer_encoding on; | |
content_by_lua_block { | |
for i=1,1000 do | |
ngx.say(tostring(math.random())) | |
end | |
} | |
} | |
} | |
} | |
hamish@Hamish-MBP ~> curl -s localhost/test | tail -n 10 | |
0.45992698036311 | |
0.62704646776877 | |
0.025398067290548 | |
0.61989906951413 | |
0.063203659824865 | |
0.5877136939493 | |
0.12636865686141 | |
0.63717704572831 | |
0.05910122988675 | |
0.78456766553458 |
Weirdly enough, this won't work on my machine (Ubuntu 16.04).
I've put together a self contained repo here: https://github.com/alubbe/ffizlibtest
My output is all marbled:
curl -s localhost:5000/test | tail -n 10
�
q�z�^jQ#�� T�Z�,�*P��P���'������`_�T���%�,��u6�p�iE-SZ4��l������(�f��b��?14hA�j��ӖQ4�L������+R�
N����2��Q!��X�5�����iS�����j̬��y�t�4>FlZ7�n����4���K
l�_�vH���:D�\NI����z-�������/w8���ک�E�X�7���P���C=!�� <�l��8K3��~�QH ��xPY;��=��=�n��r/�,4�������J����*|�0 ����|K�# Hg��U`���
�S"=6����\��ꭟg@�4`�$�DW�]�fzABs1m���8��E��[��*�0��7����0<�.�a\�Gl�x����,t��Ȫ6��jBy��Y���.�K:� ��4˂5���x,���a�3v����� ��zuO���T��(;w�-ֳKU9���o�R��2��0�suw��X�D�*��[_�0�1�BA��v)��#���m���0�{rrG��]��������2}5�`��c��V��3�D�� ��Wn�X'd:E��u�.�������s�l
Z�&�'�^it����,4y�&�T {��@�ϼT�.Gi���c�T[khQ�=�7���`�u�0�v��Vf��s��q����\�g��-T��[u�ϪHݓ��=�˒�@�>�1�H�Z(8�G���!W��⅙>���M���Ҭ��M}I���I�ʳC���%��װ��04�Jm{���CEnȓ'YO��+�pA���P�7��r���ب�vt�>�é����o����/R�����8���<e�q����I�R������B�����W���ϡ��9��ڕa}�kF����R�4[�Ԓ�8 b�SE�t�(��e�׃R����i��p���
8�/"q��� ����J>�5w�ȭp��_�Q�9v�lA�A��+�}����N��pV�&j��A1�o�����A��H'p�*
�=5y}�D7��Y�ԥ����>e�G0R��uy�/J�.�l�W�@�u������RR�<9*�LHbdT�ر�|xA�1R��W$�\�mTj�`�K#[���X�d���7R���^1Ѵ�ղUan>?��W\�hj
��H���3͚$�n[E������t0��qo�
mR%ƃhj,�C���)�<����S`ĺW���
���td!8���MB�X���?������p)`}������6E�������qc>��:-x5�mj�$Nu����t�h�X���FhOUc�D�L�{�C����J*�3����L� т Q��upp���"��*�6ڳ�:���x�w�g���[�W����ϱ��A����r���Ӡ
{G>֦*Q�| �ﬞ�@�;�����_3�U�U͘�OĤB���1�;6rX�qԟ���8�n��F�r�4,+x���N��-h�BWp�����p�Y��D���}��L�
�Am��g���B�`ڦ���
��>���;}��Dx��[h8��6�D��T���%��1����뼻��4��S��O��o�F>�)�aO5%qӦ�(��#�-P0�|���h�F���˩�DBp�c�p��f����J,�F������o��+��
�
e�����D]����R%b���DE��l|�%n�N��?m�r�[2����.֧�}' szB�n�ѧ��c��J����k
��S��eB���%��F�ӄ��ꔰ���M�d0bS8���+�e����
�]��ׇZ�J"����&2Eʷ����FW*U�ZQ(N���Zd���H[#l&�������t�H�Ғ�K���������w��J�Ȕ�����%���2��jI_���`u�Z(">zQ)�Z�X[���k�<j|�:�;:�J��4��e��� ^*��j�$Ӑ�xWhaH��WN3���re���Y��ʤ�2 -����~�Rw����[��s�~��0����c���9��"��<)p��Z��S�?±WxL�O%]-e�.��C��G(�X/�,RA[ԹBf�t�8���o���8�p��N�h�x^v����@j��\>4���i���v#�0���ć��u�9"�q�&��m�h{�!4�<+J{�;���T"���R�T��s&���CG��
���ɘ6&�\� �G����5��
7#.���Y�6���XN*��Z��,��Kl�����<)���W����V�2-0A��+��H�j�*��*�Jg0��c�:��uP>��.��>6M�C�D'�e���ʟ0�Z����7�E5��*�r����\K7������,�i�)`�vI�k�쁃�J���I���\��+E�.}r筦cc:����gr~��_}��E���.��Gf�L��`JWSp��vtGъ K{�/��5��#���i��N/:ߊ�,$e����AS��0����;�,I��2=��dz�W5��.���d�'���t|�����������X�������7[8��.�b��(���slJBi���k{d�SA��w����l�����7.��&��
�$��(��q� Ry3L%
��F]">�$��5>h3�*Rͽҥ2���P����&t�?�����ilB
And here are the logs
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] header_filter_by_lua:14: Stream initialised while reading response header from upstream, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:11: BODY FILTER while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:16: input(): INPUT: 8192 8192 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:25: output(): OUTPUT: 8192 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:25: output(): OUTPUT: 8192 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:25: output(): OUTPUT: 157 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:16: input(): INPUT: 8192 235 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:76: FLATE: data error while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:11: BODY FILTER while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:16: input(): INPUT: 8192 0 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:76: FLATE: stream error while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
In the repository, I've added ffi-zlib.lua directly and switched to printing the info log directly to stdout for convenience, otherwise it's the same as this gist.
@hamishforbes is it working on os X just fine? I've tried Ubuntu 14.04 LTS today and I get the same, incorrect output. I'm a bit puzzled what might be causing the differences
@hamishforbes Any feedback would be really helpful - I'm stuck :(
@hamishforbes cautious poke
hi @hamishforbes how are things looking after the summer? I'd love to work on this and maybe create a PR or a repo for it, so let me know what the status is on your end.
@hamishforbes cautious poke, would love to put the work in, but would need some help
I think this happens to work because the output is small enough to fit into one buffer ("BODY FILTER" is printed only once).
If you increase the size of the proxied body, it stops flushing the output and the proxied response is invalid.