Skip to content

Instantly share code, notes, and snippets.

@hamishforbes
Last active November 7, 2016 16:15
Show Gist options
  • Save hamishforbes/ea0e9446c3b6add0df559ba4bce2cc12 to your computer and use it in GitHub Desktop.
Save hamishforbes/ea0e9446c3b6add0df559ba4bce2cc12 to your computer and use it in GitHub Desktop.
FFI zlib decompress with body_filter
error_log logs/error.log debug;
events {
worker_connections 1024;
}
http {
default_type text/html;
lua_package_path "/Users/hamish/lua-ffi-zlib/lib/?.lua;;";
gunzip off;
server {
listen 80;
location / {
header_filter_by_lua_block {
local ffi_zlib = require("ffi-zlib")
local stream, inbuf, outbuf = ffi_zlib.createStream(8192)
local ok = ffi_zlib.initInflate(stream)
if ok ~= 0 then
ngx.log(ngx.ERR, "Could not init stream: ", ffi_zlib.zlib_err(ok))
end
local ctx = ngx.ctx
ctx.stream, ctx.inbuf, ctx.outbuf = stream, inbuf, outbuf
ngx.log(ngx.DEBUG, "Stream initialised")
}
body_filter_by_lua_block {
local ctx = ngx.ctx
if not ctx.stream then
return
end
local ffi_zlib = require("ffi-zlib")
local body_chunk = ngx.arg[1]
local str_sub = string.sub
local tbl_insert = table.insert
ngx.log(ngx.DEBUG, "BODY FILTER")
local input = function(bufsize)
local input_chunk = str_sub(body_chunk, 1, bufsize)
body_chunk = str_sub(body_chunk, bufsize, -1)
ngx.log(ngx.DEBUG, "INPUT: ", bufsize, " ", #input_chunk)
if #input_chunk == 0 then
return nil
end
return input_chunk
end
local inflated = {}
local output = function(out_chunk)
ngx.log(ngx.DEBUG, "OUTPUT: ", #out_chunk)
tbl_insert(inflated, out_chunk)
end
local ctx = ngx.ctx
--local ok, err = ffi_zlib.flate(ffi_zlib.inflate, ffi_zlib.inflateEnd, input, output, 8192, ctx.stream, ctx.inbuf, ctx.outbuf)
local stream, inbuf, outbuf = ctx.stream, ctx.inbuf, ctx.outbuf
local zlib_inflate = ffi_zlib.zlib.inflate
local bufsize = 8192
local ffi = require("ffi")
local ffi_copy = ffi.copy
local ffi_str = ffi.string
local function flushOutput(stream, bufsize, output, outbuf)
-- Calculate available output bytes
local out_sz = bufsize - stream.avail_out
if out_sz == 0 then
return
end
-- Read bytes from output buffer and pass to output function
output(ffi_str(outbuf, out_sz))
end
local err = 0
local mode = ffi_zlib.zlib.Z_NO_FLUSH
repeat
-- Read some input
local data = input(bufsize)
if data ~= nil then
ffi_copy(inbuf, data)
stream.next_in, stream.avail_in = inbuf, #data
end
if ngx.arg[2] then
-- EOF, try and finish up if last chunk in response
mode = ffi_zlib.zlib.Z_FINISH
stream.avail_in = 0
end
if ngx.arg[2] or data ~= nil then
-- While the output buffer is being filled completely just keep going
repeat
stream.next_out = outbuf
stream.avail_out = bufsize
-- Process the stream
err = zlib_inflate(stream, mode)
if err < ffi_zlib.zlib.Z_OK then
-- Error, clean up and return
ffi_zlib.zlib.inflateEnd(stream)
ngx.log(ngx.DEBUG, "FLATE: "..ffi_zlib.zlib_err(err))
return false, "FLATE: "..ffi_zlib.zlib_err(err), stream
end
-- Write the data out
flushOutput(stream, bufsize, output, outbuf)
until stream.avail_out ~= 0
else
ngx.log(ngx.DEBUG, "Nil input and not EOF")
end
until data == nil or err == ffi_zlib.zlib.Z_STREAM_END
if err == ffi_zlib.zlib.Z_STREAM_END then
-- Stream finished, clean up and return
ngx.log(ngx.DEBUG, "Ending stream")
ffi_zlib.zlib.inflateEnd(stream)
end
ngx.arg[1] = table.concat(inflated, "")
}
proxy_set_header 'Accept-Encoding' 'gzip';
proxy_pass http://127.0.0.1:81/;
}
}
server {
listen 81;
location / {
gzip on;
gzip_min_length 1;
gzip_proxied any;
gzip_http_version 1.0;
chunked_transfer_encoding on;
content_by_lua_block {
for i=1,1000 do
ngx.say(tostring(math.random()))
end
}
}
}
}
hamish@Hamish-MBP ~> curl -s localhost/test | tail -n 10
0.45992698036311
0.62704646776877
0.025398067290548
0.61989906951413
0.063203659824865
0.5877136939493
0.12636865686141
0.63717704572831
0.05910122988675
0.78456766553458
@alubbe
Copy link

alubbe commented May 24, 2016

I think this happens to work because the output is small enough to fit into one buffer ("BODY FILTER" is printed only once).
If you increase the size of the proxied body, it stops flushing the output and the proxied response is invalid.

            content_by_lua_block {
                for i=1, 1000 do
                    ngx.say(tostring(math.random()))
                end
            }

@alubbe
Copy link

alubbe commented May 31, 2016

Weirdly enough, this won't work on my machine (Ubuntu 16.04).
I've put together a self contained repo here: https://github.com/alubbe/ffizlibtest
My output is all marbled:

curl -s localhost:5000/test | tail -n 10
�
 q�z�^jQ#�� T�Z�,�*P��P���'������`_�T���%�,��u6�p�iE-SZ4��l������(�f��b��?14hA�j��ӖQ4�L������+R�
N����2��Q!��X�5�����iS�����j̬��y�t�4>FlZ7�n����4���K
l�_�vH���:D�\NI����z-�������/w8���ک�E�X�7���P���C=!�� <�l��8K3��~�QH ��xPY;��=��=�n��r/�,4�������J����*|�0 ����|K�#      Hg��U`���
                                                         �S"=6����\��ꭟg@�4`�$�DW�]�fzABs1m���8��E��[��*�0��7����0<�.�a\�Gl�x����,t��Ȫ6��jBy��Y���.�K:� ��4˂5���x,���a�3v����� ��zuO���T��(;w�-ֳKU9���o�R��2��0�suw��X�D�*��[_�0�1�BA��v)��#���m���0�{rrG��]��������2}5�`��c��V��3�D�� ��Wn�X'd:E��u�.�������s�l
Z�&�'�^it����,4y�&�T    {��@�ϼT�.Gi���c�T[khQ�=�7���`�u�0�v��Vf��s��q����\�g��-T��[u�ϪHݓ��=�˒�@�>�1�H�Z(8�G���!W��⅙>���M���Ҭ��M}I���I�ʳC���%��װ��04�Jm{���CEnȓ'YO��+�pA���P�7��r���ب�vt�>�é����o����/R�����8���<e�q����I�R������B�����W���ϡ��9��ڕa}�kF����R�4[�Ԓ�8 b�SE�t�(��e�׃R����i��p���
8�/"q���      ����J>�5w�ȭp�޷�_�Q�9v�lA�A��+�}����N��pV�&j׌��A1�o�����A��H'p�*
�=5y}�D7��Y�ԥ����>e�G0R��uy�/J�.�l�W�@�u������RR�<9*�LHbdT�ر�|xA�1R��W$�\�mTj�`�K#[���X�d���7R���^1Ѵ�ղUan>?��W\�hj
                                                                                                                  ��H���3͚$�n[E������t0��qo�
mR%ƃhj,�C���)�<����S`ĺW���
���td!8���MB�X���?������p)`}������6E�������qc>��:-x5�mj�$Nu����t�h�X���FhOUc�D�L�{�C����J*�3����L� т   Q��upp���"��*�6ڳ�:���x�w�g���[�W����ϱ��A����r���Ӡ
{G>֦*Q�| �ﬞ�@�;�����_3�U�U͘�OĤB���1�;6rX�qԟ���8�n��F�r�4,+x���N��-h�BWp�����p�Y��D���}��L�
       �Am��g���B�`ڦ���
��>���;}��Dx��[h8��6�D��T���%��1����뼻��4��S��O��o�F>�)�aO5%qӦ�(��#�-P0�|���h�F���˩�DBp�c�p��f����J,�F������o��+��
�
e�����D]����R%b���DE��l|�%n�N��?m�r�[2����.֧�}'    szB�n�ѧ��c��J����k
��S��eB���%��F�ӄ��ꔰ���M�d0bS8���+�e����
�]��ׇZ�J"����&2Eʷ����FW*U�ZQ(N���Zd���H[#l&�������t�H�Ғ�K���������w��J�Ȕ�����%���2��jI_���`u�Z(">zQ)�Z�X[���k�<j|�:�;:�J��4��e��� ^*��j�$Ӑ�xWhaH��WN3���re���Y��ʤ�2 -����~�Rw����[��s�~��0����c���9��"��<)p��Z��S�?±WxL�O%]-e�.��C��G(�X/�,RA[ԹBf�t�8���o���8�p��N�h�x^v����@j��\>4���i���v#�0���ć��u�9"�q�&��m�h{�!4�<+J{�;���T"���R�T��s&���CG��
���ɘ6&�\�                                                                  �G����5��
7#.���Y�6���XN*��Z��,��Kl�����<)���W����V�2-0A��+��H�j�*��*�Jg0��c�:��uP>��.��>6M�C�D'�e���ʟ0�Z����7�E5��*�r����\K7������,�i�)`�vI�k�쁃�J���I���\��+E�.}r筦cc:����gr~��_}��E���.��Gf�L��`JWSp��vtGъ K{�/��5��#���i��N/:ߊ�,$e����AS��0����;�,I��2=��dz�W5��.���d�'���t|�����������X�������7[8��.�b��(���slJBi���k{d�SA��w����l�����7.��&��
                                                          �$��(��q� Ry3L%
��F]">�$��5>h3�*Rͽҥ2���P����&t�?�����ilB

And here are the logs

2016/05/31 11:20:21 [info] 13847#0: *1 [lua] header_filter_by_lua:14: Stream initialised while reading response header from upstream, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:11: BODY FILTER while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:16: input(): INPUT: 8192 8192 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:25: output(): OUTPUT: 8192 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:25: output(): OUTPUT: 8192 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:25: output(): OUTPUT: 157 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:16: input(): INPUT: 8192 235 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:76: FLATE: data error while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:11: BODY FILTER while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:16: input(): INPUT: 8192 0 while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"
2016/05/31 11:20:21 [info] 13847#0: *1 [lua] body_filter_by_lua:76: FLATE: stream error while sending to client, client: 127.0.0.1, server: , request: "GET /test HTTP/1.1", upstream: "http://127.0.0.1:5001/test", host: "localhost:5000"

@alubbe
Copy link

alubbe commented May 31, 2016

In the repository, I've added ffi-zlib.lua directly and switched to printing the info log directly to stdout for convenience, otherwise it's the same as this gist.

@alubbe
Copy link

alubbe commented Jun 14, 2016

@hamishforbes is it working on os X just fine? I've tried Ubuntu 14.04 LTS today and I get the same, incorrect output. I'm a bit puzzled what might be causing the differences

@alubbe
Copy link

alubbe commented Jun 30, 2016

@hamishforbes Any feedback would be really helpful - I'm stuck :(

@alubbe
Copy link

alubbe commented Jul 22, 2016

@hamishforbes cautious poke

@alubbe
Copy link

alubbe commented Sep 27, 2016

hi @hamishforbes how are things looking after the summer? I'd love to work on this and maybe create a PR or a repo for it, so let me know what the status is on your end.

@alubbe
Copy link

alubbe commented Nov 7, 2016

@hamishforbes cautious poke, would love to put the work in, but would need some help

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment