Last active
November 7, 2016 16:15
-
-
Save hamishforbes/ea0e9446c3b6add0df559ba4bce2cc12 to your computer and use it in GitHub Desktop.
FFI zlib decompress with body_filter
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| error_log logs/error.log debug; | |
| events { | |
| worker_connections 1024; | |
| } | |
| http { | |
| default_type text/html; | |
| lua_package_path "/Users/hamish/lua-ffi-zlib/lib/?.lua;;"; | |
| gunzip off; | |
| server { | |
| listen 80; | |
| location / { | |
| header_filter_by_lua_block { | |
| local ffi_zlib = require("ffi-zlib") | |
| local stream, inbuf, outbuf = ffi_zlib.createStream(8192) | |
| local ok = ffi_zlib.initInflate(stream) | |
| if ok ~= 0 then | |
| ngx.log(ngx.ERR, "Could not init stream: ", ffi_zlib.zlib_err(ok)) | |
| end | |
| local ctx = ngx.ctx | |
| ctx.stream, ctx.inbuf, ctx.outbuf = stream, inbuf, outbuf | |
| ngx.log(ngx.DEBUG, "Stream initialised") | |
| } | |
| body_filter_by_lua_block { | |
| local ctx = ngx.ctx | |
| if not ctx.stream then | |
| return | |
| end | |
| local ffi_zlib = require("ffi-zlib") | |
| local body_chunk = ngx.arg[1] | |
| local str_sub = string.sub | |
| local tbl_insert = table.insert | |
| ngx.log(ngx.DEBUG, "BODY FILTER") | |
| local input = function(bufsize) | |
| local input_chunk = str_sub(body_chunk, 1, bufsize) | |
| body_chunk = str_sub(body_chunk, bufsize, -1) | |
| ngx.log(ngx.DEBUG, "INPUT: ", bufsize, " ", #input_chunk) | |
| if #input_chunk == 0 then | |
| return nil | |
| end | |
| return input_chunk | |
| end | |
| local inflated = {} | |
| local output = function(out_chunk) | |
| ngx.log(ngx.DEBUG, "OUTPUT: ", #out_chunk) | |
| tbl_insert(inflated, out_chunk) | |
| end | |
| local ctx = ngx.ctx | |
| --local ok, err = ffi_zlib.flate(ffi_zlib.inflate, ffi_zlib.inflateEnd, input, output, 8192, ctx.stream, ctx.inbuf, ctx.outbuf) | |
| local stream, inbuf, outbuf = ctx.stream, ctx.inbuf, ctx.outbuf | |
| local zlib_inflate = ffi_zlib.zlib.inflate | |
| local bufsize = 8192 | |
| local ffi = require("ffi") | |
| local ffi_copy = ffi.copy | |
| local ffi_str = ffi.string | |
| local function flushOutput(stream, bufsize, output, outbuf) | |
| -- Calculate available output bytes | |
| local out_sz = bufsize - stream.avail_out | |
| if out_sz == 0 then | |
| return | |
| end | |
| -- Read bytes from output buffer and pass to output function | |
| output(ffi_str(outbuf, out_sz)) | |
| end | |
| local err = 0 | |
| local mode = ffi_zlib.zlib.Z_NO_FLUSH | |
| repeat | |
| -- Read some input | |
| local data = input(bufsize) | |
| if data ~= nil then | |
| ffi_copy(inbuf, data) | |
| stream.next_in, stream.avail_in = inbuf, #data | |
| end | |
| if ngx.arg[2] then | |
| -- EOF, try and finish up if last chunk in response | |
| mode = ffi_zlib.zlib.Z_FINISH | |
| stream.avail_in = 0 | |
| end | |
| if ngx.arg[2] or data ~= nil then | |
| -- While the output buffer is being filled completely just keep going | |
| repeat | |
| stream.next_out = outbuf | |
| stream.avail_out = bufsize | |
| -- Process the stream | |
| err = zlib_inflate(stream, mode) | |
| if err < ffi_zlib.zlib.Z_OK then | |
| -- Error, clean up and return | |
| ffi_zlib.zlib.inflateEnd(stream) | |
| ngx.log(ngx.DEBUG, "FLATE: "..ffi_zlib.zlib_err(err)) | |
| return false, "FLATE: "..ffi_zlib.zlib_err(err), stream | |
| end | |
| -- Write the data out | |
| flushOutput(stream, bufsize, output, outbuf) | |
| until stream.avail_out ~= 0 | |
| else | |
| ngx.log(ngx.DEBUG, "Nil input and not EOF") | |
| end | |
| until data == nil or err == ffi_zlib.zlib.Z_STREAM_END | |
| if err == ffi_zlib.zlib.Z_STREAM_END then | |
| -- Stream finished, clean up and return | |
| ngx.log(ngx.DEBUG, "Ending stream") | |
| ffi_zlib.zlib.inflateEnd(stream) | |
| end | |
| ngx.arg[1] = table.concat(inflated, "") | |
| } | |
| proxy_set_header 'Accept-Encoding' 'gzip'; | |
| proxy_pass http://127.0.0.1:81/; | |
| } | |
| } | |
| server { | |
| listen 81; | |
| location / { | |
| gzip on; | |
| gzip_min_length 1; | |
| gzip_proxied any; | |
| gzip_http_version 1.0; | |
| chunked_transfer_encoding on; | |
| content_by_lua_block { | |
| for i=1,1000 do | |
| ngx.say(tostring(math.random())) | |
| end | |
| } | |
| } | |
| } | |
| } | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| hamish@Hamish-MBP ~> curl -s localhost/test | tail -n 10 | |
| 0.45992698036311 | |
| 0.62704646776877 | |
| 0.025398067290548 | |
| 0.61989906951413 | |
| 0.063203659824865 | |
| 0.5877136939493 | |
| 0.12636865686141 | |
| 0.63717704572831 | |
| 0.05910122988675 | |
| 0.78456766553458 |
In the repository, I've added ffi-zlib.lua directly and switched to printing the info log directly to stdout for convenience, otherwise it's the same as this gist.
@hamishforbes is it working on os X just fine? I've tried Ubuntu 14.04 LTS today and I get the same, incorrect output. I'm a bit puzzled what might be causing the differences
@hamishforbes Any feedback would be really helpful - I'm stuck :(
@hamishforbes cautious poke
hi @hamishforbes how are things looking after the summer? I'd love to work on this and maybe create a PR or a repo for it, so let me know what the status is on your end.
@hamishforbes cautious poke, would love to put the work in, but would need some help
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Weirdly enough, this won't work on my machine (Ubuntu 16.04).
I've put together a self contained repo here: https://github.com/alubbe/ffizlibtest
My output is all marbled:
And here are the logs