FFI zlib decompress with body_filter

nginx.conf

error_log  logs/error.log debug;
events {
    worker_connections  1024;
}

http {
    default_type  text/html;

    lua_package_path "/Users/hamish/lua-ffi-zlib/lib/?.lua;;";

    gunzip off;

    server {
        listen 80;

        location / {

       header_filter_by_lua_block {
                local ffi_zlib = require("ffi-zlib")

                local stream, inbuf, outbuf = ffi_zlib.createStream(8192)

                local ok = ffi_zlib.initInflate(stream)
                if ok ~= 0 then
                    ngx.log(ngx.ERR, "Could not init stream: ", ffi_zlib.zlib_err(ok))
                end

                local ctx = ngx.ctx
                ctx.stream, ctx.inbuf, ctx.outbuf = stream, inbuf, outbuf

                ngx.log(ngx.DEBUG, "Stream initialised")
            }

            body_filter_by_lua_block {
                local ctx = ngx.ctx
                if not ctx.stream then
                    return
                end
                local ffi_zlib = require("ffi-zlib")
                local body_chunk = ngx.arg[1]
                local str_sub = string.sub
                local tbl_insert = table.insert

                ngx.log(ngx.DEBUG, "BODY FILTER")

                local input = function(bufsize)
                    local input_chunk = str_sub(body_chunk, 1, bufsize)
                    body_chunk = str_sub(body_chunk, bufsize, -1)
                    ngx.log(ngx.DEBUG, "INPUT: ", bufsize, " ", #input_chunk)
                    if #input_chunk == 0 then
                        return nil
                    end
                    return input_chunk
                end

                local inflated = {}
                local output = function(out_chunk)
                    ngx.log(ngx.DEBUG, "OUTPUT: ", #out_chunk)
                    tbl_insert(inflated, out_chunk)
                end

                local ctx = ngx.ctx
                --local ok, err = ffi_zlib.flate(ffi_zlib.inflate, ffi_zlib.inflateEnd, input, output, 8192, ctx.stream, ctx.inbuf, ctx.outbuf)


                local stream, inbuf, outbuf = ctx.stream, ctx.inbuf, ctx.outbuf
                local zlib_inflate = ffi_zlib.zlib.inflate
                local bufsize = 8192
                local ffi = require("ffi")
                local ffi_copy = ffi.copy
                local ffi_str = ffi.string

                local function flushOutput(stream, bufsize, output, outbuf)
                    -- Calculate available output bytes
                    local out_sz = bufsize - stream.avail_out
                    if out_sz == 0 then
                        return
                    end
                    -- Read bytes from output buffer and pass to output function
                    output(ffi_str(outbuf, out_sz))
                end

                local err = 0
                local mode = ffi_zlib.zlib.Z_NO_FLUSH
                repeat
                    -- Read some input
                    local data = input(bufsize)
                    if data ~= nil then
                        ffi_copy(inbuf, data)
                        stream.next_in, stream.avail_in = inbuf, #data
                    end

                    if ngx.arg[2] then
                        -- EOF, try and finish up if last chunk in response
                        mode = ffi_zlib.zlib.Z_FINISH
                        stream.avail_in = 0
                    end

                    if ngx.arg[2] or data ~= nil then
                        -- While the output buffer is being filled completely just keep going
                        repeat
                            stream.next_out  = outbuf
                            stream.avail_out = bufsize
                            -- Process the stream
                            err = zlib_inflate(stream, mode)
                            if err < ffi_zlib.zlib.Z_OK then
                                -- Error, clean up and return
                                ffi_zlib.zlib.inflateEnd(stream)
                                ngx.log(ngx.DEBUG, "FLATE: "..ffi_zlib.zlib_err(err))
                                return false, "FLATE: "..ffi_zlib.zlib_err(err), stream
                            end
                            -- Write the data out
                            flushOutput(stream, bufsize, output, outbuf)
                        until stream.avail_out ~= 0
                    else
                        ngx.log(ngx.DEBUG, "Nil input and not EOF")
                    end

                until data == nil or err == ffi_zlib.zlib.Z_STREAM_END

                if err == ffi_zlib.zlib.Z_STREAM_END then
                    -- Stream finished, clean up and return
                    ngx.log(ngx.DEBUG, "Ending stream")
                    ffi_zlib.zlib.inflateEnd(stream)
                end

                ngx.arg[1] = table.concat(inflated, "")

            }

            proxy_set_header 'Accept-Encoding' 'gzip';
            proxy_pass http://127.0.0.1:81/;

        }

    }


    server {
        listen 81;
        location / {
            gzip on;
            gzip_min_length 1;
            gzip_proxied any;
            gzip_http_version 1.0;

            chunked_transfer_encoding on;

            content_by_lua_block {
                for i=1,1000 do
                    ngx.say(tostring(math.random()))
                end
            }
        }
    }

}

output

hamish@Hamish-MBP ~> curl -s localhost/test | tail -n 10
0.45992698036311
0.62704646776877
0.025398067290548
0.61989906951413
0.063203659824865
0.5877136939493
0.12636865686141
0.63717704572831
0.05910122988675
0.78456766553458

@hamishforbes is it working on os X just fine? I've tried Ubuntu 14.04 LTS today and I get the same, incorrect output. I'm a bit puzzled what might be causing the differences

@hamishforbes Any feedback would be really helpful - I'm stuck :(

@hamishforbes cautious poke

hi @hamishforbes how are things looking after the summer? I'd love to work on this and maybe create a PR or a repo for it, so let me know what the status is on your end.

@hamishforbes cautious poke, would love to put the work in, but would need some help