Lua and redis scan

redis_scan.lua

local red = redis:new()
local ok, err = red:connect("127.0.0.1", "6379")
if not ok then
    ngx.status = 503
    ngx.say("Can't connect to redis: "..err)
    ngx.exit(ngx.OK)
else
    red:set_timeout(1000) -- 1 sec
end


local files_result = {};
local cursor = 0;
local path = "dir/test/";

repeat
    local res, _ = red:scan(cursor, "match", path.."*", "count", 1000)
    local data
    cursor, data = unpack(res)
    if next(data) then
        for _, value in pairs(data) do
            local info = red:hmget(value, 'content_type', 'size', 'modified')
            files_result[value] = {
                content_type = info[1],
                size = info[2],
                modified = info[3],
            }
        end
    end
until tonumber(cursor) == 0

local files_result_json = cjson.encode(files_result)
ngx.header["Content-Type"] = "application/json";
ngx.say(files_result_json);

syncer_01.lua

-- Удаляет протухшие файлы
-- Синхронизирует редис и файловую систему
local config = require "/www/app/deploy/ngx_lua/configs/config"
local redis  = require 'cron/redis'
local static_path = config.static_path
local client = redis.connect(config.redis.host, config.redis.port)

-- TODO надо будет добавить чтение аргументов из cli
local dry_run = true -- В тестовом режиме
local verbose = true -- Показывает, что делает
local need_to_sync = false -- Синхронизирует редис и файловую систему

local stats = {
    total_files = 0,
    files_deleted = 0,
    redis_keys_deleted = 0,
    redis_keys_added = 0,
}

function getHostname()
    local f = io.popen ("/bin/hostname")
    local hostname = f:read("*a") or ""
    f:close()
    hostname = string.gsub(hostname, "\n$", "")
    return hostname
end

local host = getHostname()

function scandir(dir, recursive)
    recursive = recursive or false
    local current_dir = dir
    local file_list = {}
    local command = "ls " .. current_dir .. " -p"
    local ver_id = ""
    local prev_ver_id = ""
    local full_path = ""
    local size = ""
    local last_modified = ""
    local content_type = ""

    if recursive then
        command = command .. ' -R'
    end

    for fileName in io.popen(command):lines() do
        if string.sub(fileName, -1) == '/' then
            -- Directory, don't do anything
        elseif string.sub(fileName, -1) == ':' then
            current_dir = fileName:sub(1, -2)
            -- if currentDirectory ~= directory then
            current_dir = current_dir .. '/'
            -- end
        elseif string.len(fileName) == 0 then
            -- Blank line
            current_dir = dir
        else
            full_path = current_dir .. fileName;
            -- Получаем инфо по файлу
            local f = io.popen("stat -c %Y "..full_path)
            last_modified = f:read()

            if verbose then
                print("scan file: "..full_path)
            end

            stats.total_files = stats.total_files + 1
            -- Проверяем паттерны
            checkPatters(full_path, last_modified)

            -- Группируем файлы по версии и отправляем на синхронизацию в редис
            if need_to_sync then
                ver_id = tonumber(current_dir:match(dir.."([^/]+)"))
                if not ver_id then
                    print("Fatal: can't get ver_id: "..ver_id)
                    os.exit()
                end

                if prev_ver_id ~= "" and prev_ver_id ~= ver_id then
                    syncRedis(prev_ver_id, file_list)
                    file_list = {}
                else
                    f = io.popen("stat -c %s "..full_path)
                    size = f:read()
                    f = io.popen("file -b --mime-type "..full_path)
                    content_type = f:read()
                    table.insert(file_list, {
                        path = full_path,
                        content_type = content_type,
                        size = size,
                        modified = last_modified
                    })
                end
                prev_ver_id = ver_id
            end
        end
    end

    if need_to_sync then
        syncRedis(ver_id, file_list)
    end
end

function checkPatters(full_path, last_modified)
    local expired = false
    local need_to_delete = false
    for _,value in pairs(config.map_delete.patterns) do
        expired = checkFileExpired(last_modified, value.time)
        if string.match(full_path, value.pattern) and expired then
            need_to_delete = true
            break
        end
    end

    -- По паттерну не нашли, проверяем время по умолчанию
    if not need_to_delete then
        if checkFileExpired(last_modified, config.map_delete.default) then
            need_to_delete = true
        end
    end

    if need_to_delete then
        if verbose then
            print(" - delete expired file")
        end
        if not dry_run then
            -- Удаляем файл
            deleteFile(full_path)
        end
        stats.files_deleted = stats.files_deleted + 1
    end

end

function checkFileExpired(last_modified, ttl)
    return tonumber(last_modified) < os.time() - tonumber(ttl)
end

function deleteFile(file_path)
    os.remove(file_path)
    client:del(config.redis.key_prefix..string.gsub(file_path,"public/webdata/", ""))
end

-- Синхронизирует файлы в файловой системе и редисе
-- !!! В начале удаляет все из редиса по версии, а затем добавляет заново
function syncRedis(ver_id, file_list)
    if verbose then
        print(" - sync ver_id: "..ver_id)
    end

    local cursor = 0;
    repeat
        cursor, keys = unpack(client:scan(cursor, {match = config.redis.key_prefix..ver_id.."/*", count = 1000}))
        if next(keys) then
            -- Удаляем из редиса по ver_id
            if verbose then
                print("  -- delete key from redis: "..keys[1])
            end
            if not dry_run then
                client:del(keys[1])
            end
        end
    until tonumber(cursor) == 0

    -- Добавляем в редис
    for _, value in ipairs(file_list) do
        if verbose then
            print("  -- add info to redis: "
                    ..value.path .." : "
                    .. value.size.." : "
                    .. value.content_type.." : "
                    .. value.modified .." : "
                    .. host)
        end

        if not dry_run then
            local file_path = string.gsub(value.path,"public/webdata/", "")
            local info =  {
                name = file_path,
                host = host,
                content_type = value.content_type,
                size = tonumber(value.size),
                modified = value.modified
            }
            client:hmset(config.redis.key_prefix..file_path, info)
        end
    end
end


scandir(static_path, true)

if verbose then
    print("=================================")
    print("Stats:")
    print("Total files: "..stats.total_files)
    print("Deleted files: "..stats.files_deleted)
    print("Redis need_to_sync: "..tostring(need_to_sync))
end