Torcado194 CleanEdge Scaling for Aseprite

CleanEdge.lua

-- Pixelart Scaling inspired by torcado194 amazing CleanEdge 
--[[ MIT LICENSE
Copyright (c) 2025 xTacobaco
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
]] 

local function clamp(v, lo, hi)
  if v < lo then return lo end
  if v > hi then return hi end
  return v
end

local function round01(x)
  -- returns 0 if x < 0.5 else 1
  if x < 0.5 then return 0 else return 1 end
end

local function to255(x)
  return clamp(math.floor(x * 255 + 0.5), 0, 255)
end

local function from255(x)
  return x / 255.0
end

local function rgbaToTable(px)
  return {
    r = from255(app.pixelColor.rgbaR(px)),
    g = from255(app.pixelColor.rgbaG(px)),
    b = from255(app.pixelColor.rgbaB(px)),
    a = from255(app.pixelColor.rgbaA(px)),
  }
end

local function tableToRgba(c)
  return app.pixelColor.rgba(to255(c.r), to255(c.g), to255(c.b), to255(c.a))
end

local function colorDistance(c1, c2)
  local dr = c1.r - c2.r
  local dg = c1.g - c2.g
  local db = c1.b - c2.b
  local da = c1.a - c2.a
  return math.sqrt(dr*dr + dg*dg + db*db + da*da)
end

local function distanceRGB(c1, c2)
  local dr = c1.r - c2.r
  local dg = c1.g - c2.g
  local db = c1.b - c2.b
  return math.sqrt(dr*dr + dg*dg + db*db)
end

local function similar(c1, c2, similarThreshold)
  return (c1.a == 0 and c2.a == 0) or (colorDistance(c1, c2) <= similarThreshold)
end

local function similar3(a, b, c, similarThreshold)
  return similar(a,b,similarThreshold) and similar(b,c,similarThreshold)
end

local function similar4(a, b, c, d, similarThreshold)
  return similar(a,b,similarThreshold) and similar(b,c,similarThreshold) and similar(c,d,similarThreshold)
end

local function similar5(a, b, c, d, e, similarThreshold)
  return similar(a,b,similarThreshold) and similar(b,c,similarThreshold) and similar(c,d,similarThreshold) and similar(d,e,similarThreshold)
end

local function higher(thisCol, otherCol, highestColor, similarThreshold)
  if similar(thisCol, otherCol, similarThreshold) then return false end
  if thisCol.a == otherCol.a then
    return distanceRGB(thisCol, highestColor) < distanceRGB(otherCol, highestColor)
  else
    return thisCol.a > otherCol.a
  end
end

local function higherCol(thisCol, otherCol, highestColor, similarThreshold)
  if higher(thisCol, otherCol, highestColor, similarThreshold) then
    return thisCol
  else
    return otherCol
  end
end

local function distToLine(testPt, pt1, pt2, dir)
  local lineDir = { x = pt2.x - pt1.x, y = pt2.y - pt1.y }
  local perpDir = { x = lineDir.y, y = -lineDir.x }
  local dirToPt1 = { x = pt1.x - testPt.x, y = pt1.y - testPt.y }
  local dot_perp_dir = perpDir.x * dir.x + perpDir.y * dir.y
  local sign = (dot_perp_dir > 0.0) and 1.0 or -1.0
  local mag = math.sqrt(perpDir.x*perpDir.x + perpDir.y*perpDir.y)
  if mag == 0 then mag = 1 end
  local perpNorm = { x = perpDir.x/mag, y = perpDir.y/mag }
  local dot_val = perpNorm.x * dirToPt1.x + perpNorm.y * dirToPt1.y
  return sign * dot_val
end

local function vec(x,y) return {x=x, y=y} end

local function add(a,b) return {x=a.x+b.x, y=a.y+b.y} end
-- Multiplies a vec2 by either a scalar or another vec2 (component-wise)
local function mul(a, s)
  if type(s) == "number" then
    return { x = a.x * s, y = a.y * s }
  else
    -- assume table with x,y (vec2)
    return { x = a.x * s.x, y = a.y * s.y }
  end
end

-- Slice function ported from shader. Returns vec4 color or {r=-1} as "no slice".
local function sliceDist(point, mainDir, pointDir, u, uf, uff, b, c, f, ff, db, d, df, dff, ddb, dd, ddf, SLOPE, lineWidth, highestColor, similarThreshold)
  local minWidth = 0.0
  local maxWidth = 1.4
  if SLOPE then
    minWidth = 0.44
    maxWidth = 1.142
  end
  local _lineWidth = clamp(lineWidth, minWidth, maxWidth)

  -- flip point based on mainDir
  local p = { x = mainDir.x * (point.x - 0.5) + 0.5, y = mainDir.y * (point.y - 0.5) + 0.5 }

  -- edge detection
  local function cd(a,b) return colorDistance(a,b) end
  local distAgainst = 4.0*cd(f,d) + cd(uf,c) + cd(c,db) + cd(ff,df) + cd(df,dd)
  local distTowards = 4.0*cd(c,df) + cd(u,f) + cd(f,dff) + cd(b,d) + cd(d,ddf)

  local shouldSlice = (distAgainst < distTowards) or ((distAgainst < distTowards + 0.001) and (not higher(c, f, highestColor, similarThreshold)))
  if similar4(f, d, b, u, similarThreshold) and similar3(uf, df, db, similarThreshold) and (not similar(c, f, similarThreshold)) then
    shouldSlice = false
  end
  if not shouldSlice then return {r=-1} end

  local dist = 1.0
  local flip = false
  local center = vec(0.5, 0.5)

  if SLOPE and similar3(f, d, db, similarThreshold) and (not similar3(f, d, b, similarThreshold)) and (not similar(uf, db, similarThreshold)) then
    -- lower shallow 2:1 slant
    if similar(c, df, similarThreshold) and higher(c, f, highestColor, similarThreshold) then
      -- single pixel wide diagonal, dont flip
    else
      if higher(c, f, highestColor, similarThreshold) then flip = true end
      if similar(u, f, similarThreshold) and (not similar(c, df, similarThreshold)) and (not higher(c, u, highestColor, similarThreshold)) then flip = true end
    end

    if flip then
      dist = _lineWidth - distToLine(p, add(center, mul(vec(1.5, -1.0), pointDir)), add(center, mul(vec(-0.5, 0.0), pointDir)), mul(pointDir, -1))
    else
      dist = distToLine(p, add(center, mul(vec(1.5, 0.0), pointDir)), add(center, mul(vec(-0.5, 1.0), pointDir)), pointDir)
    end

    -- CLEANUP shallow
    if (not flip) and similar(c, uf, similarThreshold) and (not (similar3(c, uf, uff, similarThreshold) and (not similar3(c, uf, ff, similarThreshold)) and (not similar(d, uff, similarThreshold)))) then
      local dist2 = distToLine(p, add(center, mul(vec(2.0, -1.0), pointDir)), add(center, mul(vec(-0.0, 1.0), pointDir)), pointDir)
      if dist2 < dist then dist = dist2 end
    end

    dist = dist - (_lineWidth/2.0)
    return (dist <= 0.0) and ((cd(c,f) <= cd(c,d)) and f or d) or {r=-1}

  elseif SLOPE and similar3(uf, f, d, similarThreshold) and (not similar3(u, f, d, similarThreshold)) and (not similar(uf, db, similarThreshold)) then
    -- forward steep 2:1 slant
    if similar(c, df, similarThreshold) and higher(c, d, highestColor, similarThreshold) then
    else
      if higher(c, d, highestColor, similarThreshold) then flip = true end
      if similar(b, d, similarThreshold) and (not similar(c, df, similarThreshold)) and (not higher(c, d, highestColor, similarThreshold)) then flip = true end
    end

    if flip then
      dist = _lineWidth - distToLine(p, add(center, mul(vec(0.0, -0.5), pointDir)), add(center, mul(vec(-1.0, 1.5), pointDir)), mul(pointDir, -1))
    else
      dist = distToLine(p, add(center, mul(vec(1.0, -0.5), pointDir)), add(center, mul(vec(0.0, 1.5), pointDir)), pointDir)
    end

    -- CLEANUP steep
    if (not flip) and similar(c, db, similarThreshold) and (not (similar3(c, db, ddb, similarThreshold) and (not similar3(c, db, dd, similarThreshold)) and (not similar(f, ddb, similarThreshold)))) then
      local dist2 = distToLine(p, add(center, mul(vec(1.0, 0.0), pointDir)), add(center, mul(vec(-1.0, 2.0), pointDir)), pointDir)
      if dist2 < dist then dist = dist2 end
    end

    dist = dist - (_lineWidth/2.0)
    return (dist <= 0.0) and ((cd(c,f) <= cd(c,d)) and f or d) or {r=-1}

  elseif similar(f, d, similarThreshold) then
    -- 45 diagonal
    if similar(c, df, similarThreshold) and higher(c, f, highestColor, similarThreshold) then
      if (not similar(c, dd, similarThreshold)) and (not similar(c, ff, similarThreshold)) then
        flip = true
      end
    else
      if higher(c, f, highestColor, similarThreshold) then flip = true end
      if (not similar(c, b, similarThreshold)) and similar4(b, f, d, u, similarThreshold) then flip = true end
    end

    if (((similar(f, db, similarThreshold) and similar3(u, f, df, similarThreshold)) or (similar(uf, d, similarThreshold) and similar3(b, d, df, similarThreshold))) and (not similar(c, df, similarThreshold))) then
      flip = true
    end

    if flip then
      dist = _lineWidth - distToLine(p, add(center, mul(vec(1.0, -1.0), pointDir)), add(center, mul(vec(-1.0, 1.0), pointDir)), mul(pointDir, -1))
    else
      dist = distToLine(p, add(center, mul(vec(1.0, 0.0), pointDir)), add(center, mul(vec(0.0, 1.0), pointDir)), pointDir)
    end

    if SLOPE then
      if (not flip) and similar3(c, uf, uff, similarThreshold) and (not similar3(c, uf, ff, similarThreshold)) and (not similar(d, uff, similarThreshold)) then
        local dist2 = distToLine(p, add(center, mul(vec(1.5, 0.0), pointDir)), add(center, mul(vec(-0.5, 1.0), pointDir)), pointDir)
        if dist2 > dist then dist = dist2 end
      end
      if (not flip) and similar3(ddb, db, c, similarThreshold) and (not similar3(dd, db, c, similarThreshold)) and (not similar(ddb, f, similarThreshold)) then
        local dist2 = distToLine(p, add(center, mul(vec(1.0, -0.5), pointDir)), add(center, mul(vec(0.0, 1.5), pointDir)), pointDir)
        if dist2 > dist then dist = dist2 end
      end
    end

    dist = dist - (_lineWidth/2.0)
    return (dist <= 0.0) and ((cd(c,f) <= cd(c,d)) and f or d) or {r=-1}

  elseif SLOPE and similar3(ff, df, d, similarThreshold) and (not similar3(ff, df, c, similarThreshold)) and (not similar(uff, d, similarThreshold)) then
    -- far corner of shallow slant
    if similar(f, dff, similarThreshold) and higher(f, ff, highestColor, similarThreshold) then
    else
      if higher(f, ff, highestColor, similarThreshold) then flip = true end
      if similar(uf, ff, similarThreshold) and (not similar(f, dff, similarThreshold)) and (not higher(f, uf, highestColor, similarThreshold)) then flip = true end
    end

    if flip then
      dist = _lineWidth - distToLine(p, add(center, mul(vec(2.5, -1.0), pointDir)), add(center, mul(vec(0.5, 0.0), pointDir)), mul(pointDir, -1))
    else
      dist = distToLine(p, add(center, mul(vec(2.5, 0.0), pointDir)), add(center, mul(vec(0.5, 1.0), pointDir)), pointDir)
    end

    dist = dist - (_lineWidth/2.0)
    return (dist <= 0.0) and ((cd(f,ff) <= cd(f,df)) and ff or df) or {r=-1}

  elseif SLOPE and similar3(f, df, dd, similarThreshold) and (not similar3(c, df, dd, similarThreshold)) and (not similar(f, ddb, similarThreshold)) then
    -- far corner of steep slant
    if similar(d, ddf, similarThreshold) and higher(d, dd, highestColor, similarThreshold) then
    else
      if higher(d, dd, highestColor, similarThreshold) then flip = true end
      if similar(db, dd, similarThreshold) and (not similar(d, ddf, similarThreshold)) and (not higher(d, dd, highestColor, similarThreshold)) then flip = true end
    end

    if flip then
      dist = _lineWidth - distToLine(p, add(center, mul(vec(0.0, 0.5), pointDir)), add(center, mul(vec(-1.0, 2.5), pointDir)), mul(pointDir, -1))
    else
      dist = distToLine(p, add(center, mul(vec(1.0, 0.5), pointDir)), add(center, mul(vec(0.0, 2.5), pointDir)), pointDir)
    end

    dist = dist - (_lineWidth/2.0)
    return (dist <= 0.0) and ((cd(d,df) <= cd(d,dd)) and df or dd) or {r=-1}
  end

  return {r=-1}
end

local function sample(image, sx, sy)
  -- Clamp to edge
  local x = clamp(sx, 0, image.width - 1)
  local y = clamp(sy, 0, image.height - 1)
  return rgbaToTable(image:getPixel(x, y))
end

local function cleanEdgeScale(srcImage, zoom, SLOPE, lineWidth, highestColor, similarThreshold)
  -- zoom: positive number (integer recommended)
  local dstW = math.max(1, math.floor(srcImage.width * zoom + 0.5))
  local dstH = math.max(1, math.floor(srcImage.height * zoom + 0.5))
  local dst = Image(dstW, dstH, srcImage.colorMode)

  -- Iterate destination pixels
  for y = 0, dstH - 1 do
    local fy = (y + 0.5) / zoom
    local localY = fy - math.floor(fy)
    local py = math.ceil(fy) -- 1..H ; we'll convert to 0-based when sampling
    local roundY = round01(localY)
    local pdy = (roundY * 2) - 1 -- -1 or +1

    for x = 0, dstW - 1 do
      local fx = (x + 0.5) / zoom
      local localX = fx - math.floor(fx)
      local px = math.ceil(fx)
      local roundX = round01(localX)
      local pdx = (roundX * 2) - 1

      -- pointDir and point
      local pointDir = vec(pdx, pdy)
      local point = { x = localX, y = localY }

      -- Convert 1-based px/py to 0-based for sampling
      local cx = px - 1
      local cy = py - 1

      -- Neighbor sampling helpers: offset multiplied by pointDir
      local function N(ox, oy)
        local sx = cx + ox * pdx
        local sy = cy + oy * pdy
        return sample(srcImage, sx, sy)
      end

      -- Gather neighbors (naming matches shader for clarity)
      local uub = N(-1, -2)
      local uu  = N( 0, -2)
      local uuf = N( 1, -2)

      local ubb = N(-2, -2)
      local ub  = N(-1, -1)
      local u   = N( 0, -1)
      local uf  = N( 1, -1)
      local uff = N( 2, -1)

      local bb  = N(-2,  0)
      local b   = N(-1,  0)
      local c   = N( 0,  0)
      local f   = N( 1,  0)
      local ff  = N( 2,  0)

      local dbb = N(-2,  1)
      local db  = N(-1,  1)
      local d   = N( 0,  1)
      local df  = N( 1,  1)
      local dff = N( 2,  1)

      local ddb = N(-1,  2)
      local dd  = N( 0,  2)
      local ddf = N( 1,  2)

      local col = c

      local c_col = sliceDist(point, vec( 1.0,  1.0), pointDir, u, uf, uff, b, c, f, ff, db, d, df, dff, ddb, dd, ddf, SLOPE, lineWidth, highestColor, similarThreshold)
      local b_col = sliceDist(point, vec(-1.0,  1.0), pointDir, u, ub, ubb, f, c, b, bb, df, d, db, dbb, ddf, dd, ddb, SLOPE, lineWidth, highestColor, similarThreshold)
      local u_col = sliceDist(point, vec( 1.0, -1.0), pointDir, d, df, dff, b, c, f, ff, ub, u, uf, uff, uub, uu, uuf, SLOPE, lineWidth, highestColor, similarThreshold)

      -- "no slice" is indicated by r < 0
      if c_col.r and c_col.r >= 0.0 then col = c_col end
      if b_col.r and b_col.r >= 0.0 then col = b_col end
      if u_col.r and u_col.r >= 0.0 then col = u_col end

      dst:putPixel(x, y, tableToRgba(col))
    end
  end

  return dst
end

local function run()
  if not app.activeSprite then
    app.alert("No active sprite.")
    return
  end

  local sprite = app.activeSprite
  local frame = app.activeFrame or sprite.frames[1]
  local frameNumber = frame and frame.frameNumber or 1

  local dlg = Dialog{ title = "CleanEdge Scaling" }
  dlg:number{ id="scale", label="Scale", value=4.0, decimals=2 }
     :check{ id="slope", label="Slopes", selected=true }
     :slider{ id="lineWidth", label="Line Width", min=0, max=2, value=1, decimals=2 }
     :color{ id="highestColor", label="Highest Color", color=Color{ r=255, g=255, b=255, a=255 } }
     :button{ id="ok", text="Apply", focus=true }
     :button{ id="cancel", text="Cancel" }

  dlg:show()
  local data = dlg.data
  if not data.ok then return end

  local scale = tonumber(data.scale) or 4.0
  scale = clamp(scale, 1.0, 50.0)
  local slope = data.slope == nil and true or data.slope
  local lineWidth = tonumber(data.lineWidth) or 1.0
  local hc = data.highestColor or Color{ r=255, g=255, b=255, a=255 }
  local highestColor = { r = hc.red/255.0, g = hc.green/255.0, b = hc.blue/255.0, a = hc.alpha/255.0 }

  -- Build flattened source image of current visible frame
  local src = Image(sprite.spec)
  src:drawSprite(sprite, frameNumber)

  -- Ensure we work in RGBA space for predictable results
  if src.colorMode ~= ColorMode.RGB then
    local converted = Image(src.width, src.height, ColorMode.RGB)
    converted:drawImage(src, Point(0,0))
    src = converted
  end

  app.transaction(function()
    local dst = cleanEdgeScale(src, scale, slope, lineWidth, highestColor, 0.0)
    local newSprite = Sprite(dst.width, dst.height)
    newSprite:setPalette(sprite.palettes[1] or newSprite.palettes[1])
    local cel = newSprite.cels[1]
    cel.image = dst
    cel.position = Point(0,0)
    newSprite.filename = (sprite.filename ~= "" and sprite.filename or "Sprite") .. " [CleanEdge x"..tostring(scale).."]"
    app.activeSprite = newSprite
  end)
end

run()

dood.mov

Aseprite plugin, doesn't really make sence to do this on the cpu, but works great for small art pieces just for some quick checking of the art, before exporting and using it in games with the proper shader 👍