PyTorch implementation of Laplacian pyramid loss

lap_pyramid_loss.py

# MIT License
# 
# Copyright (c) 2024 Alper Ahmetoglu
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.


import torch

def gauss_kernel(size=5, device=torch.device('cpu'), channels=3):
    kernel = torch.tensor([[1., 4., 6., 4., 1],
                           [4., 16., 24., 16., 4.],
                           [6., 24., 36., 24., 6.],
                           [4., 16., 24., 16., 4.],
                           [1., 4., 6., 4., 1.]])
    kernel /= 256.
    kernel = kernel.repeat(channels, 1, 1, 1)
    kernel = kernel.to(device)
    return kernel

def downsample(x):
    return x[:, :, ::2, ::2]

def upsample(x):
    cc = torch.cat([x, torch.zeros(x.shape[0], x.shape[1], x.shape[2], x.shape[3], device=x.device)], dim=3)
    cc = cc.view(x.shape[0], x.shape[1], x.shape[2]*2, x.shape[3])
    cc = cc.permute(0,1,3,2)
    cc = torch.cat([cc, torch.zeros(x.shape[0], x.shape[1], x.shape[3], x.shape[2]*2, device=x.device)], dim=3)
    cc = cc.view(x.shape[0], x.shape[1], x.shape[3]*2, x.shape[2]*2)
    x_up = cc.permute(0,1,3,2)
    return conv_gauss(x_up, 4*gauss_kernel(channels=x.shape[1], device=x.device))

def conv_gauss(img, kernel):
    img = torch.nn.functional.pad(img, (2, 2, 2, 2), mode='reflect')
    out = torch.nn.functional.conv2d(img, kernel, groups=img.shape[1])
    return out

def laplacian_pyramid(img, kernel, max_levels=3):
    current = img
    pyr = []
    for level in range(max_levels):
        filtered = conv_gauss(current, kernel)
        down = downsample(filtered)
        up = upsample(down)
        diff = current-up
        pyr.append(diff)
        current = down
    return pyr

class LapLoss(torch.nn.Module):
    def __init__(self, max_levels=3, channels=3, device=torch.device('cpu')):
        super(LapLoss, self).__init__()
        self.max_levels = max_levels
        self.gauss_kernel = gauss_kernel(channels=channels, device=device)
        
    def forward(self, input, target):
        pyr_input  = laplacian_pyramid(img=input, kernel=self.gauss_kernel, max_levels=self.max_levels)
        pyr_target = laplacian_pyramid(img=target, kernel=self.gauss_kernel, max_levels=self.max_levels)
        return sum(torch.nn.functional.l1_loss(a, b) for a, b in zip(pyr_input, pyr_target))

I noticed this only worked for square images. I think you need to swap your [2] and [3] channels for both lines 21 and 22 so it's like:

def upsample(x):
    cc = torch.cat([x, torch.zeros(x.shape[0], x.shape[1], x.shape[2], x.shape[3], device=x.device)], dim=3)
    cc = cc.view(x.shape[0], x.shape[1], x.shape[2]*2, x.shape[3])
    cc = cc.permute(0,1,3,2)
    cc = torch.cat([cc, torch.zeros(x.shape[0], x.shape[1], x.shape[3], x.shape[2]*2, device=x.device)], dim=3)
    cc = cc.view(x.shape[0], x.shape[1], x.shape[3]*2, x.shape[2]*2)
    x_up = cc.permute(0,1,3,2)
    cv2.imwrite('test_Net2.png', x_up[0].permute(1, 2, 0).detach().cpu().numpy())
    input('break')
    return conv_gauss(x_up, 4*gauss_kernel(channels=x.shape[1], device=x.device))

Ah, I see, you are right. I will update it as soon as possible. Thank you!

What is idea behind return conv_gauss(x_up, 4*gauss_kernel(channels=x.shape[1], device=x.device)), line 24. Why not return conv_gauss(x_up, gauss_kernel(channels=x.shape[1], device=x.device)) ?

What's the license for this?

What is idea behind return conv_gauss(x_up, 4*gauss_kernel(channels=x.shape[1], device=x.device)), line 24. Why not return conv_gauss(x_up, gauss_kernel(channels=x.shape[1], device=x.device)) ?

@amulikhov-mannequin Honestly, I don't remember at the moment. We might as well divide the kernel by 64 at #L9 instead of multiplying it with 4 later on. It was probably a temporary quick change which remained as it is.

So, checking again, it was probably due to some other reason, but I couldn't remember why. One possible explanation is when we upsample, we increase the width and the height by two, so it might be due to that.

What's the license for this?

@el3ment No licence, use it as you like.

@alper111 no license usually means others do not have permission to use your code (based on my understanding of licenses - https://choosealicense.com/no-permission/). If I may, I'd suggest MIT https://choosealicense.com/licenses/mit/. I think all you'd need to do is copy the license text into the top of your code file.

Oh, I see. I have added the license. Thanks for the suggestion.

What is idea behind return conv_gauss(x_up, 4*gauss_kernel(channels=x.shape[1], device=x.device)), line 24. Why not return conv_gauss(x_up, gauss_kernel(channels=x.shape[1], device=x.device)) ?

@amulikhov-mannequin Honestly, I don't remember at the moment. We might as well divide the kernel by 64 at #L9 instead of multiplying it with 4 later on. It was probably a temporary quick change which remained as it is.

So, checking again, it was probably due to some other reason, but I couldn't remember why. One possible explanation is when we upsample, we increase the width and the height by two, so it might be due to that.

The gaussian filter is normalized to 4 rather than 1 when upsampling to recover the average brightness after the addition of the zero rows and columns. Ref: (https://learning.oreilly.com/library/view/learning-opencv-3/9781491937983/ch11.html#ch11fn5))