cv2 PIL.Image torch and tensorflow have sligthly different interpolation algorithms; curious

interpolation_diff.py

"""
>>> import interpolation_diff as idf
>>> algs = ("pil", "tensorflow") # ("pil", "torch")
>>> save = "/home/z/share/benchestchirescat_%s_%s"%algs
>>> idf.test_dif(dtype="float32", mode="bilinear", show=True, algs=algs, crop=(170,370, 180, 380), save=save)

# re https://discuss.pytorch.org/t/nn-interpolate-function-unexpected-behaviour/106684/6
"""
import io
import os
import os.path as osp
import requests
import numpy as np
import PIL
import cv2
import matplotlib
import matplotlib.pyplot as plt
import torch
from torch.nn.functional import interpolate

import torchvision
from torchvision.transforms import functional as fun

matplotlib.rcParams.update({"axes.labelsize": "large"})
matplotlib.rcParams.update({'font.family': "monospace"})
matplotlib.rcParams.update({'font.size': 14})


def get_pimg(fname=None):
    """
    get from default image net or self image
    fname   str
                full path to local file
                url full path
                if None defaults to url
    """
    _image_types = ("image/jpeg", "image/png")
    # local file full path
    if fname is not None and osp.isfile(fname):
        print("Opened local Image <%s>"%(fname))
        out = PIL.Image.open(fname)
        return out

    # url
    if fname is None:
        fname = "https://ichef.bbci.co.uk/news/976/cpsprodpb/10207/production/_116155066_campercats.png"
    _cache = osp.expanduser("~/.cache/images")
    _fname = osp.join(_cache, osp.split(fname)[-1])
    _name, _ext = osp.splitext(_fname)
    _exts = ".jpeg", ".jpg", ".png"
    if _ext.lower() not in _exts:
        for _e in _exts:
            if _e in _ext.lower():
                _ext = _e
        if _ext not in _exts:
            _ext = ".png"
        _fname = _name + _ext


    # open cached file
    if osp.isfile(_fname):
        out = PIL.Image.open(_fname)
        print("Opened Image <%s> from cache"%(_fname))
        return out

    # open url and save cache
    out = None
    _res = requests.get(fname)
    if _res.ok and _res.headers.get('content-type') in _image_types:
        if not osp.isdir(_cache):
            os.makedirs(_cache)
        try:
            out = PIL.Image.open(io.BytesIO(_res.content))
            # save to cache
            out.save(_fname)
            print("Opened Image <%s> from url, saved to <%s>"%(fname, _fname))
        except:
            print("could not open data in pil <%s>"%fname)
    return out


# convert to numpy
tonumpy = lambda pimg, dtype: (np.array(pimg)/255).astype(dtype)

# convert to torch
def totorch(img):
    tensor = torch.from_numpy(img).permute(2, 0, 1).contiguous()
    return tensor.view(1, *tensor.shape)

# interpolation mode
def get_mode(alg, mode):
    """ mode in bilinear, bicubic
    """
    if alg == "cv2":
        _mode = mode.replace("bi", "INTER_").upper()
        return cv2.__dict__[_mode]
    elif alg == "torchvision":
        return torchvision.transforms.InterpolationMode.__dict__[mode.upper()]
    elif alg == "pil":
        return PIL.Image.__dict__[mode.upper()]
    elif alg in ("torch", "tensorflow"):
        return  mode
    else:
        print(alg, "not implemented")
        return mode

# resize
def resize(pimg, img, tensor, alg, mode, size, dtype):
    """"""
    if alg == "cv2":
        return cv2.resize(img, dsize=tuple(size[::-1]), interpolation=get_mode(alg, mode))
    elif alg == "torchvision":
        return fun.resize(img=tensor, size=size, interpolation=get_mode(alg, mode))[0].numpy().transpose(1, 2, 0)
    elif alg == "pil":
        return tonumpy(pimg.resize(size=size[::-1], resample=get_mode(alg, mode)), dtype)
    elif alg == "torch":
        return  interpolate(tensor, size=size, mode=mode, align_corners=False)[0].numpy().transpose(1, 2, 0)
    elif alg == "tensorflow":
        import tensorflow as tf
        if tf.__version__ >= '2.0.0':
            return tf.image.resize(img, size=size, method=mode).numpy()
        else:
            return tf.contrib.image.resize(img, size=size, method=mode).numpy()
    else:
        print(alg, "not implemented")
        return mode

def test_dif(fname=None, dtype="float32", mode="bilinear", show=False, algs=("pil", "torch"),
             crop=None, save=False, width=10):
    """ brief measure difference between interpolation algorithms
        algs    tuple, ist [("pil", "torch")] ("torchvision", "torch", "tensorflow", "pil", "cv2")
    """
    _algs = ("torchvision", "torch", "tensorflow", "pil", "cv2")
    if "torchvision" in algs:
        _tv = torchvision.__version__
        if _tv < '0.9':
            print("requires torchvision 0.9 or above to resize")
            return None

    for alg in algs:
        assert alg in _algs, "<%s> not recognized in valid %s"%(alg, str(_algs))

    # print tensor or array
    _format = "%s\tmin: %.4f, max: %.4f mean: %.4f, std: %.4f \tshape: %s, dtype: %s"
    sig = lambda x, msg="": print(_format%(msg, x.min(), x.max(), x.mean(), x.std(),
                                           str(tuple(x.shape)), str(x.dtype)))
    # pil, numpy image, torch
    pimg = get_pimg(fname)
    nimg = tonumpy(pimg, dtype)
    timg = totorch(nimg) if "torch" in algs or "torchvision" in algs else None

    new_size = [512, (512*pimg.size[0])//pimg.size[1]]

    out_0 = resize(pimg, nimg, timg, algs[0], mode, new_size, dtype)
    out_1 = resize(pimg, nimg, timg, algs[1], mode, new_size, dtype)

    title = "Resize %s, %s|%s"%(mode.upper(), algs[0], algs[1])

    diff = imdiff(out_0, out_1, norm=True)
    if crop and isinstance(crop, (list, tuple)) and len(crop) == 4:
        diff = diff[crop[0]:crop[1], crop[2]:crop[3], :]

    if show:
        showdiffs([diff], [title], width, 1, save)
        
    return diff

def showdiffs(images, titles, width=10, ncols=3, save=False):

    _h, _w, _c = images[0].shape
    ncols = min(ncols, len(images))
    nrows = int(np.ceil(len(images)/ncols))


    height = width*(_h/_w)*(nrows/ncols)

    print(height, width)

    plt.figure(figsize=(width, height))
    for i, image in enumerate(images):
        if len(images) > 1:
            plt.subplot(nrows, ncols, i+1)
        if titles is not None and i < len(titles):
            plt.title(titles[i])
        plt.imshow(image)
        plt.yticks([])
        plt.xticks([])
    plt.tight_layout()
    plt.subplots_adjust(wspace=0.1)
    if save:
        if not isinstance(save, str):
            print("CANNOT save, no path supplied, <save=path>")
        else:
            folder, _ = osp.split(save)
            assert osp.isdir(folder), "supply folder to save="
            _, ext = osp.splitext(save)
            if ext.lower() not in (".jpg", ".png", ".pdf"):
                save += ".pdf"
        
            print("saving figure to: %s"%save)
            dpi = 300
            plt.savefig(save, dpi=dpi)
    plt.show()

def imdiff(im0, im1, norm=True):
    diff = im0 - im1
    if norm and diff.sum() > 0:
        diff = (diff - diff.min())/(diff.max() - diff.min())
    return diff


if __name__ == "__main__":
    # test_dif("float64", "bicubic")
    # test_dif("float32", "bicubic")
    # test_dif("float64", "bilinear")
    # test_dif("float32", "bilinear", show=True)

    FOLDER = osp.expanduser("~/share")
    if not osp.isdir(FOLDER):
        FOLDER = osp.expanduser("~")

    ALGS = [("cv2", "torch"), ("pil", "torch"), ("pil", "cv2"), ("pil", "torch"),
            ("cv2", "tensorflow"), ("torch", "tensorflow"),
            ("pil", "tensorflow"), ("torch", "torchvision")]

    DIFFS = []
    TITLES = []
    MODE = "bilinear"

    for i, ALG in enumerate(ALGS):
        # SAVE
        _DIF = test_dif(dtype="float32", mode=MODE, show=False, algs=ALG,
                              crop=(170, 370, 180, 380), save=False)
        if _DIF is not None:
            DIFFS.append(_DIF)
            # TITLES.append("Resize %s, %s|%s"%(MODE.upper(), ALG[0], ALG[1]))
            TITLES.append(("%s|%s"%(ALG[0], ALG[1])).replace("tensorflow", "tf").replace("torchvision", "torch"))

    showdiffs(DIFFS, TITLES, width=10, ncols=3, save=osp.join(FOLDER, "ResizeDiffs3.png"))