vfdev-5 · March 29, 2023 16:34
diff --git a/20230329-181023-pr_vs_nightly-speedup.md b/20230329-181023-pr_vs_nightly-speedup.md
diff --git a/readme.md b/readme.md
diff --git a/perf_results_compute_speedup.py b/perf_results_compute_speedup.py
 import pickle
 from pathlib import Path
 from typing import List, Optional
 import unittest.mock

 import torch
 import torch.utils.benchmark as benchmark
 from torch.utils.benchmark.utils import common
 from torch.utils.benchmark.utils.compare import Table


 import fire


 def patched_as_column_strings(self):
    concrete_results = [r for r in self._results if r is not None]
    env = f"({concrete_results[0].env})" if self._render_env else ""
    env = env.ljust(self._env_str_len + 4)
    output = ["  " + env + concrete_results[0].as_row_name]
    for m, col in zip(self._results, self._columns or ()):
        if m is None:
            output.append(col.num_to_str(None, 1, None))
        else:
            if len(m.times) == 1:
                spread = 0
            else:
                spread = float(torch.tensor(m.times, dtype=torch.float64).std(unbiased=len(m.times) > 1))
                if col._trim_significant_figures:
                    spread = benchmark.utils.common.trim_sigfig(spread, m.significant_figures)
            output.append(f"{m.median / self._time_scale:>3.3f} (+-{spread / self._time_scale:>3.3f})")
    return output


 class Value(common.Measurement): pass


 class CustomizedTable(Table):

    def __init__(self, results, colorize, trim_significant_figures, highlight_warnings):
        assert len(set(r.label for r in results)) == 1

        self.results = results
        self._colorize = colorize
        self._trim_significant_figures = trim_significant_figures
        self._highlight_warnings = highlight_warnings
        self.label = results[0].label
        self.time_unit, self.time_scale = common.select_unit(
            min(r.median for r in results if not isinstance(r, Value))
        )

        self.row_keys = common.ordered_unique([self.row_fn(i) for i in results])
        self.row_keys.sort(key=lambda args: args[:2])  # preserve stmt order
        self.column_keys = common.ordered_unique([self.col_fn(i) for i in results])
        self.rows, self.columns = self.populate_rows_and_columns()


 def get_new_table(compare, col1, col2, description, debug):
    results = common.Measurement.merge(compare._results)
    grouped_results = compare._group_by_label(results)
    assert len(grouped_results.values()) == 1, grouped_results.values()
    groups_iter = iter(grouped_results.values())
    group = next(groups_iter)

    if description is None:
        description = f"Speed-up: {col1} vs {col2}"

    # Add speed-up column into results:
    updated_group = []
    sub_label = None
    v1 = None
    v2 = None
    r = None
    _, scale = common.select_unit(min([r.median for r in group]))

    for measurement in group:
        if debug:
            print("measurement.task_spec.description:", measurement.task_spec.description)

        if measurement.task_spec.description == col1:
            v1 = measurement.median
            sub_label = measurement.task_spec.sub_label
            if debug:
                print("Matched col1:", col1, v1, sub_label)

        measurement2 = None
        for m2 in group:
            d2 = m2.task_spec.description
            sl2 = m2.task_spec.sub_label
            if d2 == col2 and sl2 == sub_label:
                v2 = m2.median
                if debug:
                    print("Matched col2:", col2, v2)
                measurement2 = m2
                break

        if measurement not in updated_group:
            updated_group.append(measurement)
        if v1 is not None and v2 is not None:
            if measurement2 not in updated_group:
                updated_group.append(measurement2)
            r = v2 / v1 * scale
            if debug:
                print("ratio is: ", r)
            v1 = None
            v2 = None
            sub_label = None
            speedup_task = common.TaskSpec(
                "",
                setup="",
                label=measurement.label,
                sub_label=measurement.sub_label,
                num_threads=measurement.num_threads,
                env=measurement.env,
                description=description
            )
            speedup_measurement = Value(1, [r, ], speedup_task)
            r = None
            updated_group.append(speedup_measurement)

    assert len(updated_group) > len(group), "Seems like nothing was added. Run with --debug"

    table = CustomizedTable(
        updated_group,
        compare._colorize,
        compare._trim_significant_figures,
        compare._highlight_warnings
    )
    return table


 def main(
    output_filepath: str,
    perf_files: List[str],
    *,
    col1: str,
    col2: str,
    description: Optional[str] = None,
    debug: bool = False
 ):
    output_filepath = Path(output_filepath)
    if output_filepath.exists():
        raise FileExistsError(f"Output file '{output_filepath}' exists. Please provide a path to non-existing file")

    if debug:
        print("output_filepath:", output_filepath)
        print("perf_files:", perf_files, type(perf_files))
        print("col1:", col1, type(col1))
        print("col2:", col2, type(col2))
        print("description:", description, type(description))

    ab_results = []
    ab_configs = []
    for perf_filepath in perf_files:
        assert Path(perf_filepath).exists(), f"{perf_filepath} is not found"
        with open(perf_filepath, "rb") as handler:
            output = pickle.load(handler)
            ab_configs.append(
                f"Torch version: {output['torch_version']}\n"
                f"Torch config: {output['torch_config']}\n"
            )
            ab_results.extend(output["test_results"])

    assert len(ab_configs) == len(perf_files), (len(ab_configs), len(perf_files))
    compare = benchmark.Compare(ab_results)

    table = get_new_table(compare, col1=col1, col2=col2, description=description, debug=debug)

    if debug:
        print(table.render())

    with output_filepath.open("w") as handler:
        handler.write(f"Description:\n")
        with unittest.mock.patch(
            "torch.utils.benchmark.utils.compare._Row.as_column_strings", patched_as_column_strings
        ):
            for in_filepath, config in zip(perf_files, ab_configs):
                handler.write(f"- {Path(in_filepath).stem}\n")
                handler.write(f"{config}\n")

            handler.write(f"\n")
            handler.write(table.render())


 if __name__ == "__main__":
    fire.Fire(main)
diff --git a/run_bench_interp.py b/run_bench_interp.py

 import pickle
 from pathlib import Path
 import unittest.mock


 import numpy as np
 import PIL.Image

 import torch
 import torch.utils.benchmark as benchmark

 import fire


 from torchvision_functional_tensor import resize


 def pth_downsample_i8(img, mode, size, aa=True):

    align_corners = False
    if mode == "nearest":
        align_corners = None

    out = torch.nn.functional.interpolate(
        img, size=size,
        mode=mode,
        align_corners=align_corners,
        antialias=aa,
    )
    return out


 def torchvision_resize(img, mode, size, aa=True):
    return resize(img, size=size, interpolation=mode, antialias=aa)


 if not hasattr(PIL.Image, "Resampling"):
    resampling_map = {
        "bilinear": PIL.Image.BILINEAR,
        "nearest": PIL.Image.NEAREST,
        "bicubic": PIL.Image.BICUBIC,
    }
 else:
    resampling_map = {
        "bilinear": PIL.Image.Resampling.BILINEAR,
        "nearest": PIL.Image.Resampling.NEAREST,
        "bicubic": PIL.Image.Resampling.BICUBIC,
    }


 def patched_as_column_strings(self):
    concrete_results = [r for r in self._results if r is not None]
    env = f"({concrete_results[0].env})" if self._render_env else ""
    env = env.ljust(self._env_str_len + 4)
    output = ["  " + env + concrete_results[0].as_row_name]
    for m, col in zip(self._results, self._columns or ()):
        if m is None:
            output.append(col.num_to_str(None, 1, None))
        else:
            if len(m.times) == 1:
                spread = 0
            else:
                spread = float(torch.tensor(m.times, dtype=torch.float64).std(unbiased=len(m.times) > 1))
                if col._trim_significant_figures:
                    spread = benchmark.utils.common.trim_sigfig(spread, m.significant_figures)
            output.append(f"{m.median / self._time_scale:>3.3f} (+-{spread / self._time_scale:>3.3f})")
    return output


 def run_benchmark(c, dtype, size, osize, aa, mode, mf="channels_first", min_run_time=10, tag="", with_torchvision=False):
    results = []
    torch.manual_seed(12)

    if dtype == torch.bool:
        tensor = torch.randint(0, 2, size=(c, size[0], size[1]), dtype=dtype)
    elif dtype == torch.complex64:
        real = torch.randint(0, 256, size=(c, size[0], size[1]), dtype=torch.float32)
        imag = torch.randint(0, 256, size=(c, size[0], size[1]), dtype=torch.float32)
        tensor = torch.complex(real, imag)
    elif dtype == torch.int8:
        tensor = torch.randint(-127, 127, size=(c, size[0], size[1]), dtype=dtype)
    else:
        tensor = torch.randint(0, 256, size=(c, size[0], size[1]), dtype=dtype)

    expected_pil = None
    pil_img = None
    if dtype == torch.uint8 and c == 3 and aa:
        np_array = tensor.clone().permute(1, 2, 0).contiguous().numpy()
        pil_img = PIL.Image.fromarray(np_array)
        output_pil_img = pil_img.resize(osize[::-1], resample=resampling_map[mode])
        expected_pil = torch.from_numpy(np.asarray(output_pil_img)).clone().permute(2, 0, 1).contiguous()

    memory_format = torch.channels_last if mf == "channels_last" else torch.contiguous_format
    tensor = tensor[None, ...].contiguous(memory_format=memory_format)

    output = pth_downsample_i8(tensor, mode=mode, size=osize, aa=aa)
    output = output[0, ...]

    if expected_pil is not None:
        abs_diff = torch.abs(expected_pil.float() - output.float())
        mae = torch.mean(abs_diff)
        max_abs_err = torch.max(abs_diff)

        if mode == "bilinear":
            assert mae.item() < 1.0, mae.item()
            assert max_abs_err.item() < 2.0 + 1e-5, max_abs_err.item()
        else:
            raise RuntimeError(f"Unsupported mode: {mode}")

    # PIL
    if pil_img is not None:
        results.append(
            benchmark.Timer(
                # pil_img = pil_img.resize((osize, osize), resample=resampling_map[mode])
                stmt=f"data.resize({osize[::-1]}, resample=resample_val)",
                globals={
                    "data": pil_img,
                    "resample_val": resampling_map[mode],
                },
                num_threads=torch.get_num_threads(),
                label="Resize",
                sub_label=f"{c} {dtype} {mf} {mode} {size} -> {osize} aa={aa}",
                description=f"Pillow ({PIL.__version__})",
            ).blocked_autorange(min_run_time=min_run_time)
        )
    # Tensor interp
    results.append(
        benchmark.Timer(
            # output = pth_downsample_i8(tensor, mode=mode, size=(osize, osize), aa=aa)
            stmt=f"fn(data, mode='{mode}', size={osize}, aa={aa})",
            globals={
                "data": tensor,
                "fn": pth_downsample_i8
            },
            num_threads=torch.get_num_threads(),
            label="Resize",
            sub_label=f"{c} {dtype} {mf} {mode} {size} -> {osize} aa={aa}",
            description=f"torch ({torch.__version__}) {tag}",
        ).blocked_autorange(min_run_time=min_run_time)
    )
    # Torchvision resize
    if with_torchvision:
        results.append(
            benchmark.Timer(
                # output = torchvision_resize(tensor, mode=mode, size=(osize, osize), aa=aa)
                stmt=f"fn(data, mode='{mode}', size={osize}, aa={aa})",
                globals={
                    "data": tensor,
                    "fn": torchvision_resize
                },
                num_threads=torch.get_num_threads(),
                label="Resize",
                sub_label=f"{c} {dtype} {mf} {mode} {size} -> {osize} aa={aa}",
                description=f"torchvision resize",
            ).blocked_autorange(min_run_time=min_run_time)
        )

    return results


 def main(
    output_filepath: str,
    min_run_time: int = 10,
    tag: str = "",
    display: bool = True,
    with_torchvision: bool = False,
    extended_test_cases=True
 ):

    output_filepath = Path(output_filepath)

    test_results = []
    for mf in ["channels_last", "channels_first"]:
        for c, dtype in [
            (3, torch.uint8),
            (4, torch.uint8),
        ]:
            for size in [256, 520, 712]:
                if isinstance(size, int):
                    size = (size, size)

                osize_aa_mode_list = [
                    (32, True, "bilinear"),
                    (32, False, "bilinear"),
                    (224, True, "bilinear"),
                    (224, False, "bilinear"),
                ]

                if size == (256, 256):
                    osize_aa_mode_list += [
                        (320, True, "bilinear"),
                        (320, False, "bilinear"),
                    ]

                for osize, aa, mode in osize_aa_mode_list:
                    if isinstance(osize, int):
                        osize = (osize, osize)

                    test_results += run_benchmark(
                        c=c, dtype=dtype, size=size,
                        osize=osize, aa=aa, mode=mode, mf=mf,
                        min_run_time=min_run_time, tag=tag, with_torchvision=with_torchvision
                    )

            if not extended_test_cases:
                continue

            for aa in [True, False]:
                mode = "bilinear"

                size_osize_list = [
                    (64, 224),
                    (224, (270, 268)),
                    (256, (1024, 1024)),
                    (224, 64),
                    ((270, 268), 224),
                    (1024, 256),
                ]

                for size, osize in size_osize_list:
                    if isinstance(size, int):
                        size = (size, size)

                    if isinstance(osize, int):
                        osize = (osize, osize)

                    test_results += run_benchmark(
                        c=c, dtype=dtype, size=size,
                        osize=osize, aa=aa, mode=mode, mf=mf,
                        min_run_time=min_run_time, tag=tag, with_torchvision=with_torchvision
                    )

    with open(output_filepath, "wb") as handler:
        output = {
            "torch_version": torch.__version__,
            "torch_config": torch.__config__.show(),
            "num_threads": torch.get_num_threads(),
            "pil_version": PIL.__version__,
            "test_results": test_results,
        }
        pickle.dump(output, handler)

    if display:
        with unittest.mock.patch(
            "torch.utils.benchmark.utils.compare._Row.as_column_strings", patched_as_column_strings
        ):
            compare = benchmark.Compare(test_results)
            compare.print()


 if __name__ == "__main__":

    torch.set_num_threads(1)

    from datetime import datetime

    print(f"Timestamp: {datetime.now().strftime('%Y%m%d-%H%M%S')}")
    print(f"Torch version: {torch.__version__}")
    print(f"Torch config: {torch.__config__.show()}")
    print(f"Num threads: {torch.get_num_threads()}")
    print("")
    print("PIL version: ", PIL.__version__)

    fire.Fire(main)
	import pickle
	from pathlib import Path
	from typing import List, Optional
	import unittest.mock

	import torch
	import torch.utils.benchmark as benchmark
	from torch.utils.benchmark.utils import common
	from torch.utils.benchmark.utils.compare import Table


	import fire


	def patched_as_column_strings(self):
	concrete_results = [r for r in self._results if r is not None]
	env = f"({concrete_results[0].env})" if self._render_env else ""
	env = env.ljust(self._env_str_len + 4)
	output = [" " + env + concrete_results[0].as_row_name]
	for m, col in zip(self._results, self._columns or ()):
	if m is None:
	output.append(col.num_to_str(None, 1, None))
	else:
	if len(m.times) == 1:
	spread = 0
	else:
	spread = float(torch.tensor(m.times, dtype=torch.float64).std(unbiased=len(m.times) > 1))
	if col._trim_significant_figures:
	spread = benchmark.utils.common.trim_sigfig(spread, m.significant_figures)
	output.append(f"{m.median / self._time_scale:>3.3f} (+-{spread / self._time_scale:>3.3f})")
	return output


	class Value(common.Measurement): pass


	class CustomizedTable(Table):

	def __init__(self, results, colorize, trim_significant_figures, highlight_warnings):
	assert len(set(r.label for r in results)) == 1

	self.results = results
	self._colorize = colorize
	self._trim_significant_figures = trim_significant_figures
	self._highlight_warnings = highlight_warnings
	self.label = results[0].label
	self.time_unit, self.time_scale = common.select_unit(
	min(r.median for r in results if not isinstance(r, Value))
	)

	self.row_keys = common.ordered_unique([self.row_fn(i) for i in results])
	self.row_keys.sort(key=lambda args: args[:2]) # preserve stmt order
	self.column_keys = common.ordered_unique([self.col_fn(i) for i in results])
	self.rows, self.columns = self.populate_rows_and_columns()


	def get_new_table(compare, col1, col2, description, debug):
	results = common.Measurement.merge(compare._results)
	grouped_results = compare._group_by_label(results)
	assert len(grouped_results.values()) == 1, grouped_results.values()
	groups_iter = iter(grouped_results.values())
	group = next(groups_iter)

	if description is None:
	description = f"Speed-up: {col1} vs {col2}"

	# Add speed-up column into results:
	updated_group = []
	sub_label = None
	v1 = None
	v2 = None
	r = None
	_, scale = common.select_unit(min([r.median for r in group]))

	for measurement in group:
	if debug:
	print("measurement.task_spec.description:", measurement.task_spec.description)

	if measurement.task_spec.description == col1:
	v1 = measurement.median
	sub_label = measurement.task_spec.sub_label
	if debug:
	print("Matched col1:", col1, v1, sub_label)

	measurement2 = None
	for m2 in group:
	d2 = m2.task_spec.description
	sl2 = m2.task_spec.sub_label
	if d2 == col2 and sl2 == sub_label:
	v2 = m2.median
	if debug:
	print("Matched col2:", col2, v2)
	measurement2 = m2
	break

	if measurement not in updated_group:
	updated_group.append(measurement)
	if v1 is not None and v2 is not None:
	if measurement2 not in updated_group:
	updated_group.append(measurement2)
	r = v2 / v1 * scale
	if debug:
	print("ratio is: ", r)
	v1 = None
	v2 = None
	sub_label = None
	speedup_task = common.TaskSpec(
	"",
	setup="",
	label=measurement.label,
	sub_label=measurement.sub_label,
	num_threads=measurement.num_threads,
	env=measurement.env,
	description=description
	)
	speedup_measurement = Value(1, [r, ], speedup_task)
	r = None
	updated_group.append(speedup_measurement)

	assert len(updated_group) > len(group), "Seems like nothing was added. Run with --debug"

	table = CustomizedTable(
	updated_group,
	compare._colorize,
	compare._trim_significant_figures,
	compare._highlight_warnings
	)
	return table


	def main(
	output_filepath: str,
	perf_files: List[str],
	*,
	col1: str,
	col2: str,
	description: Optional[str] = None,
	debug: bool = False
	):
	output_filepath = Path(output_filepath)
	if output_filepath.exists():
	raise FileExistsError(f"Output file '{output_filepath}' exists. Please provide a path to non-existing file")

	if debug:
	print("output_filepath:", output_filepath)
	print("perf_files:", perf_files, type(perf_files))
	print("col1:", col1, type(col1))
	print("col2:", col2, type(col2))
	print("description:", description, type(description))

	ab_results = []
	ab_configs = []
	for perf_filepath in perf_files:
	assert Path(perf_filepath).exists(), f"{perf_filepath} is not found"
	with open(perf_filepath, "rb") as handler:
	output = pickle.load(handler)
	ab_configs.append(
	f"Torch version: {output['torch_version']}\n"
	f"Torch config: {output['torch_config']}\n"
	)
	ab_results.extend(output["test_results"])

	assert len(ab_configs) == len(perf_files), (len(ab_configs), len(perf_files))
	compare = benchmark.Compare(ab_results)

	table = get_new_table(compare, col1=col1, col2=col2, description=description, debug=debug)

	if debug:
	print(table.render())

	with output_filepath.open("w") as handler:
	handler.write(f"Description:\n")
	with unittest.mock.patch(
	"torch.utils.benchmark.utils.compare._Row.as_column_strings", patched_as_column_strings
	):
	for in_filepath, config in zip(perf_files, ab_configs):
	handler.write(f"- {Path(in_filepath).stem}\n")
	handler.write(f"{config}\n")

	handler.write(f"\n")
	handler.write(table.render())


	if __name__ == "__main__":
	fire.Fire(main)